diff --git a/main.py b/main.py index 00c20f5..f05bf3a 100644 --- a/main.py +++ b/main.py @@ -9,38 +9,64 @@ import sys import time from typing import Optional -sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) +plugin_dir = os.path.dirname(os.path.abspath(__file__)) +py_modules_dir = os.path.join(plugin_dir, "py_modules") + +sys.path.insert(0, py_modules_dir) +sys.path.insert(0, plugin_dir) + +def _ensure_deps(): + try: + import cryptography # noqa: F401 + return + except ImportError: + pass + + import importlib.util + cffi_path = os.path.join(py_modules_dir, "_cffi_backend.cpython-311-x86_64-linux-gnu.so") + if os.path.exists(cffi_path) and "_cffi_backend" not in sys.modules: + spec = importlib.util.spec_from_file_location("_cffi_backend", cffi_path) + mod = importlib.util.module_from_spec(spec) + sys.modules["_cffi_backend"] = mod + spec.loader.exec_module(mod) + +_ensure_deps() import decky_plugin -from bitwarden_client import BitwardenClient -from crypto import BitwardenCrypto, PinCrypto -from totp import generate_totp, get_totp_remaining_seconds -from clipboard import copy_to_clipboard, clear_all_clipboards class Plugin: """Main plugin class for Decky Vaultwarden.""" def __init__(self): - self.client: Optional[BitwardenClient] = None - self.pin_crypto = PinCrypto() - self.crypto = BitwardenCrypto() - self._vault_data: Optional[dict] = None - self._decrypted_vault: Optional[dict] = None - self._master_key: Optional[bytes] = None - self._enc_key: Optional[bytes] = None - self._mac_key: Optional[bytes] = None - self._session_active: bool = False + self.client = None + self.client_class = None + self.pin_crypto = None + self.crypto = None + self._vault_data = None + self._decrypted_vault = None + self._master_key = None + self._enc_key = None + self._mac_key = None + self._session_active = False self._settings_path = os.path.join( os.path.expanduser("~"), ".config", "decky-vaultwarden" ) self._pin_attempts = 0 self._MAX_PIN_ATTEMPTS = 5 + self._deps_ready = False # ===== Lifecycle Methods ===== async def _main(self): """Called when the plugin is loaded.""" + from bitwarden_client import BitwardenClient + from crypto import BitwardenCrypto, PinCrypto + + self.client_class = BitwardenClient + self.pin_crypto = PinCrypto() + self.crypto = BitwardenCrypto() + self._deps_ready = True decky_plugin.logger.info("Decky Vaultwarden plugin loaded") os.makedirs(self._settings_path, exist_ok=True) @@ -60,7 +86,7 @@ class Plugin: ) -> dict: """Login with email and master password.""" try: - self.client = BitwardenClient(server_url) + self.client = self.client_class(server_url) result = await self.client.login_password(email, password, two_factor_token) if result.get("two_factor_required"): @@ -88,7 +114,7 @@ class Plugin: ) -> dict: """Login with API key.""" try: - self.client = BitwardenClient(server_url) + self.client = self.client_class(server_url) result = await self.client.login_api_key(client_id, client_secret, email) if result.get("success"): @@ -197,6 +223,7 @@ class Plugin: async def copy_password(self, cipher_id: str) -> dict: """Copy a password to clipboard with 60s auto-clear.""" + from clipboard import copy_to_clipboard password = self._get_cipher_field(cipher_id, "password") if password is None: return {"success": False, "error": "Password not found"} @@ -204,6 +231,7 @@ class Plugin: async def copy_username(self, cipher_id: str) -> dict: """Copy a username to clipboard with 60s auto-clear.""" + from clipboard import copy_to_clipboard username = self._get_cipher_field(cipher_id, "username") if username is None: return {"success": False, "error": "Username not found"} @@ -211,6 +239,8 @@ class Plugin: async def copy_totp(self, cipher_id: str) -> dict: """Copy TOTP code to clipboard with 60s auto-clear.""" + from clipboard import copy_to_clipboard + from totp import generate_totp, get_totp_remaining_seconds totp_secret = self._get_cipher_field(cipher_id, "totp") if totp_secret: code = generate_totp(totp_secret) @@ -222,6 +252,7 @@ class Plugin: async def get_totp_code(self, cipher_id: str) -> dict: """Get current TOTP code without copying.""" + from totp import generate_totp, get_totp_remaining_seconds totp_secret = self._get_cipher_field(cipher_id, "totp") if totp_secret: code = generate_totp(totp_secret) @@ -346,6 +377,7 @@ class Plugin: async def lock_vault(self) -> dict: """Lock the vault and clear sensitive data.""" + from clipboard import clear_all_clipboards await clear_all_clipboards() self._decrypted_vault = None self._vault_data = None diff --git a/package.json b/package.json index 74013f2..dac6537 100644 --- a/package.json +++ b/package.json @@ -4,7 +4,8 @@ "description": "Bitwarden/Vaultwarden password manager plugin for Decky Loader", "type": "module", "scripts": { - "build": "rollup -c && mkdir -p out && rm -rf out/staging && mkdir -p out/staging/Vaultwarden && cp -r dist main.py bitwarden_client.py crypto.py totp.py clipboard.py plugin.json defaults requirements.txt package.json out/staging/Vaultwarden/ && cd out/staging && zip -r ../Vaultwarden.zip Vaultwarden/ && rm -rf out/staging", + "bundle-deps": "bash scripts/bundle-deps.sh", + "build": "rollup -c && mkdir -p out && rm -rf out/staging && mkdir -p out/staging/Vaultwarden && cp -r dist main.py bitwarden_client.py crypto.py totp.py clipboard.py plugin.json defaults requirements.txt package.json py_modules out/staging/Vaultwarden/ && cd out/staging && zip -r ../Vaultwarden.zip Vaultwarden/ && rm -rf out/staging", "watch": "rollup -c -w", "typecheck": "tsc --noEmit" }, diff --git a/plugin.json b/plugin.json index 7312ae6..54e66c8 100644 --- a/plugin.json +++ b/plugin.json @@ -1,7 +1,7 @@ { "name": "Vaultwarden", "author": "JustZoe101", - "flags": ["needs_community_python"], + "flags": [], "api_version": 1, "content": { "name": "Vaultwarden - Password Manager", diff --git a/py_modules/_argon2_cffi_bindings/__init__.py b/py_modules/_argon2_cffi_bindings/__init__.py new file mode 100644 index 0000000..8def07a --- /dev/null +++ b/py_modules/_argon2_cffi_bindings/__init__.py @@ -0,0 +1,6 @@ +# SPDX-License-Identifier: MIT + +from ._ffi import ffi, lib + + +__all__ = ["ffi", "lib"] diff --git a/py_modules/_argon2_cffi_bindings/_ffi.abi3.so b/py_modules/_argon2_cffi_bindings/_ffi.abi3.so new file mode 100755 index 0000000..79ffef0 Binary files /dev/null and b/py_modules/_argon2_cffi_bindings/_ffi.abi3.so differ diff --git a/py_modules/_argon2_cffi_bindings/_ffi_build.py b/py_modules/_argon2_cffi_bindings/_ffi_build.py new file mode 100644 index 0000000..0d4d0be --- /dev/null +++ b/py_modules/_argon2_cffi_bindings/_ffi_build.py @@ -0,0 +1,228 @@ +# SPDX-License-Identifier: MIT + +import os +import platform + +from pathlib import Path + +from cffi import FFI + + +use_system_argon2 = os.environ.get("ARGON2_CFFI_USE_SYSTEM", "0") == "1" +use_sse2 = os.environ.get("ARGON2_CFFI_USE_SSE2", None) +windows = platform.system() == "Windows" + + +# Try to detect cross-compilation. +def _get_target_platform(arch_flags, default): + flags = [f for f in arch_flags.split(" ") if f.strip() != ""] + try: + pos = flags.index("-arch") + + return flags[pos + 1].lower() + except ValueError: + pass + + return default + + +target_platform = _get_target_platform( + os.environ.get("ARCHFLAGS", ""), platform.machine() +) + + +if use_sse2 == "1": + optimized = True +elif use_sse2 == "0": + optimized = False +else: + # Optimized version requires SSE2 extensions. They have been around since + # 2001 so we try to compile it on every recent-ish x86. + optimized = target_platform in ("i686", "x86", "x86_64", "AMD64") + + +ffi = FFI() + +if use_system_argon2: + ffi.set_source( + "_ffi", + "#include ", + libraries=["argon2"], + ) +else: + lib_base = Path("extras") / "libargon2" / "src" + ffi.set_source( + "_ffi", + "#include ", + extra_compile_args=["-msse2"] if (optimized and not windows) else None, + include_dirs=[os.path.join("extras", "libargon2", "include")], + sources=[ + str(lib_base / path) + for path in [ + "argon2.c", + Path("blake2") / "blake2b.c", + "core.c", + "encoding.c", + "opt.c" if optimized else "ref.c", + "thread.c", + ] + ], + ) + + +ffi.cdef( + """\ +typedef enum Argon2_type { + Argon2_d = ..., + Argon2_i = ..., + Argon2_id = ..., +} argon2_type; +typedef enum Argon2_version { + ARGON2_VERSION_10 = ..., + ARGON2_VERSION_13 = ..., + ARGON2_VERSION_NUMBER = ... +} argon2_version; + +int argon2_hash(const uint32_t t_cost, const uint32_t m_cost, + const uint32_t parallelism, const void *pwd, + const size_t pwdlen, const void *salt, + const size_t saltlen, void *hash, + const size_t hashlen, char *encoded, + const size_t encodedlen, argon2_type type, + const uint32_t version); + +int argon2_verify(const char *encoded, const void *pwd, + const size_t pwdlen, argon2_type type); + +const char *argon2_error_message(int error_code); + + +typedef int (*allocate_fptr)(uint8_t **memory, size_t bytes_to_allocate); +typedef void (*deallocate_fptr)(uint8_t *memory, size_t bytes_to_allocate); + +typedef struct Argon2_Context { + uint8_t *out; /* output array */ + uint32_t outlen; /* digest length */ + + uint8_t *pwd; /* password array */ + uint32_t pwdlen; /* password length */ + + uint8_t *salt; /* salt array */ + uint32_t saltlen; /* salt length */ + + uint8_t *secret; /* key array */ + uint32_t secretlen; /* key length */ + + uint8_t *ad; /* associated data array */ + uint32_t adlen; /* associated data length */ + + uint32_t t_cost; /* number of passes */ + uint32_t m_cost; /* amount of memory requested (KB) */ + uint32_t lanes; /* number of lanes */ + uint32_t threads; /* maximum number of threads */ + + uint32_t version; /* version number */ + + allocate_fptr allocate_cbk; /* pointer to memory allocator */ + deallocate_fptr free_cbk; /* pointer to memory deallocator */ + + uint32_t flags; /* array of bool options */ +} argon2_context; + +int argon2_ctx(argon2_context *context, argon2_type type); + +/* Error codes */ +typedef enum Argon2_ErrorCodes { + ARGON2_OK = ..., + + ARGON2_OUTPUT_PTR_NULL = ..., + + ARGON2_OUTPUT_TOO_SHORT = ..., + ARGON2_OUTPUT_TOO_LONG = ..., + + ARGON2_PWD_TOO_SHORT = ..., + ARGON2_PWD_TOO_LONG = ..., + + ARGON2_SALT_TOO_SHORT = ..., + ARGON2_SALT_TOO_LONG = ..., + + ARGON2_AD_TOO_SHORT = ..., + ARGON2_AD_TOO_LONG = ..., + + ARGON2_SECRET_TOO_SHORT = ..., + ARGON2_SECRET_TOO_LONG = ..., + + ARGON2_TIME_TOO_SMALL = ..., + ARGON2_TIME_TOO_LARGE = ..., + + ARGON2_MEMORY_TOO_LITTLE = ..., + ARGON2_MEMORY_TOO_MUCH = ..., + + ARGON2_LANES_TOO_FEW = ..., + ARGON2_LANES_TOO_MANY = ..., + + ARGON2_PWD_PTR_MISMATCH = ..., /* NULL ptr with non-zero length */ + ARGON2_SALT_PTR_MISMATCH = ..., /* NULL ptr with non-zero length */ + ARGON2_SECRET_PTR_MISMATCH = ..., /* NULL ptr with non-zero length */ + ARGON2_AD_PTR_MISMATCH = ..., /* NULL ptr with non-zero length */ + + ARGON2_MEMORY_ALLOCATION_ERROR = ..., + + ARGON2_FREE_MEMORY_CBK_NULL = ..., + ARGON2_ALLOCATE_MEMORY_CBK_NULL = ..., + + ARGON2_INCORRECT_PARAMETER = ..., + ARGON2_INCORRECT_TYPE = ..., + + ARGON2_OUT_PTR_MISMATCH = ..., + + ARGON2_THREADS_TOO_FEW = ..., + ARGON2_THREADS_TOO_MANY = ..., + + ARGON2_MISSING_ARGS = ..., + + ARGON2_ENCODING_FAIL = ..., + + ARGON2_DECODING_FAIL = ..., + + ARGON2_THREAD_FAIL = ..., + + ARGON2_DECODING_LENGTH_FAIL= ..., + + ARGON2_VERIFY_MISMATCH = ..., +} argon2_error_codes; + +#define ARGON2_FLAG_CLEAR_PASSWORD ... +#define ARGON2_FLAG_CLEAR_SECRET ... +#define ARGON2_DEFAULT_FLAGS ... + +#define ARGON2_MIN_LANES ... +#define ARGON2_MAX_LANES ... +#define ARGON2_MIN_THREADS ... +#define ARGON2_MAX_THREADS ... +#define ARGON2_SYNC_POINTS ... +#define ARGON2_MIN_OUTLEN ... +#define ARGON2_MAX_OUTLEN ... +#define ARGON2_MIN_MEMORY ... +#define ARGON2_MAX_MEMORY_BITS ... +#define ARGON2_MAX_MEMORY ... +#define ARGON2_MIN_TIME ... +#define ARGON2_MAX_TIME ... +#define ARGON2_MIN_PWD_LENGTH ... +#define ARGON2_MAX_PWD_LENGTH ... +#define ARGON2_MIN_AD_LENGTH ... +#define ARGON2_MAX_AD_LENGTH ... +#define ARGON2_MIN_SALT_LENGTH ... +#define ARGON2_MAX_SALT_LENGTH ... +#define ARGON2_MIN_SECRET ... +#define ARGON2_MAX_SECRET ... + +uint32_t argon2_encodedlen(uint32_t t_cost, uint32_t m_cost, + uint32_t parallelism, uint32_t saltlen, + uint32_t hashlen, argon2_type type); + +""" +) + +if __name__ == "__main__": + ffi.compile() diff --git a/py_modules/_cffi_backend.cpython-311-x86_64-linux-gnu.so b/py_modules/_cffi_backend.cpython-311-x86_64-linux-gnu.so new file mode 100755 index 0000000..d487bf9 Binary files /dev/null and b/py_modules/_cffi_backend.cpython-311-x86_64-linux-gnu.so differ diff --git a/py_modules/aiohappyeyeballs/__init__.py b/py_modules/aiohappyeyeballs/__init__.py new file mode 100644 index 0000000..d7b322d --- /dev/null +++ b/py_modules/aiohappyeyeballs/__init__.py @@ -0,0 +1,14 @@ +__version__ = "2.7.1" + +from .impl import start_connection +from .types import AddrInfoType, SocketFactoryType +from .utils import addr_to_addr_infos, pop_addr_infos_interleave, remove_addr_infos + +__all__ = ( + "AddrInfoType", + "SocketFactoryType", + "addr_to_addr_infos", + "pop_addr_infos_interleave", + "remove_addr_infos", + "start_connection", +) diff --git a/py_modules/aiohappyeyeballs/_staggered.py b/py_modules/aiohappyeyeballs/_staggered.py new file mode 100644 index 0000000..bb745dc --- /dev/null +++ b/py_modules/aiohappyeyeballs/_staggered.py @@ -0,0 +1,197 @@ +import asyncio +import contextlib +from collections.abc import Awaitable, Callable, Iterable +from typing import ( + TYPE_CHECKING, + Any, + TypeVar, +) + +_T = TypeVar("_T") + +RE_RAISE_EXCEPTIONS = (SystemExit, KeyboardInterrupt) + + +def _set_result(wait_next: "asyncio.Future[None]") -> None: + """Set the result of a future if it is not already done.""" + if not wait_next.done(): + wait_next.set_result(None) + + +async def _wait_one( + futures: "Iterable[asyncio.Future[Any]]", + loop: asyncio.AbstractEventLoop, +) -> _T: + """Wait for the first future to complete.""" + wait_next = loop.create_future() + + def _on_completion(fut: "asyncio.Future[Any]") -> None: + if not wait_next.done(): + wait_next.set_result(fut) + + for f in futures: + f.add_done_callback(_on_completion) + + try: + return await wait_next + finally: + for f in futures: + f.remove_done_callback(_on_completion) + + +async def staggered_race( + coro_fns: Iterable[Callable[[], Awaitable[_T]]], + delay: float | None, + *, + loop: asyncio.AbstractEventLoop | None = None, +) -> tuple[_T | None, int | None, list[BaseException | None]]: + """ + Run coroutines with staggered start times and take the first to finish. + + This method takes an iterable of coroutine functions. The first one is + started immediately. From then on, whenever the immediately preceding one + fails (raises an exception), or when *delay* seconds has passed, the next + coroutine is started. This continues until one of the coroutines complete + successfully, in which case all others are cancelled, or until all + coroutines fail. + + The coroutines provided should be well-behaved in the following way: + + * They should only ``return`` if completed successfully. + + * They should always raise an exception if they did not complete + successfully. In particular, if they handle cancellation, they should + probably reraise, like this:: + + try: + # do work + except asyncio.CancelledError: + # undo partially completed work + raise + + Args: + ---- + coro_fns: an iterable of coroutine functions, i.e. callables that + return a coroutine object when called. Use ``functools.partial`` or + lambdas to pass arguments. + + delay: amount of time, in seconds, between starting coroutines. If + ``None``, the coroutines will run sequentially. + + loop: the event loop to use. If ``None``, the running loop is used. + + Returns: + ------- + tuple *(winner_result, winner_index, exceptions)* where + + - *winner_result*: the result of the winning coroutine, or ``None`` + if no coroutines won. + + - *winner_index*: the index of the winning coroutine in + ``coro_fns``, or ``None`` if no coroutines won. If the winning + coroutine may return None on success, *winner_index* can be used + to definitively determine whether any coroutine won. + + - *exceptions*: list of exceptions returned by the coroutines. + ``len(exceptions)`` is equal to the number of coroutines actually + started, and the order is the same as in ``coro_fns``. The winning + coroutine's entry is ``None``. + + """ + loop = loop or asyncio.get_running_loop() + exceptions: list[BaseException | None] = [] + tasks: set[asyncio.Task[tuple[_T, int] | None]] = set() + + async def run_one_coro( + coro_fn: Callable[[], Awaitable[_T]], + this_index: int, + start_next: "asyncio.Future[None]", + ) -> tuple[_T, int] | None: + """ + Run a single coroutine. + + If the coroutine fails, set the exception in the exceptions list and + start the next coroutine by setting the result of the start_next. + + If the coroutine succeeds, return the result and the index of the + coroutine in the coro_fns list. + + If SystemExit or KeyboardInterrupt is raised, re-raise it. + """ + try: + result = await coro_fn() + except RE_RAISE_EXCEPTIONS: + raise + except BaseException as e: + exceptions[this_index] = e + _set_result(start_next) # Kickstart the next coroutine + return None + + return result, this_index + + start_next_timer: asyncio.TimerHandle | None = None + start_next: asyncio.Future[None] | None + task: asyncio.Task[tuple[_T, int] | None] + done: asyncio.Future[None] | asyncio.Task[tuple[_T, int] | None] + coro_iter = iter(coro_fns) + this_index = -1 + try: + while True: + if coro_fn := next(coro_iter, None): + this_index += 1 + exceptions.append(None) + start_next = loop.create_future() + task = loop.create_task(run_one_coro(coro_fn, this_index, start_next)) + tasks.add(task) + start_next_timer = ( + loop.call_later(delay, _set_result, start_next) if delay else None + ) + elif not tasks: + # We exhausted the coro_fns list and no tasks are running + # so we have no winner and all coroutines failed. + break + + while tasks or start_next: + done = await _wait_one( + (*tasks, start_next) if start_next else tasks, loop + ) + if done is start_next: + # The current task has failed or the timer has expired + # so we need to start the next task. + start_next = None + if start_next_timer: + start_next_timer.cancel() + start_next_timer = None + + # Break out of the task waiting loop to start the next + # task. + break + + if TYPE_CHECKING: + assert isinstance(done, asyncio.Task) + + tasks.remove(done) + if winner := done.result(): + return *winner, exceptions + finally: + # We either have: + # - a winner + # - all tasks failed + # - a KeyboardInterrupt or SystemExit. + + # + # If the timer is still running, cancel it. + # + if start_next_timer: + start_next_timer.cancel() + + # + # If there are any tasks left, cancel them and than + # wait them so they fill the exceptions list. + # + for task in tasks: + task.cancel() + with contextlib.suppress(asyncio.CancelledError): + await task + + return None, None, exceptions diff --git a/py_modules/aiohappyeyeballs/impl.py b/py_modules/aiohappyeyeballs/impl.py new file mode 100644 index 0000000..9544ea3 --- /dev/null +++ b/py_modules/aiohappyeyeballs/impl.py @@ -0,0 +1,247 @@ +"""Base implementation.""" + +import asyncio +import contextlib +import functools +import itertools +import socket +from collections import defaultdict +from collections.abc import Sequence + +from . import _staggered +from .types import AddrInfoType, SocketFactoryType + + +async def start_connection( + addr_infos: Sequence[AddrInfoType], + *, + local_addr_infos: Sequence[AddrInfoType] | None = None, + happy_eyeballs_delay: float | None = None, + interleave: int | None = None, + loop: asyncio.AbstractEventLoop | None = None, + socket_factory: SocketFactoryType | None = None, +) -> socket.socket: + """ + Connect to a TCP server. + + Create a socket connection to a specified destination. The + destination is specified as a list of AddrInfoType tuples as + returned from getaddrinfo(). + + The arguments are, in order: + + * ``family``: the address family, e.g. ``socket.AF_INET`` or + ``socket.AF_INET6``. + * ``type``: the socket type, e.g. ``socket.SOCK_STREAM`` or + ``socket.SOCK_DGRAM``. + * ``proto``: the protocol, e.g. ``socket.IPPROTO_TCP`` or + ``socket.IPPROTO_UDP``. + * ``canonname``: the canonical name of the address, e.g. + ``"www.python.org"``. + * ``sockaddr``: the socket address + + This method is a coroutine which will try to establish the connection + in the background. When successful, the coroutine returns a + socket. + + The expected use case is to use this method in conjunction with + loop.create_connection() to establish a connection to a server:: + + socket = await start_connection(addr_infos) + transport, protocol = await loop.create_connection( + MyProtocol, sock=socket, ...) + """ + if not addr_infos: + raise ValueError("addr_infos must not be empty") + + current_loop = loop or asyncio.get_running_loop() + + single_addr_info = len(addr_infos) == 1 + + if happy_eyeballs_delay is not None and interleave is None: + # If using happy eyeballs, default to interleave addresses by family + interleave = 1 + + if interleave and not single_addr_info: + addr_infos = _interleave_addrinfos(addr_infos, interleave) + + sock: socket.socket | None = None + # uvloop can raise RuntimeError instead of OSError + exceptions: list[list[OSError | RuntimeError]] = [] + if happy_eyeballs_delay is None or single_addr_info: + # not using happy eyeballs + for addrinfo in addr_infos: + try: + sock = await _connect_sock( + current_loop, + exceptions, + addrinfo, + local_addr_infos, + None, + socket_factory, + ) + break + except (RuntimeError, OSError): + continue + else: # using happy eyeballs + open_sockets: set[socket.socket] = set() + try: + sock, _, _ = await _staggered.staggered_race( + ( + functools.partial( + _connect_sock, + current_loop, + exceptions, + addrinfo, + local_addr_infos, + open_sockets, + socket_factory, + ) + for addrinfo in addr_infos + ), + happy_eyeballs_delay, + ) + finally: + # If we have a winner, staggered_race will + # cancel the other tasks, however there is a + # small race window where any of the other tasks + # can be done before they are cancelled which + # will leave the socket open. To avoid this problem + # we pass a set to _connect_sock to keep track of + # the open sockets and close them here if there + # are any "runner up" sockets. + for s in open_sockets: + if s is not sock: + with contextlib.suppress(OSError): + s.close() + open_sockets = None # type: ignore[assignment] + + if sock is None: + all_exceptions = [exc for sub in exceptions for exc in sub] + try: + first_exception = all_exceptions[0] + if len(all_exceptions) == 1: + raise first_exception + else: + # If they all have the same str(), raise one. + model = str(first_exception) + if all(str(exc) == model for exc in all_exceptions): + raise first_exception + # Raise a combined exception so the user can see all + # the various error messages. + msg = "Multiple exceptions: {}".format( + ", ".join(str(exc) for exc in all_exceptions) + ) + # If the errno is the same for all exceptions, raise + # an OSError with that errno. + if isinstance(first_exception, OSError): + first_errno = first_exception.errno + if all( + isinstance(exc, OSError) and exc.errno == first_errno + for exc in all_exceptions + ): + raise OSError(first_errno, msg) + elif isinstance(first_exception, RuntimeError) and all( + isinstance(exc, RuntimeError) for exc in all_exceptions + ): + raise RuntimeError(msg) + # We have a mix of OSError and RuntimeError + # so we have to pick which one to raise. + # and we raise OSError for compatibility + raise OSError(msg) + finally: + all_exceptions = None # type: ignore[assignment] + exceptions = None # type: ignore[assignment] + + return sock + + +async def _connect_sock( + loop: asyncio.AbstractEventLoop, + exceptions: list[list[OSError | RuntimeError]], + addr_info: AddrInfoType, + local_addr_infos: Sequence[AddrInfoType] | None = None, + open_sockets: set[socket.socket] | None = None, + socket_factory: SocketFactoryType | None = None, +) -> socket.socket: + """ + Create, bind and connect one socket. + + If open_sockets is passed, add the socket to the set of open sockets. + Any failure caught here will remove the socket from the set and close it. + + Callers can use this set to close any sockets that are not the winner + of all staggered tasks in the result there are runner up sockets aka + multiple winners. + """ + my_exceptions: list[OSError | RuntimeError] = [] + exceptions.append(my_exceptions) + family, type_, proto, _, address = addr_info + sock = None + try: + if socket_factory is not None: + sock = socket_factory(addr_info) + else: + sock = socket.socket(family=family, type=type_, proto=proto) + if open_sockets is not None: + open_sockets.add(sock) + sock.setblocking(False) + if local_addr_infos is not None: + for lfamily, _, _, _, laddr in local_addr_infos: + # skip local addresses of different family + if lfamily != family: + continue + try: + sock.bind(laddr) + break + except OSError as exc: + msg = ( + f"error while attempting to bind on " + f"address {laddr!r}: " + f"{(exc.strerror or '').lower()}" + ) + exc = OSError(exc.errno, msg) + my_exceptions.append(exc) + else: # all bind attempts failed + if my_exceptions: + raise my_exceptions.pop() + else: + raise OSError(f"no matching local address with {family=} found") + await loop.sock_connect(sock, address) + return sock + except BaseException as exc: + if isinstance(exc, (RuntimeError, OSError)): + my_exceptions.append(exc) + if sock is not None: + if open_sockets is not None: + open_sockets.remove(sock) + try: + sock.close() + except OSError as e: + my_exceptions.append(e) + raise + raise + finally: + exceptions = my_exceptions = None # type: ignore[assignment] + + +def _interleave_addrinfos( + addrinfos: Sequence[AddrInfoType], first_address_family_count: int = 1 +) -> list[AddrInfoType]: + """Interleave list of addrinfo tuples by family.""" + # Group addresses by family + addrinfos_by_family: defaultdict[int, list[AddrInfoType]] = defaultdict(list) + for addr in addrinfos: + addrinfos_by_family[addr[0]].append(addr) + addrinfos_lists = list(addrinfos_by_family.values()) + + reordered: list[AddrInfoType] = [] + if first_address_family_count > 1: + reordered.extend(addrinfos_lists[0][: first_address_family_count - 1]) + del addrinfos_lists[0][: first_address_family_count - 1] + reordered.extend( + a + for a in itertools.chain.from_iterable(itertools.zip_longest(*addrinfos_lists)) + if a is not None + ) + return reordered diff --git a/py_modules/aiohappyeyeballs/py.typed b/py_modules/aiohappyeyeballs/py.typed new file mode 100644 index 0000000..e69de29 diff --git a/py_modules/aiohappyeyeballs/types.py b/py_modules/aiohappyeyeballs/types.py new file mode 100644 index 0000000..80f09af --- /dev/null +++ b/py_modules/aiohappyeyeballs/types.py @@ -0,0 +1,14 @@ +"""Types for aiohappyeyeballs.""" + +import socket +from collections.abc import Callable + +AddrInfoType = tuple[ + int | socket.AddressFamily, + int | socket.SocketKind, + int, + str, + tuple, # type: ignore[type-arg] +] + +SocketFactoryType = Callable[[AddrInfoType], socket.socket] diff --git a/py_modules/aiohappyeyeballs/utils.py b/py_modules/aiohappyeyeballs/utils.py new file mode 100644 index 0000000..689414b --- /dev/null +++ b/py_modules/aiohappyeyeballs/utils.py @@ -0,0 +1,82 @@ +"""Utility functions for aiohappyeyeballs.""" + +import ipaddress +import socket + +from .types import AddrInfoType + + +def addr_to_addr_infos( + addr: tuple[str, int, int, int] | tuple[str, int, int] | tuple[str, int] | None, +) -> list[AddrInfoType] | None: + """Convert an address tuple to a list of addr_info tuples.""" + if addr is None: + return None + host = addr[0] + port = addr[1] + is_ipv6 = ":" in host + if is_ipv6: + flowinfo = 0 + scopeid = 0 + addr_len = len(addr) + if addr_len >= 4: + scopeid = addr[3] # type: ignore[misc] + if addr_len >= 3: + flowinfo = addr[2] # type: ignore[misc] + addr = (host, port, flowinfo, scopeid) + family = socket.AF_INET6 + else: + addr = (host, port) + family = socket.AF_INET + return [(family, socket.SOCK_STREAM, socket.IPPROTO_TCP, "", addr)] + + +def pop_addr_infos_interleave( + addr_infos: list[AddrInfoType], interleave: int | None = None +) -> None: + """ + Pop addr_info from the list of addr_infos by family up to interleave times. + + The interleave parameter is used to know how many addr_infos for + each family should be popped of the top of the list. + """ + if interleave is None: + interleave = 1 + seen: dict[int, int] = {} + kept: list[AddrInfoType] = [] + for addr_info in addr_infos: + family = addr_info[0] + count = seen.get(family, 0) + if count >= interleave: + kept.append(addr_info) + seen[family] = count + 1 + addr_infos[:] = kept + + +def _addr_tuple_to_ip_address( + addr: tuple[str, int] | tuple[str, int, int, int], +) -> tuple[ipaddress.IPv4Address, int] | tuple[ipaddress.IPv6Address, int, int, int]: + """Convert an address tuple to an IPv4Address.""" + return (ipaddress.ip_address(addr[0]), *addr[1:]) + + +def remove_addr_infos( + addr_infos: list[AddrInfoType], + addr: tuple[str, int] | tuple[str, int, int, int], +) -> None: + """ + Remove an address from the list of addr_infos. + + The addr value is typically the return value of + sock.getpeername(). + """ + kept = [ai for ai in addr_infos if ai[-1] != addr] + if len(kept) == len(addr_infos): + # Slow path in case addr is formatted differently + match_addr = _addr_tuple_to_ip_address(addr) + kept = [ + ai for ai in addr_infos if _addr_tuple_to_ip_address(ai[-1]) != match_addr + ] + if len(kept) == len(addr_infos): + raise ValueError(f"Address {addr} not found in addr_infos") + addr_infos[:] = kept diff --git a/py_modules/aiohttp/__init__.py b/py_modules/aiohttp/__init__.py new file mode 100644 index 0000000..e548b4f --- /dev/null +++ b/py_modules/aiohttp/__init__.py @@ -0,0 +1,279 @@ +__version__ = "3.14.3" + +from typing import TYPE_CHECKING + +from . import hdrs as hdrs +from .client import ( + BaseConnector, + ClientConnectionError, + ClientConnectionResetError, + ClientConnectorCertificateError, + ClientConnectorDNSError, + ClientConnectorError, + ClientConnectorSSLError, + ClientError, + ClientHttpProxyError, + ClientOSError, + ClientPayloadError, + ClientProxyConnectionError, + ClientRequest, + ClientResponse, + ClientResponseError, + ClientSession, + ClientSSLError, + ClientTimeout, + ClientWebSocketResponse, + ClientWSTimeout, + ConnectionTimeoutError, + ContentTypeError, + Fingerprint, + InvalidURL, + InvalidUrlClientError, + InvalidUrlRedirectClientError, + NamedPipeConnector, + NonHttpUrlClientError, + NonHttpUrlRedirectClientError, + RedirectClientError, + RequestInfo, + ServerConnectionError, + ServerDisconnectedError, + ServerFingerprintMismatch, + ServerTimeoutError, + SocketTimeoutError, + TCPConnector, + TooManyRedirects, + UnixConnector, + WSMessageTypeError, + WSServerHandshakeError, + request, +) +from .client_middleware_digest_auth import DigestAuthMiddleware +from .client_middlewares import ClientHandlerType, ClientMiddlewareType +from .compression_utils import set_zlib_backend +from .connector import ( + AddrInfoType as AddrInfoType, + SocketFactoryType as SocketFactoryType, +) +from .cookiejar import CookieJar as CookieJar, DummyCookieJar as DummyCookieJar +from .formdata import FormData as FormData +from .helpers import BasicAuth, ChainMapProxy, ETag, encode_basic_auth +from .http import ( + HttpVersion as HttpVersion, + HttpVersion10 as HttpVersion10, + HttpVersion11 as HttpVersion11, + WebSocketError as WebSocketError, + WSCloseCode as WSCloseCode, + WSMessage as WSMessage, + WSMsgType as WSMsgType, +) +from .multipart import ( + BadContentDispositionHeader as BadContentDispositionHeader, + BadContentDispositionParam as BadContentDispositionParam, + BodyPartReader as BodyPartReader, + MultipartReader as MultipartReader, + MultipartWriter as MultipartWriter, + content_disposition_filename as content_disposition_filename, + parse_content_disposition as parse_content_disposition, +) +from .payload import ( + PAYLOAD_REGISTRY as PAYLOAD_REGISTRY, + AsyncIterablePayload as AsyncIterablePayload, + BufferedReaderPayload as BufferedReaderPayload, + BytesIOPayload as BytesIOPayload, + BytesPayload as BytesPayload, + IOBasePayload as IOBasePayload, + JsonPayload as JsonPayload, + Payload as Payload, + StringIOPayload as StringIOPayload, + StringPayload as StringPayload, + TextIOPayload as TextIOPayload, + get_payload as get_payload, + payload_type as payload_type, +) +from .payload_streamer import streamer as streamer +from .resolver import ( + AsyncResolver as AsyncResolver, + DefaultResolver as DefaultResolver, + ThreadedResolver as ThreadedResolver, +) +from .streams import ( + EMPTY_PAYLOAD as EMPTY_PAYLOAD, + DataQueue as DataQueue, + EofStream as EofStream, + FlowControlDataQueue as FlowControlDataQueue, + StreamReader as StreamReader, +) +from .tracing import ( + TraceConfig as TraceConfig, + TraceConnectionCreateEndParams as TraceConnectionCreateEndParams, + TraceConnectionCreateStartParams as TraceConnectionCreateStartParams, + TraceConnectionQueuedEndParams as TraceConnectionQueuedEndParams, + TraceConnectionQueuedStartParams as TraceConnectionQueuedStartParams, + TraceConnectionReuseconnParams as TraceConnectionReuseconnParams, + TraceDnsCacheHitParams as TraceDnsCacheHitParams, + TraceDnsCacheMissParams as TraceDnsCacheMissParams, + TraceDnsResolveHostEndParams as TraceDnsResolveHostEndParams, + TraceDnsResolveHostStartParams as TraceDnsResolveHostStartParams, + TraceRequestChunkSentParams as TraceRequestChunkSentParams, + TraceRequestEndParams as TraceRequestEndParams, + TraceRequestExceptionParams as TraceRequestExceptionParams, + TraceRequestHeadersSentParams as TraceRequestHeadersSentParams, + TraceRequestRedirectParams as TraceRequestRedirectParams, + TraceRequestStartParams as TraceRequestStartParams, + TraceResponseChunkReceivedParams as TraceResponseChunkReceivedParams, +) + +if TYPE_CHECKING: + # At runtime these are lazy-loaded at the bottom of the file. + from .worker import ( + GunicornUVLoopWebWorker as GunicornUVLoopWebWorker, + GunicornWebWorker as GunicornWebWorker, + ) + +__all__: tuple[str, ...] = ( + "hdrs", + # client + "AddrInfoType", + "BaseConnector", + "ClientConnectionError", + "ClientConnectionResetError", + "ClientConnectorCertificateError", + "ClientConnectorDNSError", + "ClientConnectorError", + "ClientConnectorSSLError", + "ClientError", + "ClientHttpProxyError", + "ClientOSError", + "ClientPayloadError", + "ClientProxyConnectionError", + "ClientResponse", + "ClientRequest", + "ClientResponseError", + "ClientSSLError", + "ClientSession", + "ClientTimeout", + "ClientWebSocketResponse", + "ClientWSTimeout", + "ConnectionTimeoutError", + "ContentTypeError", + "Fingerprint", + "FlowControlDataQueue", + "InvalidURL", + "InvalidUrlClientError", + "InvalidUrlRedirectClientError", + "NonHttpUrlClientError", + "NonHttpUrlRedirectClientError", + "RedirectClientError", + "RequestInfo", + "ServerConnectionError", + "ServerDisconnectedError", + "ServerFingerprintMismatch", + "ServerTimeoutError", + "SocketFactoryType", + "SocketTimeoutError", + "TCPConnector", + "TooManyRedirects", + "UnixConnector", + "NamedPipeConnector", + "WSServerHandshakeError", + "request", + # client_middleware + "ClientMiddlewareType", + "ClientHandlerType", + # cookiejar + "CookieJar", + "DummyCookieJar", + # formdata + "FormData", + # helpers + "BasicAuth", + "ChainMapProxy", + "DigestAuthMiddleware", + "ETag", + "encode_basic_auth", + "set_zlib_backend", + # http + "HttpVersion", + "HttpVersion10", + "HttpVersion11", + "WSMsgType", + "WSCloseCode", + "WSMessage", + "WebSocketError", + # multipart + "BadContentDispositionHeader", + "BadContentDispositionParam", + "BodyPartReader", + "MultipartReader", + "MultipartWriter", + "content_disposition_filename", + "parse_content_disposition", + # payload + "AsyncIterablePayload", + "BufferedReaderPayload", + "BytesIOPayload", + "BytesPayload", + "IOBasePayload", + "JsonPayload", + "PAYLOAD_REGISTRY", + "Payload", + "StringIOPayload", + "StringPayload", + "TextIOPayload", + "get_payload", + "payload_type", + # payload_streamer + "streamer", + # resolver + "AsyncResolver", + "DefaultResolver", + "ThreadedResolver", + # streams + "DataQueue", + "EMPTY_PAYLOAD", + "EofStream", + "StreamReader", + # tracing + "TraceConfig", + "TraceConnectionCreateEndParams", + "TraceConnectionCreateStartParams", + "TraceConnectionQueuedEndParams", + "TraceConnectionQueuedStartParams", + "TraceConnectionReuseconnParams", + "TraceDnsCacheHitParams", + "TraceDnsCacheMissParams", + "TraceDnsResolveHostEndParams", + "TraceDnsResolveHostStartParams", + "TraceRequestChunkSentParams", + "TraceRequestEndParams", + "TraceRequestExceptionParams", + "TraceRequestHeadersSentParams", + "TraceRequestRedirectParams", + "TraceRequestStartParams", + "TraceResponseChunkReceivedParams", + # workers (imported lazily with __getattr__) + "GunicornUVLoopWebWorker", + "GunicornWebWorker", + "WSMessageTypeError", +) + + +def __dir__() -> tuple[str, ...]: + return __all__ + ("__doc__",) + + +def __getattr__(name: str) -> object: + global GunicornUVLoopWebWorker, GunicornWebWorker + + # Importing gunicorn takes a long time (>100ms), so only import if actually needed. + if name in ("GunicornUVLoopWebWorker", "GunicornWebWorker"): + try: + from .worker import GunicornUVLoopWebWorker as guv, GunicornWebWorker as gw + except ImportError: + return None + + GunicornUVLoopWebWorker = guv # type: ignore[misc] + GunicornWebWorker = gw # type: ignore[misc] + return guv if name == "GunicornUVLoopWebWorker" else gw + + raise AttributeError(f"module {__name__} has no attribute {name}") diff --git a/py_modules/aiohttp/_cookie_helpers.py b/py_modules/aiohttp/_cookie_helpers.py new file mode 100644 index 0000000..00ca820 --- /dev/null +++ b/py_modules/aiohttp/_cookie_helpers.py @@ -0,0 +1,361 @@ +""" +Internal cookie handling helpers. + +This module contains internal utilities for cookie parsing and manipulation. +These are not part of the public API and may change without notice. +""" + +import re +from collections.abc import Sequence +from http.cookies import CookieError, Morsel +from typing import cast + +from .log import internal_logger + +__all__ = ( + "parse_set_cookie_headers", + "parse_cookie_header", + "preserve_morsel_with_coded_value", +) + +# Cookie parsing constants +# Allow more characters in cookie names to handle real-world cookies +# that don't strictly follow RFC standards (fixes #2683) +# RFC 6265 defines cookie-name token as per RFC 2616 Section 2.2, +# but many servers send cookies with characters like {} [] () etc. +# This makes the cookie parser more tolerant of real-world cookies +# while still providing some validation to catch obviously malformed names. +_COOKIE_NAME_RE = re.compile(r"^[!#$%&\'()*+\-./0-9:<=>?@A-Z\[\]^_`a-z{|}~]+$") +_COOKIE_KNOWN_ATTRS = frozenset( # AKA Morsel._reserved + ( + "path", + "domain", + "max-age", + "expires", + "secure", + "httponly", + "samesite", + "partitioned", + "version", + "comment", + ) +) +_COOKIE_BOOL_ATTRS = frozenset( # AKA Morsel._flags + ("secure", "httponly", "partitioned") +) + +# SimpleCookie's pattern for parsing cookies with relaxed validation +# Based on http.cookies pattern but extended to allow more characters in cookie names +# to handle real-world cookies (fixes #2683) +_COOKIE_PATTERN = re.compile( + r""" + \s* # Optional whitespace at start of cookie + (?P # Start of group 'key' + # aiohttp has extended to include [] for compatibility with real-world cookies + [\w\d!#%&'~_`><@,:/\$\*\+\-\.\^\|\)\(\?\}\{\[\]]+ # Any word of at least one letter + ) # End of group 'key' + ( # Optional group: there may not be a value. + \s*=\s* # Equal Sign + (?P # Start of group 'val' + "(?:[^\\"]|\\.)*" # Any double-quoted string (properly closed) + | # or + "[^";]* # Unmatched opening quote (differs from SimpleCookie - issue #7993) + | # or + # Special case for "expires" attr - RFC 822, RFC 850, RFC 1036, RFC 1123 + (\w{3,6}day|\w{3}),\s # Day of the week or abbreviated day (with comma) + [\w\d\s-]{9,11}\s[\d:]{8}\s # Date and time in specific format + (GMT|[+-]\d{4}) # Timezone: GMT or RFC 2822 offset like -0000, +0100 + # NOTE: RFC 2822 timezone support is an aiohttp extension + # for issue #4493 - SimpleCookie does NOT support this + | # or + # ANSI C asctime() format: "Wed Jun 9 10:18:14 2021" + # NOTE: This is an aiohttp extension for issue #4327 - SimpleCookie does NOT support this format + \w{3}\s+\w{3}\s+[\s\d]\d\s+\d{2}:\d{2}:\d{2}\s+\d{4} + | # or + [\w\d!#%&'~_`><@,:/\$\*\+\-\.\^\|\)\(\?\}\{\=\[\]]* # Any word or empty string + ) # End of group 'val' + )? # End of optional value group + \s* # Any number of spaces. + (\s+|;|$) # Ending either at space, semicolon, or EOS. + """, + re.VERBOSE | re.ASCII, +) + + +def preserve_morsel_with_coded_value(cookie: Morsel[str]) -> Morsel[str]: + """ + Preserve a Morsel's coded_value exactly as received from the server. + + This function ensures that cookie encoding is preserved exactly as sent by + the server, which is critical for compatibility with old servers that have + strict requirements about cookie formats. + + This addresses the issue described in https://github.com/aio-libs/aiohttp/pull/1453 + where Python's SimpleCookie would re-encode cookies, breaking authentication + with certain servers. + + Args: + cookie: A Morsel object from SimpleCookie + + Returns: + A Morsel object with preserved coded_value + + """ + mrsl_val = cast("Morsel[str]", cookie.get(cookie.key, Morsel())) + # We use __setstate__ instead of the public set() API because it allows us to + # bypass validation and set already validated state. This is more stable than + # setting protected attributes directly and unlikely to change since it would + # break pickling. + try: + mrsl_val.__setstate__( # type: ignore[attr-defined] + { + "key": cookie.key, + "value": cookie.value, + "coded_value": cookie.coded_value, + } + ) + except CookieError: + return cookie + return mrsl_val + + +_unquote_sub = re.compile(r"\\(?:([0-3][0-7][0-7])|(.))").sub + + +def _unquote_replace(m: re.Match[str]) -> str: + """ + Replace function for _unquote_sub regex substitution. + + Handles escaped characters in cookie values: + - Octal sequences are converted to their character representation + - Other escaped characters are unescaped by removing the backslash + """ + if m[1]: + return chr(int(m[1], 8)) + return m[2] + + +def _unquote(value: str) -> str: + """ + Unquote a cookie value. + + Vendored from http.cookies._unquote to ensure compatibility. + + Note: The original implementation checked for None, but we've removed + that check since all callers already ensure the value is not None. + """ + # If there aren't any doublequotes, + # then there can't be any special characters. See RFC 2109. + if len(value) < 2: + return value + if value[0] != '"' or value[-1] != '"': + return value + + # We have to assume that we must decode this string. + # Down to work. + + # Remove the "s + value = value[1:-1] + + # Check for special sequences. Examples: + # \012 --> \n + # \" --> " + # + return _unquote_sub(_unquote_replace, value) + + +def parse_cookie_header(header: str) -> list[tuple[str, Morsel[str]]]: + """ + Parse a Cookie header according to RFC 6265 Section 5.4. + + Cookie headers contain only name-value pairs separated by semicolons. + There are no attributes in Cookie headers - even names that match + attribute names (like 'path' or 'secure') should be treated as cookies. + + This parser uses the same regex-based approach as parse_set_cookie_headers + to properly handle quoted values that may contain semicolons. When the + regex fails to match a malformed cookie, it falls back to simple parsing + to ensure subsequent cookies are not lost + https://github.com/aio-libs/aiohttp/issues/11632 + + Args: + header: The Cookie header value to parse + + Returns: + List of (name, Morsel) tuples for compatibility with SimpleCookie.update() + """ + if not header: + return [] + + cookies: list[tuple[str, Morsel[str]]] = [] + morsel: Morsel[str] + i = 0 + n = len(header) + + invalid_names = [] + while i < n: + # Use the same pattern as parse_set_cookie_headers to find cookies + match = _COOKIE_PATTERN.match(header, i) + if not match: + # Fallback for malformed cookies https://github.com/aio-libs/aiohttp/issues/11632 + # Find next semicolon to skip or attempt simple key=value parsing + next_semi = header.find(";", i) + eq_pos = header.find("=", i) + + # Try to extract key=value if '=' comes before ';' + if eq_pos != -1 and (next_semi == -1 or eq_pos < next_semi): + end_pos = next_semi if next_semi != -1 else n + key = header[i:eq_pos].strip() + value = header[eq_pos + 1 : end_pos].strip() + + # Validate the name (same as regex path) + if not _COOKIE_NAME_RE.match(key): + invalid_names.append(key) + else: + morsel = Morsel() + try: + morsel.__setstate__( # type: ignore[attr-defined] + { + "key": key, + "value": _unquote(value), + "coded_value": value, + } + ) + except CookieError: + pass + else: + cookies.append((key, morsel)) + + # Move to next cookie or end + i = next_semi + 1 if next_semi != -1 else n + continue + + key = match.group("key") + value = match.group("val") or "" + i = match.end(0) + + # Validate the name + if not key or not _COOKIE_NAME_RE.match(key): + invalid_names.append(key) + continue + + # Create new morsel + morsel = Morsel() + # Preserve the original value as coded_value (with quotes if present) + # We use __setstate__ instead of the public set() API because it allows us to + # bypass validation and set already validated state. This is more stable than + # setting protected attributes directly and unlikely to change since it would + # break pickling. + try: + morsel.__setstate__( # type: ignore[attr-defined] + {"key": key, "value": _unquote(value), "coded_value": value} + ) + except CookieError: + continue + + cookies.append((key, morsel)) + + if invalid_names: + internal_logger.debug( + "Cannot load cookie. Illegal cookie names: %r", invalid_names + ) + + return cookies + + +def parse_set_cookie_headers(headers: Sequence[str]) -> list[tuple[str, Morsel[str]]]: + """ + Parse cookie headers using a vendored version of SimpleCookie parsing. + + This implementation is based on SimpleCookie.__parse_string to ensure + compatibility with how SimpleCookie parses cookies, including handling + of malformed cookies with missing semicolons. + + This function is used for both Cookie and Set-Cookie headers in order to be + forgiving. Ideally we would have followed RFC 6265 Section 5.2 (for Cookie + headers) and RFC 6265 Section 4.2.1 (for Set-Cookie headers), but the + real world data makes it impossible since we need to be a bit more forgiving. + + NOTE: This implementation differs from SimpleCookie in handling unmatched quotes. + SimpleCookie will stop parsing when it encounters a cookie value with an unmatched + quote (e.g., 'cookie="value'), causing subsequent cookies to be silently dropped. + This implementation handles unmatched quotes more gracefully to prevent cookie loss. + See https://github.com/aio-libs/aiohttp/issues/7993 + """ + parsed_cookies: list[tuple[str, Morsel[str]]] = [] + + for header in headers: + if not header: + continue + + # Parse cookie string using SimpleCookie's algorithm + i = 0 + n = len(header) + current_morsel: Morsel[str] | None = None + morsel_seen = False + + while 0 <= i < n: + # Start looking for a cookie + match = _COOKIE_PATTERN.match(header, i) + if not match: + # No more cookies + break + + key, value = match.group("key"), match.group("val") + i = match.end(0) + lower_key = key.lower() + + if key[0] == "$": + if not morsel_seen: + # We ignore attributes which pertain to the cookie + # mechanism as a whole, such as "$Version". + continue + # Process as attribute + if current_morsel is not None: + attr_lower_key = lower_key[1:] + if attr_lower_key in _COOKIE_KNOWN_ATTRS: + current_morsel[attr_lower_key] = value or "" + elif lower_key in _COOKIE_KNOWN_ATTRS: + if not morsel_seen: + # Invalid cookie string - attribute before cookie + break + if lower_key in _COOKIE_BOOL_ATTRS: + # Boolean attribute with any value should be True + if current_morsel is not None and current_morsel.isReservedKey(key): + current_morsel[lower_key] = True + elif value is None: + # Invalid cookie string - non-boolean attribute without value + break + elif current_morsel is not None: + # Regular attribute with value + current_morsel[lower_key] = _unquote(value) + elif value is not None: + # This is a cookie name=value pair + # Validate the name + if key in _COOKIE_KNOWN_ATTRS or not _COOKIE_NAME_RE.match(key): + internal_logger.warning( + "Can not load cookies: Illegal cookie name %r", key + ) + current_morsel = None + else: + # Create new morsel + current_morsel = Morsel() + # Preserve the original value as coded_value (with quotes if present) + try: + current_morsel.__setstate__( # type: ignore[attr-defined] + { + "key": key, + "value": _unquote(value), + "coded_value": value, + } + ) + except CookieError: + current_morsel = None + else: + parsed_cookies.append((key, current_morsel)) + morsel_seen = True + else: + # Invalid cookie string - no value for non-attribute + break + + return parsed_cookies diff --git a/py_modules/aiohttp/_cparser.pxd b/py_modules/aiohttp/_cparser.pxd new file mode 100644 index 0000000..903f35c --- /dev/null +++ b/py_modules/aiohttp/_cparser.pxd @@ -0,0 +1,89 @@ +# This file contains just the definitions needed in our Cython code. + +from libc.stdint cimport uint8_t, uint16_t, uint64_t + + +cdef extern from "llhttp.h": + + struct llhttp__internal_s: + void* data + uint64_t content_length + uint8_t type + uint8_t method + uint8_t http_major + uint8_t http_minor + uint8_t upgrade + uint16_t flags + uint16_t status_code + + ctypedef llhttp__internal_s llhttp__internal_t + ctypedef llhttp__internal_t llhttp_t + + ctypedef int (*llhttp_data_cb)(llhttp_t*, const char *at, size_t length) except -1 + ctypedef int (*llhttp_cb)(llhttp_t*) except -1 + + struct llhttp_settings_s: + llhttp_cb on_message_begin + llhttp_data_cb on_url + llhttp_data_cb on_status + llhttp_data_cb on_header_field + llhttp_data_cb on_header_value + llhttp_cb on_headers_complete + llhttp_data_cb on_body + llhttp_cb on_message_complete + llhttp_cb on_chunk_header + llhttp_cb on_chunk_complete + + ctypedef llhttp_settings_s llhttp_settings_t + + enum llhttp_errno: + HPE_OK, + HPE_INVALID_METHOD, + HPE_INVALID_URL, + HPE_INVALID_CONSTANT, + HPE_INVALID_VERSION, + HPE_INVALID_HEADER_TOKEN, + HPE_INVALID_CONTENT_LENGTH, + HPE_INVALID_CHUNK_SIZE, + HPE_INVALID_STATUS, + HPE_INVALID_EOF_STATE, + HPE_INVALID_TRANSFER_ENCODING, + HPE_CB_MESSAGE_BEGIN, + HPE_CB_HEADERS_COMPLETE, + HPE_CB_MESSAGE_COMPLETE, + HPE_CB_CHUNK_HEADER, + HPE_CB_CHUNK_COMPLETE, + HPE_PAUSED, + HPE_PAUSED_UPGRADE + + ctypedef llhttp_errno llhttp_errno_t + + enum llhttp_flags: + F_CHUNKED, + F_CONTENT_LENGTH + + enum llhttp_type: + HTTP_REQUEST, + HTTP_RESPONSE + + enum llhttp_method: + HTTP_CONNECT + + void llhttp_settings_init(llhttp_settings_t* settings) + void llhttp_init(llhttp_t* parser, llhttp_type type, + const llhttp_settings_t* settings) + + llhttp_errno_t llhttp_execute(llhttp_t* parser, const char* data, size_t len) + + int llhttp_should_keep_alive(const llhttp_t* parser) + + void llhttp_resume(llhttp_t* parser) + void llhttp_resume_after_upgrade(llhttp_t* parser) + + llhttp_errno_t llhttp_get_errno(const llhttp_t* parser) + const char* llhttp_get_error_reason(const llhttp_t* parser) + const char* llhttp_get_error_pos(const llhttp_t* parser) + + void llhttp_set_lenient_headers(llhttp_t* parser, int enabled) + void llhttp_set_lenient_optional_cr_before_lf(llhttp_t* parser, int enabled) + void llhttp_set_lenient_spaces_after_chunk_size(llhttp_t* parser, int enabled) diff --git a/py_modules/aiohttp/_find_header.pxd b/py_modules/aiohttp/_find_header.pxd new file mode 100644 index 0000000..37a6c37 --- /dev/null +++ b/py_modules/aiohttp/_find_header.pxd @@ -0,0 +1,2 @@ +cdef extern from "_find_header.h": + int find_header(char *, int) diff --git a/py_modules/aiohttp/_headers.pxi b/py_modules/aiohttp/_headers.pxi new file mode 100644 index 0000000..3744721 --- /dev/null +++ b/py_modules/aiohttp/_headers.pxi @@ -0,0 +1,83 @@ +# The file is autogenerated from aiohttp/hdrs.py +# Run ./tools/gen.py to update it after the origin changing. + +from . import hdrs +cdef tuple headers = ( + hdrs.ACCEPT, + hdrs.ACCEPT_CHARSET, + hdrs.ACCEPT_ENCODING, + hdrs.ACCEPT_LANGUAGE, + hdrs.ACCEPT_RANGES, + hdrs.ACCESS_CONTROL_ALLOW_CREDENTIALS, + hdrs.ACCESS_CONTROL_ALLOW_HEADERS, + hdrs.ACCESS_CONTROL_ALLOW_METHODS, + hdrs.ACCESS_CONTROL_ALLOW_ORIGIN, + hdrs.ACCESS_CONTROL_EXPOSE_HEADERS, + hdrs.ACCESS_CONTROL_MAX_AGE, + hdrs.ACCESS_CONTROL_REQUEST_HEADERS, + hdrs.ACCESS_CONTROL_REQUEST_METHOD, + hdrs.AGE, + hdrs.ALLOW, + hdrs.AUTHORIZATION, + hdrs.CACHE_CONTROL, + hdrs.CONNECTION, + hdrs.CONTENT_DISPOSITION, + hdrs.CONTENT_ENCODING, + hdrs.CONTENT_LANGUAGE, + hdrs.CONTENT_LENGTH, + hdrs.CONTENT_LOCATION, + hdrs.CONTENT_MD5, + hdrs.CONTENT_RANGE, + hdrs.CONTENT_TRANSFER_ENCODING, + hdrs.CONTENT_TYPE, + hdrs.COOKIE, + hdrs.DATE, + hdrs.DESTINATION, + hdrs.DIGEST, + hdrs.ETAG, + hdrs.EXPECT, + hdrs.EXPIRES, + hdrs.FORWARDED, + hdrs.FROM, + hdrs.HOST, + hdrs.IF_MATCH, + hdrs.IF_MODIFIED_SINCE, + hdrs.IF_NONE_MATCH, + hdrs.IF_RANGE, + hdrs.IF_UNMODIFIED_SINCE, + hdrs.KEEP_ALIVE, + hdrs.LAST_EVENT_ID, + hdrs.LAST_MODIFIED, + hdrs.LINK, + hdrs.LOCATION, + hdrs.MAX_FORWARDS, + hdrs.ORIGIN, + hdrs.PRAGMA, + hdrs.PROXY_AUTHENTICATE, + hdrs.PROXY_AUTHORIZATION, + hdrs.RANGE, + hdrs.REFERER, + hdrs.RETRY_AFTER, + hdrs.SEC_WEBSOCKET_ACCEPT, + hdrs.SEC_WEBSOCKET_EXTENSIONS, + hdrs.SEC_WEBSOCKET_KEY, + hdrs.SEC_WEBSOCKET_KEY1, + hdrs.SEC_WEBSOCKET_PROTOCOL, + hdrs.SEC_WEBSOCKET_VERSION, + hdrs.SERVER, + hdrs.SET_COOKIE, + hdrs.TE, + hdrs.TRAILER, + hdrs.TRANSFER_ENCODING, + hdrs.URI, + hdrs.UPGRADE, + hdrs.USER_AGENT, + hdrs.VARY, + hdrs.VIA, + hdrs.WWW_AUTHENTICATE, + hdrs.WANT_DIGEST, + hdrs.WARNING, + hdrs.X_FORWARDED_FOR, + hdrs.X_FORWARDED_HOST, + hdrs.X_FORWARDED_PROTO, +) diff --git a/py_modules/aiohttp/_http_parser.cpython-311-x86_64-linux-gnu.so b/py_modules/aiohttp/_http_parser.cpython-311-x86_64-linux-gnu.so new file mode 100755 index 0000000..ad5d0f0 Binary files /dev/null and b/py_modules/aiohttp/_http_parser.cpython-311-x86_64-linux-gnu.so differ diff --git a/py_modules/aiohttp/_http_parser.pyx b/py_modules/aiohttp/_http_parser.pyx new file mode 100644 index 0000000..0fe6084 --- /dev/null +++ b/py_modules/aiohttp/_http_parser.pyx @@ -0,0 +1,984 @@ +# Based on https://github.com/MagicStack/httptools +# + +from cpython cimport ( + Py_buffer, + PyBUF_SIMPLE, + PyBuffer_Release, + PyBytes_AsString, + PyBytes_AsStringAndSize, + PyObject_GetBuffer, +) +from cpython.mem cimport PyMem_Free, PyMem_Malloc +from libc.limits cimport ULLONG_MAX +from libc.string cimport memcpy + +from multidict import CIMultiDict as _CIMultiDict, CIMultiDictProxy as _CIMultiDictProxy +from yarl import URL as _URL + +from aiohttp import hdrs +from aiohttp.helpers import DEBUG, set_exception + +from .http_exceptions import ( + BadHttpMessage, + BadHttpMethod, + BadStatusLine, + ContentLengthError, + InvalidHeader, + InvalidURLError, + LineTooLong, + PayloadEncodingError, + TransferEncodingError, +) +from .http_parser import DeflateBuffer as _DeflateBuffer +from .http_writer import ( + HttpVersion as _HttpVersion, + HttpVersion10 as _HttpVersion10, + HttpVersion11 as _HttpVersion11, +) +from .streams import EMPTY_PAYLOAD as _EMPTY_PAYLOAD, StreamReader as _StreamReader + +cimport cython + +from aiohttp cimport _cparser as cparser + +include "_headers.pxi" + +from aiohttp cimport _find_header + + +cdef frozenset ALLOWED_UPGRADES = frozenset({"websocket"}) +DEF DEFAULT_FREELIST_SIZE = 250 + +cdef extern from "Python.h": + int PyByteArray_Resize(object, Py_ssize_t) except -1 + Py_ssize_t PyByteArray_Size(object) except -1 + char* PyByteArray_AsString(object) + +__all__ = ('HttpRequestParser', 'HttpResponseParser', + 'RawRequestMessage', 'RawResponseMessage') + +cdef object URL = _URL +cdef object URL_build = URL.build +cdef object CIMultiDict = _CIMultiDict +cdef object CIMultiDictProxy = _CIMultiDictProxy +cdef object HttpVersion = _HttpVersion +cdef object HttpVersion10 = _HttpVersion10 +cdef object HttpVersion11 = _HttpVersion11 +cdef object SEC_WEBSOCKET_KEY1 = hdrs.SEC_WEBSOCKET_KEY1 +cdef object CONTENT_ENCODING = hdrs.CONTENT_ENCODING +cdef object EMPTY_PAYLOAD = _EMPTY_PAYLOAD +cdef object StreamReader = _StreamReader +cdef object DeflateBuffer = _DeflateBuffer +cdef tuple EMPTY_FEED_DATA_RESULT = ((), False, b"") + +# RFC 9110 singleton headers — duplicates are rejected in strict mode. +# In lax mode (response parser default), the check is skipped entirely +# since real-world servers (e.g. Google APIs, Werkzeug) commonly send +# duplicate headers like Content-Type or Server. +cdef frozenset SINGLETON_HEADERS = frozenset({ + hdrs.CONTENT_LENGTH, + hdrs.CONTENT_LOCATION, + hdrs.CONTENT_RANGE, + hdrs.CONTENT_TYPE, + hdrs.ETAG, + hdrs.HOST, + hdrs.MAX_FORWARDS, + hdrs.SERVER, + hdrs.TRANSFER_ENCODING, + hdrs.USER_AGENT, +}) + +cdef inline object extend(object buf, const char* at, size_t length): + cdef Py_ssize_t s + cdef char* ptr + s = PyByteArray_Size(buf) + PyByteArray_Resize(buf, s + length) + ptr = PyByteArray_AsString(buf) + memcpy(ptr + s, at, length) + + +# The method-name table and its length come straight from llhttp's canonical +# HTTP_ALL_METHOD_MAP, so they track the vendored llhttp version automatically +# instead of relying on a hand-maintained method count. +cdef extern from *: + """ + #include "llhttp.h" + + #define _AIOHTTP_METHOD_NAME(NUM, NAME, STRING) [NUM] = #STRING, + static const char* const _aiohttp_method_names[] = { + HTTP_ALL_METHOD_MAP(_AIOHTTP_METHOD_NAME) + }; + #undef _AIOHTTP_METHOD_NAME + """ + const char* _aiohttp_method_names[] + const int METHODS_COUNT "((int)(sizeof(_aiohttp_method_names) / sizeof(_aiohttp_method_names[0])))" + + +cdef list _http_method = [] + +for i in range(METHODS_COUNT): + assert _aiohttp_method_names[i] is not NULL + _http_method.append(_aiohttp_method_names[i].decode('ascii')) + + +cdef inline object find_header(bytes raw_header): + cdef Py_ssize_t size + cdef char *buf + cdef int idx + PyBytes_AsStringAndSize(raw_header, &buf, &size) + idx = _find_header.find_header(buf, size) + if idx == -1: + return raw_header.decode('utf-8', 'surrogateescape') + return headers[idx] + + +@cython.freelist(DEFAULT_FREELIST_SIZE) +cdef class RawRequestMessage: + cdef readonly str method + cdef readonly str path + cdef readonly object version # HttpVersion + cdef readonly object headers # CIMultiDict + cdef readonly object raw_headers # tuple + cdef readonly object should_close + cdef readonly object compression + cdef readonly object upgrade + cdef readonly object chunked + cdef readonly object url # yarl.URL + + def __init__(self, method, path, version, headers, raw_headers, + should_close, compression, upgrade, chunked, url): + self.method = method + self.path = path + self.version = version + self.headers = headers + self.raw_headers = raw_headers + self.should_close = should_close + self.compression = compression + self.upgrade = upgrade + self.chunked = chunked + self.url = url + + def __repr__(self): + info = [] + info.append(("method", self.method)) + info.append(("path", self.path)) + info.append(("version", self.version)) + info.append(("headers", self.headers)) + info.append(("raw_headers", self.raw_headers)) + info.append(("should_close", self.should_close)) + info.append(("compression", self.compression)) + info.append(("upgrade", self.upgrade)) + info.append(("chunked", self.chunked)) + info.append(("url", self.url)) + sinfo = ', '.join(name + '=' + repr(val) for name, val in info) + return '' + + def _replace(self, **dct): + cdef RawRequestMessage ret + ret = _new_request_message(self.method, + self.path, + self.version, + self.headers, + self.raw_headers, + self.should_close, + self.compression, + self.upgrade, + self.chunked, + self.url) + if "method" in dct: + ret.method = dct["method"] + if "path" in dct: + ret.path = dct["path"] + if "version" in dct: + ret.version = dct["version"] + if "headers" in dct: + ret.headers = dct["headers"] + if "raw_headers" in dct: + ret.raw_headers = dct["raw_headers"] + if "should_close" in dct: + ret.should_close = dct["should_close"] + if "compression" in dct: + ret.compression = dct["compression"] + if "upgrade" in dct: + ret.upgrade = dct["upgrade"] + if "chunked" in dct: + ret.chunked = dct["chunked"] + if "url" in dct: + ret.url = dct["url"] + return ret + +cdef _new_request_message(str method, + str path, + object version, + object headers, + object raw_headers, + bint should_close, + object compression, + bint upgrade, + bint chunked, + object url): + cdef RawRequestMessage ret + ret = RawRequestMessage.__new__(RawRequestMessage) + ret.method = method + ret.path = path + ret.version = version + ret.headers = headers + ret.raw_headers = raw_headers + ret.should_close = should_close + ret.compression = compression + ret.upgrade = upgrade + ret.chunked = chunked + ret.url = url + return ret + + +@cython.freelist(DEFAULT_FREELIST_SIZE) +cdef class RawResponseMessage: + cdef readonly object version # HttpVersion + cdef readonly int code + cdef readonly str reason + cdef readonly object headers # CIMultiDict + cdef readonly object raw_headers # tuple + cdef readonly object should_close + cdef readonly object compression + cdef readonly object upgrade + cdef readonly object chunked + + def __init__(self, version, code, reason, headers, raw_headers, + should_close, compression, upgrade, chunked): + self.version = version + self.code = code + self.reason = reason + self.headers = headers + self.raw_headers = raw_headers + self.should_close = should_close + self.compression = compression + self.upgrade = upgrade + self.chunked = chunked + + def __repr__(self): + info = [] + info.append(("version", self.version)) + info.append(("code", self.code)) + info.append(("reason", self.reason)) + info.append(("headers", self.headers)) + info.append(("raw_headers", self.raw_headers)) + info.append(("should_close", self.should_close)) + info.append(("compression", self.compression)) + info.append(("upgrade", self.upgrade)) + info.append(("chunked", self.chunked)) + sinfo = ', '.join(name + '=' + repr(val) for name, val in info) + return '' + + +cdef _new_response_message(object version, + int code, + str reason, + object headers, + object raw_headers, + bint should_close, + object compression, + bint upgrade, + bint chunked): + cdef RawResponseMessage ret + ret = RawResponseMessage.__new__(RawResponseMessage) + ret.version = version + ret.code = code + ret.reason = reason + ret.headers = headers + ret.raw_headers = raw_headers + ret.should_close = should_close + ret.compression = compression + ret.upgrade = upgrade + ret.chunked = chunked + return ret + + +@cython.internal +cdef class HttpParser: + + cdef: + cparser.llhttp_t* _cparser + cparser.llhttp_settings_t* _csettings + + bytes _raw_name + object _name + bytes _raw_value + bint _has_value + int _header_name_size + + readonly object protocol + object _loop + object _timer + + size_t _max_line_size + size_t _max_field_size + size_t _max_headers + bint _response_with_body + bint _read_until_eof + bint _lax + + bytes _tail + bint _started + object _url + bytearray _buf + str _path + str _reason + list _headers + set _seen_singletons + list _raw_headers + bint _upgraded + bint _pending_upgrade + list _messages + bint _more_data_available + bint _paused + Py_ssize_t _msg_in_flight + Py_ssize_t _max_msg_queue_size + bint _eof_pending + object _payload + unsigned long long _content_length_expected + bint _payload_error + object _payload_exception + object _last_error + bint _auto_decompress + int _limit + + str _content_encoding + + Py_buffer py_buf + + def __cinit__(self): + self._cparser = \ + PyMem_Malloc(sizeof(cparser.llhttp_t)) + if self._cparser is NULL: + raise MemoryError() + + self._csettings = \ + PyMem_Malloc(sizeof(cparser.llhttp_settings_t)) + if self._csettings is NULL: + raise MemoryError() + + def __dealloc__(self): + PyMem_Free(self._cparser) + PyMem_Free(self._csettings) + + cdef _init( + self, cparser.llhttp_type mode, + object protocol, object loop, int limit, + object timer=None, + size_t max_line_size=8190, size_t max_headers=128, + size_t max_field_size=8190, payload_exception=None, + bint response_with_body=True, bint read_until_eof=False, + bint auto_decompress=True, + Py_ssize_t max_msg_queue_size=0, + ): + cparser.llhttp_settings_init(self._csettings) + cparser.llhttp_init(self._cparser, mode, self._csettings) + self._cparser.data = self + self._cparser.content_length = 0 + self._content_length_expected = 0 + + self.protocol = protocol + self._loop = loop + self._timer = timer + + self._buf = bytearray() + self._more_data_available = False + self._paused = False + self._msg_in_flight = 0 + self._max_msg_queue_size = max_msg_queue_size + self._eof_pending = False + self._payload = None + self._payload_error = 0 + self._payload_exception = payload_exception + self._messages = [] + + self._raw_name = b"" + self._raw_value = b"" + self._tail = b"" + self._has_value = False + self._header_name_size = 0 + + self._max_line_size = max_line_size + self._max_headers = max_headers + self._max_field_size = max_field_size + self._response_with_body = response_with_body + self._read_until_eof = read_until_eof + self._upgraded = False + self._pending_upgrade = False + self._auto_decompress = auto_decompress + self._content_encoding = None + self._lax = False + self._seen_singletons = set() + + self._csettings.on_url = cb_on_url + self._csettings.on_status = cb_on_status + self._csettings.on_header_field = cb_on_header_field + self._csettings.on_header_value = cb_on_header_value + self._csettings.on_headers_complete = cb_on_headers_complete + self._csettings.on_body = cb_on_body + self._csettings.on_message_begin = cb_on_message_begin + self._csettings.on_message_complete = cb_on_message_complete + self._csettings.on_chunk_header = cb_on_chunk_header + self._csettings.on_chunk_complete = cb_on_chunk_complete + + self._last_error = None + self._limit = limit + + cdef _process_header(self): + cdef str value + if self._raw_name != b"": + name = find_header(self._raw_name) + value = self._raw_value.decode('utf-8', 'surrogateescape') + + # reject null bytes in header values - matches the Python parser + # check at http_parser.py. llhttp in lenient mode doesn't reject + # these itself, so we need to catch them here. + # ref: RFC 9110 section 5.5 (CTL chars forbidden in field values) + if "\x00" in value: + raise InvalidHeader(self._raw_value) + + if not self._lax and name in SINGLETON_HEADERS: + if name in self._seen_singletons: + raise BadHttpMessage(f"Duplicate '{name}' header found.") + self._seen_singletons.add(name) + self._headers.append((name, value)) + if len(self._headers) > self._max_headers: + raise BadHttpMessage("Too many headers received") + + if name is CONTENT_ENCODING: + self._content_encoding = value + + self._has_value = False + self._header_name_size = 0 + self._raw_headers.append((self._raw_name, self._raw_value)) + self._raw_name = b"" + self._raw_value = b"" + + cdef _on_header_field(self, char* at, size_t length): + if self._has_value: + self._process_header() + + if self._raw_name == b"": + self._raw_name = at[:length] + else: + self._raw_name += at[:length] + + cdef _on_header_value(self, char* at, size_t length): + if self._raw_value == b"": + self._raw_value = at[:length] + else: + self._raw_value += at[:length] + self._has_value = True + + cdef _on_headers_complete(self): + cdef str h_upg + cdef str enc + + self._process_header() + + http_version = self.http_version() + should_close = not cparser.llhttp_should_keep_alive(self._cparser) + upgrade = self._cparser.upgrade + chunked = self._cparser.flags & cparser.F_CHUNKED + + raw_headers = tuple(self._raw_headers) + headers = CIMultiDictProxy(CIMultiDict(self._headers)) + + if self._cparser.type == cparser.HTTP_REQUEST: + if http_version == HttpVersion11 and hdrs.HOST not in headers: + raise BadHttpMessage("Missing 'Host' header in request.") + h_upg = headers.get("upgrade", "") + if (upgrade and h_upg.isascii() and h_upg.lower() in ALLOWED_UPGRADES) or self._cparser.method == cparser.HTTP_CONNECT: + # https://www.rfc-editor.org/info/rfc9110/#section-7.8-15 + # Defer the protocol switch until the complete request has been + # received. + self._pending_upgrade = True + else: + if upgrade and self._cparser.status_code == 101: + # llhttp pauses for a 101 on its own; just mark the pending + # switch so feed_data returns the upgraded-protocol tail. + self._pending_upgrade = True + + # do not support old websocket spec + if SEC_WEBSOCKET_KEY1 in headers: + raise InvalidHeader(SEC_WEBSOCKET_KEY1) + + encoding = None + enc = self._content_encoding + if enc is not None: + self._content_encoding = None + if enc.isascii() and enc.lower() in {"gzip", "deflate", "br", "zstd"}: + encoding = enc + + if self._cparser.type == cparser.HTTP_REQUEST: + method = _http_method[self._cparser.method] + msg = _new_request_message( + method, self._path, + http_version, headers, raw_headers, + should_close, encoding, upgrade, chunked, self._url) + else: + msg = _new_response_message( + http_version, self._cparser.status_code, self._reason, + headers, raw_headers, should_close, encoding, + upgrade, chunked) + + if ( + self._response_with_body + and ( + ULLONG_MAX > self._cparser.content_length > 0 or chunked or + self._cparser.method == cparser.HTTP_CONNECT or + (self._cparser.status_code >= 199 and + self._cparser.content_length == 0 and + self._read_until_eof) + ) + ): + payload = StreamReader( + self.protocol, timer=self._timer, loop=self._loop, + limit=self._limit) + else: + payload = EMPTY_PAYLOAD + + self._payload = payload + self._content_length_expected = self._cparser.content_length + if encoding is not None and self._auto_decompress: + self._payload = DeflateBuffer(payload, encoding, max_decompress_size=self._limit) + + self._messages.append((msg, payload)) + + cdef _on_message_complete(self): + self._payload.feed_eof() + self._payload = None + + cdef _on_chunk_header(self): + self._payload.begin_http_chunk_receiving() + + cdef _on_chunk_complete(self): + self._payload.end_http_chunk_receiving() + + cdef object _on_status_complete(self): + pass + + cdef inline http_version(self): + cdef cparser.llhttp_t* parser = self._cparser + + if parser.http_major == 1: + if parser.http_minor == 0: + return HttpVersion10 + elif parser.http_minor == 1: + return HttpVersion11 + + return HttpVersion(parser.http_major, parser.http_minor) + + ### Public API ### + + def pause_reading(self): + assert self._payload is not None + self._paused = True + + def message_consumed(self): + # Protocol drained a queued message; free a slot for parsing. + if self._msg_in_flight > 0: + self._msg_in_flight -= 1 + + def feed_eof(self): + cdef bytes desc + + if self._payload is not None: + if self._cparser.flags & cparser.F_CHUNKED: + raise TransferEncodingError( + "Not enough data to satisfy transfer length header.") + elif self._cparser.flags & cparser.F_CONTENT_LENGTH: + received = self._content_length_expected - self._cparser.content_length + raise ContentLengthError( + f"Not enough data to satisfy content length header " + f"(received {received} of {self._content_length_expected} bytes).") + elif cparser.llhttp_get_errno(self._cparser) != cparser.HPE_OK: + desc = cparser.llhttp_get_error_reason(self._cparser) + raise PayloadEncodingError(desc.decode('latin-1')) + else: + self._eof_pending = True + while self._more_data_available: + if self._paused: + self._paused = False + return # Will resume via feed_data(b"") later + self._more_data_available = self._payload.feed_data(b"", 0) + self._payload.feed_eof() + self._payload = None + self._more_data_available = False + self._eof_pending = False + elif self._started: + self._on_headers_complete() + if self._messages: + return self._messages[-1][0] + + def feed_data(self, incoming_data): + cdef: + size_t data_len + size_t nb + char* base + cdef cparser.llhttp_errno_t errno + cdef bytes data + + # Proactor loop sends bytearray. + # Ensure cython sees `data` as bytes + if type(incoming_data) is not bytes: + data = bytes(incoming_data) + else: + data = incoming_data + + if self._tail: + data, self._tail = self._tail + data, b"" + + if self._more_data_available: + result = cb_on_body(self._cparser, b"", 0) + if result is cparser.HPE_PAUSED: + self._tail = data + return EMPTY_FEED_DATA_RESULT + + if self._eof_pending: + self._payload.feed_eof() + self._payload = None + self._eof_pending = False + # We can't have new messages here, otherwise we wouldn't have + # received EOF. + return EMPTY_FEED_DATA_RESULT + + PyObject_GetBuffer(data, &self.py_buf, PyBUF_SIMPLE) + # Cache buffer pointer before PyBuffer_Release to avoid use-after-release. + base = self.py_buf.buf + data_len = self.py_buf.len + + errno = cparser.llhttp_execute( + self._cparser, + base, + data_len) + + if errno is cparser.HPE_PAUSED_UPGRADE: + cparser.llhttp_resume_after_upgrade(self._cparser) + nb = cparser.llhttp_get_error_pos(self._cparser) - base + if self._pending_upgrade: + # A supported upgrade whose request body has now been fully read. + self._upgraded = True + self._pending_upgrade = False + elif errno is cparser.HPE_PAUSED: + cparser.llhttp_resume(self._cparser) + pos = cparser.llhttp_get_error_pos(self._cparser) - base + self._tail = data[pos:] + + PyBuffer_Release(&self.py_buf) + + if errno not in (cparser.HPE_OK, cparser.HPE_PAUSED, cparser.HPE_PAUSED_UPGRADE): + if self._payload_error == 0: + if self._last_error is not None: + ex = self._last_error + self._last_error = None + else: + error_pos = cparser.llhttp_get_error_pos(self._cparser) + error_off = error_pos - base + before = data[:error_off] + after = data[error_off:].split(b"\r\n", 1)[0] + before = before.rsplit(b"\r\n", 1)[-1] + data = before + after + pointer = " " * (len(repr(before))-1) + "^" + ex = parser_error_from_errno(self._cparser, data, pointer) + self._payload = None + raise ex + + if self._messages: + messages = self._messages + self._messages = [] + else: + messages = () + + if self._upgraded: + return messages, True, data[nb:] + if not messages: # Shortcut to reduce Python overhead + return EMPTY_FEED_DATA_RESULT + return messages, False, b"" + + def set_upgraded(self, val): + self._upgraded = val + + +cdef class HttpRequestParser(HttpParser): + + def __init__( + self, protocol, loop, int limit, timer=None, + size_t max_line_size=8190, size_t max_headers=128, + size_t max_field_size=8190, payload_exception=None, + bint response_with_body=True, bint read_until_eof=False, + bint auto_decompress=True, Py_ssize_t max_msg_queue_size=0, + ): + self._init(cparser.HTTP_REQUEST, protocol, loop, limit, timer, + max_line_size, max_headers, max_field_size, + payload_exception, response_with_body, read_until_eof, + auto_decompress, max_msg_queue_size) + + cdef object _on_status_complete(self): + cdef int idx1, idx2 + if not self._buf: + return + self._path = self._buf.decode('utf-8', 'surrogateescape') + try: + idx3 = len(self._path) + if self._cparser.method == cparser.HTTP_CONNECT: + # authority-form, + # https://datatracker.ietf.org/doc/html/rfc7230#section-5.3.3 + self._url = URL.build(authority=self._path, encoded=True) + elif idx3 > 1 and self._path[0] == '/': + # origin-form, + # https://datatracker.ietf.org/doc/html/rfc7230#section-5.3.1 + idx1 = self._path.find("?") + if idx1 == -1: + query = "" + idx2 = self._path.find("#") + if idx2 == -1: + path = self._path + fragment = "" + else: + path = self._path[0: idx2] + fragment = self._path[idx2+1:] + + else: + path = self._path[0:idx1] + idx1 += 1 + idx2 = self._path.find("#", idx1) + if idx2 == -1: + query = self._path[idx1:] + fragment = "" + else: + query = self._path[idx1: idx2] + fragment = self._path[idx2+1:] + + self._url = URL.build( + path=path, + query_string=query, + fragment=fragment, + encoded=True, + ) + else: + # absolute-form for proxy maybe, + # https://datatracker.ietf.org/doc/html/rfc7230#section-5.3.2 + self._url = URL(self._path, encoded=True) + finally: + PyByteArray_Resize(self._buf, 0) + + +cdef class HttpResponseParser(HttpParser): + + def __init__( + self, protocol, loop, int limit, timer=None, + size_t max_line_size=8190, size_t max_headers=128, + size_t max_field_size=8190, payload_exception=None, + bint response_with_body=True, bint read_until_eof=False, + bint auto_decompress=True + ): + self._init(cparser.HTTP_RESPONSE, protocol, loop, limit, timer, + max_line_size, max_headers, max_field_size, + payload_exception, response_with_body, read_until_eof, + auto_decompress) + # Use strict parsing on dev mode, so users are warned about broken servers. + if not DEBUG: + cparser.llhttp_set_lenient_headers(self._cparser, 1) + cparser.llhttp_set_lenient_optional_cr_before_lf(self._cparser, 1) + cparser.llhttp_set_lenient_spaces_after_chunk_size(self._cparser, 1) + self._lax = True + + cdef object _on_status_complete(self): + if self._buf: + self._reason = self._buf.decode('utf-8', 'surrogateescape') + PyByteArray_Resize(self._buf, 0) + else: + self._reason = self._reason or '' + +cdef int cb_on_message_begin(cparser.llhttp_t* parser) except -1: + cdef HttpParser pyparser = parser.data + + pyparser._started = True + pyparser._headers = [] + pyparser._seen_singletons = set() + pyparser._raw_headers = [] + PyByteArray_Resize(pyparser._buf, 0) + pyparser._path = None + pyparser._reason = None + return 0 + + +cdef int cb_on_url(cparser.llhttp_t* parser, + const char *at, size_t length) except -1: + cdef HttpParser pyparser = parser.data + try: + if len(pyparser._buf) + length > pyparser._max_line_size: + status = pyparser._buf + at[:length] + raise LineTooLong(status[:100] + b"...", pyparser._max_line_size) + extend(pyparser._buf, at, length) + except BaseException as ex: + pyparser._last_error = ex + return -1 + else: + return 0 + + +cdef int cb_on_status(cparser.llhttp_t* parser, + const char *at, size_t length) except -1: + cdef HttpParser pyparser = parser.data + try: + if len(pyparser._buf) + length > pyparser._max_line_size: + reason = pyparser._buf + at[:length] + raise LineTooLong(reason[:100] + b"...", pyparser._max_line_size) + extend(pyparser._buf, at, length) + except BaseException as ex: + pyparser._last_error = ex + return -1 + else: + return 0 + + +cdef int cb_on_header_field(cparser.llhttp_t* parser, + const char *at, size_t length) except -1: + cdef HttpParser pyparser = parser.data + cdef Py_ssize_t size + try: + pyparser._on_status_complete() + size = len(pyparser._raw_name) + length + if size > pyparser._max_field_size: + name = pyparser._raw_name + at[:length] + raise LineTooLong(name[:100] + b"...", pyparser._max_field_size) + pyparser._header_name_size = size + pyparser._on_header_field(at, length) + except BaseException as ex: + pyparser._last_error = ex + return -1 + else: + return 0 + + +cdef int cb_on_header_value(cparser.llhttp_t* parser, + const char *at, size_t length) except -1: + cdef HttpParser pyparser = parser.data + cdef Py_ssize_t size + try: + size = len(pyparser._raw_value) + length + if pyparser._header_name_size + size > pyparser._max_field_size: + value = pyparser._raw_value + at[:length] + raise LineTooLong(value[:100] + b"...", pyparser._max_field_size) + pyparser._on_header_value(at, length) + except BaseException as ex: + pyparser._last_error = ex + return -1 + else: + return 0 + + +cdef int cb_on_headers_complete(cparser.llhttp_t* parser) except -1: + cdef HttpParser pyparser = parser.data + try: + pyparser._on_status_complete() + pyparser._on_headers_complete() + except BaseException as exc: + pyparser._last_error = exc + return -1 + else: + if not pyparser._response_with_body: + return 1 + return 0 + + +cdef int cb_on_body(cparser.llhttp_t* parser, + const char *at, size_t length) except -1: + cdef HttpParser pyparser = parser.data + cdef bytes body = at[:length] + while body or pyparser._more_data_available: + try: + pyparser._more_data_available = pyparser._payload.feed_data(body, length) + except BaseException as underlying_exc: + reraised_exc = underlying_exc + if pyparser._payload_exception is not None: + reraised_exc = pyparser._payload_exception(str(underlying_exc)) + + set_exception(pyparser._payload, reraised_exc, underlying_exc) + + pyparser._payload_error = 1 + pyparser._paused = False + return -1 + body = b"" + length = 0 + + if pyparser._paused: + pyparser._paused = False + return cparser.HPE_PAUSED + pyparser._paused = False + return 0 + + +cdef int cb_on_message_complete(cparser.llhttp_t* parser) except -1: + cdef HttpParser pyparser = parser.data + try: + pyparser._started = False + pyparser._on_message_complete() + except BaseException as exc: + pyparser._last_error = exc + return -1 + else: + if pyparser._max_msg_queue_size: + pyparser._msg_in_flight += 1 + if pyparser._msg_in_flight >= pyparser._max_msg_queue_size: + # Queue full: pause llhttp between messages. feed_data() buffers + # the remainder as tail; resumes once the queue drains. + return cparser.HPE_PAUSED + return 0 + + +cdef int cb_on_chunk_header(cparser.llhttp_t* parser) except -1: + cdef HttpParser pyparser = parser.data + try: + pyparser._on_chunk_header() + except BaseException as exc: + pyparser._last_error = exc + return -1 + else: + return 0 + + +cdef int cb_on_chunk_complete(cparser.llhttp_t* parser) except -1: + cdef HttpParser pyparser = parser.data + try: + pyparser._on_chunk_complete() + except BaseException as exc: + pyparser._last_error = exc + return -1 + else: + return 0 + + +cdef parser_error_from_errno(cparser.llhttp_t* parser, data, pointer): + cdef cparser.llhttp_errno_t errno = cparser.llhttp_get_errno(parser) + cdef bytes desc = cparser.llhttp_get_error_reason(parser) + + err_msg = "{}:\n\n {!r}\n {}".format(desc.decode("latin-1"), data, pointer) + + if errno in {cparser.HPE_CB_MESSAGE_BEGIN, + cparser.HPE_CB_HEADERS_COMPLETE, + cparser.HPE_CB_MESSAGE_COMPLETE, + cparser.HPE_CB_CHUNK_HEADER, + cparser.HPE_CB_CHUNK_COMPLETE, + cparser.HPE_INVALID_HEADER_TOKEN, + cparser.HPE_INVALID_CONTENT_LENGTH, + cparser.HPE_INVALID_CHUNK_SIZE, + cparser.HPE_INVALID_EOF_STATE, + cparser.HPE_INVALID_TRANSFER_ENCODING}: + return BadHttpMessage(err_msg) + elif errno == cparser.HPE_INVALID_METHOD: + if data.startswith(b"\x16\x03"): + return BadHttpMethod(error="Received HTTPS traffic on an HTTP port") + return BadHttpMethod(error=err_msg) + elif errno in {cparser.HPE_INVALID_STATUS, + cparser.HPE_INVALID_VERSION, + cparser.HPE_INVALID_CONSTANT}: + return BadStatusLine(error=f"Bad status line:\n {err_msg}") + elif errno == cparser.HPE_INVALID_URL: + return InvalidURLError(err_msg) + + return BadHttpMessage(err_msg) diff --git a/py_modules/aiohttp/_http_writer.cpython-311-x86_64-linux-gnu.so b/py_modules/aiohttp/_http_writer.cpython-311-x86_64-linux-gnu.so new file mode 100755 index 0000000..2443b5f Binary files /dev/null and b/py_modules/aiohttp/_http_writer.cpython-311-x86_64-linux-gnu.so differ diff --git a/py_modules/aiohttp/_http_writer.pyx b/py_modules/aiohttp/_http_writer.pyx new file mode 100644 index 0000000..7859ebd --- /dev/null +++ b/py_modules/aiohttp/_http_writer.pyx @@ -0,0 +1,164 @@ +from cpython.bytes cimport PyBytes_FromStringAndSize +from cpython.exc cimport PyErr_NoMemory +from cpython.mem cimport PyMem_Free, PyMem_Malloc, PyMem_Realloc +from cpython.object cimport PyObject_Str +from libc.stdint cimport uint8_t, uint64_t +from libc.string cimport memcpy + +from multidict import istr + +DEF BUF_SIZE = 16 * 1024 # 16KiB + +cdef object _istr = istr + + +# ----------------- writer --------------------------- + +cdef struct Writer: + char *buf + Py_ssize_t size + Py_ssize_t pos + bint heap_allocated + +cdef inline void _init_writer(Writer* writer, char *buf): + writer.buf = buf + writer.size = BUF_SIZE + writer.pos = 0 + writer.heap_allocated = 0 + + +cdef inline void _release_writer(Writer* writer): + if writer.heap_allocated: + PyMem_Free(writer.buf) + + +cdef inline int _write_byte(Writer* writer, uint8_t ch): + cdef char * buf + cdef Py_ssize_t size + + if writer.pos == writer.size: + # reallocate + size = writer.size + BUF_SIZE + if not writer.heap_allocated: + buf = PyMem_Malloc(size) + if buf == NULL: + PyErr_NoMemory() + return -1 + memcpy(buf, writer.buf, writer.size) + else: + buf = PyMem_Realloc(writer.buf, size) + if buf == NULL: + PyErr_NoMemory() + return -1 + writer.buf = buf + writer.size = size + writer.heap_allocated = 1 + writer.buf[writer.pos] = ch + writer.pos += 1 + return 0 + + +cdef inline int _write_utf8(Writer* writer, Py_UCS4 symbol): + cdef uint64_t utf = symbol + + if utf < 0x80: + return _write_byte(writer, utf) + elif utf < 0x800: + if _write_byte(writer, (0xc0 | (utf >> 6))) < 0: + return -1 + return _write_byte(writer, (0x80 | (utf & 0x3f))) + elif 0xD800 <= utf <= 0xDFFF: + # surogate pair, ignored + return 0 + elif utf < 0x10000: + if _write_byte(writer, (0xe0 | (utf >> 12))) < 0: + return -1 + if _write_byte(writer, (0x80 | ((utf >> 6) & 0x3f))) < 0: + return -1 + return _write_byte(writer, (0x80 | (utf & 0x3f))) + elif utf > 0x10FFFF: + # symbol is too large + return 0 + else: + if _write_byte(writer, (0xf0 | (utf >> 18))) < 0: + return -1 + if _write_byte(writer, + (0x80 | ((utf >> 12) & 0x3f))) < 0: + return -1 + if _write_byte(writer, + (0x80 | ((utf >> 6) & 0x3f))) < 0: + return -1 + return _write_byte(writer, (0x80 | (utf & 0x3f))) + + +cdef inline int _write_str(Writer* writer, str s): + cdef Py_UCS4 ch + for ch in s: + if _write_utf8(writer, ch) < 0: + return -1 + + +cdef inline int _write_str_raise_on_nlcr(Writer* writer, object s): + cdef Py_UCS4 ch + cdef str out_str + if type(s) is str: + out_str = s + elif type(s) is _istr: + out_str = PyObject_Str(s) + elif not isinstance(s, str): + raise TypeError("Cannot serialize non-str key {!r}".format(s)) + else: + out_str = str(s) + + for ch in out_str: + # https://www.rfc-editor.org/info/rfc9110/#section-5.5-5 + # https://www.rfc-editor.org/info/rfc9112/#section-4-3 + if (ch < 0x20 and ch != 0x09) or ch == 0x7F: + raise ValueError( + "Forbidden control character detected in headers. " + "Potential header injection attack." + ) + if _write_utf8(writer, ch) < 0: + return -1 + + +# --------------- _serialize_headers ---------------------- + +def _serialize_headers(str status_line, headers): + cdef Writer writer + cdef object key + cdef object val + cdef char buf[BUF_SIZE] + + _init_writer(&writer, buf) + + try: + if _write_str_raise_on_nlcr(&writer, status_line) < 0: + raise + if _write_byte(&writer, b'\r') < 0: + raise + if _write_byte(&writer, b'\n') < 0: + raise + + for key, val in headers.items(): + if _write_str_raise_on_nlcr(&writer, key) < 0: + raise + if _write_byte(&writer, b':') < 0: + raise + if _write_byte(&writer, b' ') < 0: + raise + if _write_str_raise_on_nlcr(&writer, val) < 0: + raise + if _write_byte(&writer, b'\r') < 0: + raise + if _write_byte(&writer, b'\n') < 0: + raise + + if _write_byte(&writer, b'\r') < 0: + raise + if _write_byte(&writer, b'\n') < 0: + raise + + return PyBytes_FromStringAndSize(writer.buf, writer.pos) + finally: + _release_writer(&writer) diff --git a/py_modules/aiohttp/_websocket/__init__.py b/py_modules/aiohttp/_websocket/__init__.py new file mode 100644 index 0000000..836257c --- /dev/null +++ b/py_modules/aiohttp/_websocket/__init__.py @@ -0,0 +1 @@ +"""WebSocket protocol versions 13 and 8.""" diff --git a/py_modules/aiohttp/_websocket/helpers.py b/py_modules/aiohttp/_websocket/helpers.py new file mode 100644 index 0000000..6a42a00 --- /dev/null +++ b/py_modules/aiohttp/_websocket/helpers.py @@ -0,0 +1,148 @@ +"""Helpers for WebSocket protocol versions 13 and 8.""" + +import functools +import re +from re import Pattern +from struct import Struct +from typing import TYPE_CHECKING, Final + +from ..helpers import NO_EXTENSIONS +from .models import WSHandshakeError + +UNPACK_LEN3 = Struct("!Q").unpack_from +UNPACK_CLOSE_CODE = Struct("!H").unpack +PACK_LEN1 = Struct("!BB").pack +PACK_LEN2 = Struct("!BBH").pack +PACK_LEN3 = Struct("!BBQ").pack +PACK_CLOSE_CODE = Struct("!H").pack +PACK_RANDBITS = Struct("!L").pack +MSG_SIZE: Final[int] = 2**14 +MASK_LEN: Final[int] = 4 + +WS_KEY: Final[bytes] = b"258EAFA5-E914-47DA-95CA-C5AB0DC85B11" + + +# Used by _websocket_mask_python +@functools.lru_cache +def _xor_table() -> list[bytes]: + return [bytes(a ^ b for a in range(256)) for b in range(256)] + + +def _websocket_mask_python(mask: bytes, data: bytearray) -> None: + """Websocket masking function. + + `mask` is a `bytes` object of length 4; `data` is a `bytearray` + object of any length. The contents of `data` are masked with `mask`, + as specified in section 5.3 of RFC 6455. + + Note that this function mutates the `data` argument. + + This pure-python implementation may be replaced by an optimized + version when available. + + """ + assert isinstance(data, bytearray), data + assert len(mask) == 4, mask + + if data: + _XOR_TABLE = _xor_table() + a, b, c, d = (_XOR_TABLE[n] for n in mask) + data[::4] = data[::4].translate(a) + data[1::4] = data[1::4].translate(b) + data[2::4] = data[2::4].translate(c) + data[3::4] = data[3::4].translate(d) + + +if TYPE_CHECKING or NO_EXTENSIONS: # pragma: no cover + websocket_mask = _websocket_mask_python +else: + try: + from .mask import _websocket_mask_cython # type: ignore[import-not-found] + + websocket_mask = _websocket_mask_cython + except ImportError: # pragma: no cover + websocket_mask = _websocket_mask_python + + +_WS_EXT_RE: Final[Pattern[str]] = re.compile( + r"^(?:;\s*(?:" + r"(server_no_context_takeover)|" + r"(client_no_context_takeover)|" + r"(server_max_window_bits(?:=(\d+))?)|" + r"(client_max_window_bits(?:=(\d+))?)))*$" +) + +_WS_EXT_RE_SPLIT: Final[Pattern[str]] = re.compile(r"permessage-deflate([^,]+)?") + + +def ws_ext_parse(extstr: str | None, isserver: bool = False) -> tuple[int, bool]: + if not extstr: + return 0, False + + compress = 0 + notakeover = False + for ext in _WS_EXT_RE_SPLIT.finditer(extstr): + defext = ext.group(1) + # Return compress = 15 when get `permessage-deflate` + if not defext: + compress = 15 + break + match = _WS_EXT_RE.match(defext) + if match: + compress = 15 + if isserver: + # Server never fail to detect compress handshake. + # Server does not need to send max wbit to client + if match.group(4): + compress = int(match.group(4)) + # Group3 must match if group4 matches + # Compress wbit 8 does not support in zlib + # If compress level not support, + # CONTINUE to next extension + if compress > 15 or compress < 9: + compress = 0 + continue + if match.group(1): + notakeover = True + # Ignore regex group 5 & 6 for client_max_window_bits + break + else: + if match.group(6): + compress = int(match.group(6)) + # Group5 must match if group6 matches + # Compress wbit 8 does not support in zlib + # If compress level not support, + # FAIL the parse progress + if compress > 15 or compress < 9: + raise WSHandshakeError("Invalid window size") + if match.group(2): + notakeover = True + # Ignore regex group 5 & 6 for client_max_window_bits + break + # Return Fail if client side and not match + elif not isserver: + raise WSHandshakeError("Extension for deflate not supported" + ext.group(1)) + + return compress, notakeover + + +def ws_ext_gen( + compress: int = 15, isserver: bool = False, server_notakeover: bool = False +) -> str: + # client_notakeover=False not used for server + # compress wbit 8 does not support in zlib + if compress < 9 or compress > 15: + raise ValueError( + "Compress wbits must between 9 and 15, zlib does not support wbits=8" + ) + enabledext = ["permessage-deflate"] + if not isserver: + enabledext.append("client_max_window_bits") + + if compress < 15: + enabledext.append("server_max_window_bits=" + str(compress)) + if server_notakeover: + enabledext.append("server_no_context_takeover") + # if client_notakeover: + # enabledext.append('client_no_context_takeover') + return "; ".join(enabledext) diff --git a/py_modules/aiohttp/_websocket/mask.cpython-311-x86_64-linux-gnu.so b/py_modules/aiohttp/_websocket/mask.cpython-311-x86_64-linux-gnu.so new file mode 100755 index 0000000..5092599 Binary files /dev/null and b/py_modules/aiohttp/_websocket/mask.cpython-311-x86_64-linux-gnu.so differ diff --git a/py_modules/aiohttp/_websocket/mask.pxd b/py_modules/aiohttp/_websocket/mask.pxd new file mode 100644 index 0000000..90983de --- /dev/null +++ b/py_modules/aiohttp/_websocket/mask.pxd @@ -0,0 +1,3 @@ +"""Cython declarations for websocket masking.""" + +cpdef void _websocket_mask_cython(bytes mask, bytearray data) diff --git a/py_modules/aiohttp/_websocket/mask.pyx b/py_modules/aiohttp/_websocket/mask.pyx new file mode 100644 index 0000000..2d956c8 --- /dev/null +++ b/py_modules/aiohttp/_websocket/mask.pyx @@ -0,0 +1,48 @@ +from cpython cimport PyBytes_AsString + + +#from cpython cimport PyByteArray_AsString # cython still not exports that +cdef extern from "Python.h": + char* PyByteArray_AsString(bytearray ba) except NULL + +from libc.stdint cimport uint32_t, uint64_t, uintmax_t + + +cpdef void _websocket_mask_cython(bytes mask, bytearray data): + """Note, this function mutates its `data` argument + """ + cdef: + Py_ssize_t data_len, i + # bit operations on signed integers are implementation-specific + unsigned char * in_buf + const unsigned char * mask_buf + uint32_t uint32_msk + uint64_t uint64_msk + + assert len(mask) == 4 + + data_len = len(data) + in_buf = PyByteArray_AsString(data) + mask_buf = PyBytes_AsString(mask) + uint32_msk = (mask_buf)[0] + + # TODO: align in_data ptr to achieve even faster speeds + # does it need in python ?! malloc() always aligns to sizeof(long) bytes + + if sizeof(size_t) >= 8: + uint64_msk = uint32_msk + uint64_msk = (uint64_msk << 32) | uint32_msk + + while data_len >= 8: + (in_buf)[0] ^= uint64_msk + in_buf += 8 + data_len -= 8 + + + while data_len >= 4: + (in_buf)[0] ^= uint32_msk + in_buf += 4 + data_len -= 4 + + for i in range(0, data_len): + in_buf[i] ^= mask_buf[i] diff --git a/py_modules/aiohttp/_websocket/models.py b/py_modules/aiohttp/_websocket/models.py new file mode 100644 index 0000000..3cdaa92 --- /dev/null +++ b/py_modules/aiohttp/_websocket/models.py @@ -0,0 +1,107 @@ +"""Models for WebSocket protocol versions 13 and 8.""" + +import json +from collections.abc import Callable +from enum import IntEnum +from typing import Any, Final, NamedTuple, cast + +WS_DEFLATE_TRAILING: Final[bytes] = bytes([0x00, 0x00, 0xFF, 0xFF]) + + +class WSCloseCode(IntEnum): + OK = 1000 + GOING_AWAY = 1001 + PROTOCOL_ERROR = 1002 + UNSUPPORTED_DATA = 1003 + ABNORMAL_CLOSURE = 1006 + INVALID_TEXT = 1007 + POLICY_VIOLATION = 1008 + MESSAGE_TOO_BIG = 1009 + MANDATORY_EXTENSION = 1010 + INTERNAL_ERROR = 1011 + SERVICE_RESTART = 1012 + TRY_AGAIN_LATER = 1013 + BAD_GATEWAY = 1014 + + +class WSMsgType(IntEnum): + # websocket spec types + CONTINUATION = 0x0 + TEXT = 0x1 + BINARY = 0x2 + PING = 0x9 + PONG = 0xA + CLOSE = 0x8 + + # aiohttp specific types + CLOSING = 0x100 + CLOSED = 0x101 + ERROR = 0x102 + + text = TEXT + binary = BINARY + ping = PING + pong = PONG + close = CLOSE + closing = CLOSING + closed = CLOSED + error = ERROR + + +class WSMessage(NamedTuple): + type: WSMsgType + # To type correctly, this would need some kind of tagged union for each type. + data: Any + extra: str | None + + def json(self, *, loads: Callable[[Any], Any] = json.loads) -> Any: + """Return parsed JSON data. + + .. versionadded:: 0.22 + """ + return loads(self.data) + + +class WSMessageTextBytes(NamedTuple): + """WebSocket TEXT message with raw bytes (no UTF-8 decoding).""" + + type: WSMsgType + # To type correctly, this would need some kind of tagged union for each type. + # In 4.0, we use a union of message types to properly type data, but in 3.x + # we keep it as Any to avoid a breaking change. + data: Any + extra: str | None + + def json(self, *, loads: Callable[[Any], Any] = json.loads) -> Any: + """Return parsed JSON data.""" + return loads(self.data) + + +# Type aliases for message types based on decode_text setting +# When decode_text=True, TEXT messages have str data (WSMessage) +# When decode_text=False, TEXT messages have bytes data (WSMessageTextBytes) +WSMessageDecodeText = WSMessage +WSMessageNoDecodeText = WSMessage | WSMessageTextBytes + + +# Constructing the tuple directly to avoid the overhead of +# the lambda and arg processing since NamedTuples are constructed +# with a run time built lambda +# https://github.com/python/cpython/blob/d83fcf8371f2f33c7797bc8f5423a8bca8c46e5c/Lib/collections/__init__.py#L441 +WS_CLOSED_MESSAGE = tuple.__new__(WSMessage, (WSMsgType.CLOSED, None, None)) +WS_CLOSING_MESSAGE = tuple.__new__(WSMessage, (WSMsgType.CLOSING, None, None)) + + +class WebSocketError(Exception): + """WebSocket protocol parser error.""" + + def __init__(self, code: int, message: str) -> None: + self.code = code + super().__init__(code, message) + + def __str__(self) -> str: + return cast(str, self.args[1]) + + +class WSHandshakeError(Exception): + """WebSocket protocol handshake error.""" diff --git a/py_modules/aiohttp/_websocket/reader.py b/py_modules/aiohttp/_websocket/reader.py new file mode 100644 index 0000000..23f3226 --- /dev/null +++ b/py_modules/aiohttp/_websocket/reader.py @@ -0,0 +1,31 @@ +"""Reader for WebSocket protocol versions 13 and 8.""" + +from typing import TYPE_CHECKING + +from ..helpers import NO_EXTENSIONS + +if TYPE_CHECKING or NO_EXTENSIONS: # pragma: no cover + from .reader_py import ( + WebSocketDataQueue as WebSocketDataQueuePython, + WebSocketReader as WebSocketReaderPython, + ) + + WebSocketReader = WebSocketReaderPython + WebSocketDataQueue = WebSocketDataQueuePython +else: + try: + from .reader_c import ( # type: ignore[import-not-found] + WebSocketDataQueue as WebSocketDataQueueCython, + WebSocketReader as WebSocketReaderCython, + ) + + WebSocketReader = WebSocketReaderCython + WebSocketDataQueue = WebSocketDataQueueCython + except ImportError: # pragma: no cover + from .reader_py import ( + WebSocketDataQueue as WebSocketDataQueuePython, + WebSocketReader as WebSocketReaderPython, + ) + + WebSocketReader = WebSocketReaderPython + WebSocketDataQueue = WebSocketDataQueuePython diff --git a/py_modules/aiohttp/_websocket/reader_c.cpython-311-x86_64-linux-gnu.so b/py_modules/aiohttp/_websocket/reader_c.cpython-311-x86_64-linux-gnu.so new file mode 100755 index 0000000..3f73230 Binary files /dev/null and b/py_modules/aiohttp/_websocket/reader_c.cpython-311-x86_64-linux-gnu.so differ diff --git a/py_modules/aiohttp/_websocket/reader_c.pxd b/py_modules/aiohttp/_websocket/reader_c.pxd new file mode 100644 index 0000000..5aa067f --- /dev/null +++ b/py_modules/aiohttp/_websocket/reader_c.pxd @@ -0,0 +1,112 @@ +import cython + +from .mask cimport _websocket_mask_cython as websocket_mask + + +cdef unsigned int READ_HEADER +cdef unsigned int READ_PAYLOAD_LENGTH +cdef unsigned int READ_PAYLOAD_MASK +cdef unsigned int READ_PAYLOAD + +cdef int OP_CODE_NOT_SET +cdef int OP_CODE_CONTINUATION +cdef int OP_CODE_TEXT +cdef int OP_CODE_BINARY +cdef int OP_CODE_CLOSE +cdef int OP_CODE_PING +cdef int OP_CODE_PONG + +cdef int COMPRESSED_NOT_SET +cdef int COMPRESSED_FALSE +cdef int COMPRESSED_TRUE + +cdef object UNPACK_LEN3 +cdef object UNPACK_CLOSE_CODE +cdef object TUPLE_NEW + +cdef object WSMsgType +cdef object WSMessage +cdef object WSMessageTextBytes + +cdef object WS_MSG_TYPE_TEXT +cdef object WS_MSG_TYPE_BINARY + +cdef set ALLOWED_CLOSE_CODES +cdef set MESSAGE_TYPES_WITH_CONTENT + +cdef tuple EMPTY_FRAME +cdef tuple EMPTY_FRAME_ERROR + +cdef class WebSocketDataQueue: + + cdef unsigned int _size + cdef public object _protocol + cdef unsigned int _limit + cdef object _loop + cdef bint _eof + cdef object _waiter + cdef object _exception + cdef public object _buffer + cdef object _get_buffer + cdef object _put_buffer + + cdef void _release_waiter(self) + + cpdef void feed_data(self, object data, unsigned int size) + + @cython.locals(size="unsigned int") + cdef _read_from_buffer(self) + +cdef class WebSocketReader: + + cdef WebSocketDataQueue queue + cdef unsigned int _max_msg_size + cdef bint _decode_text + + cdef Exception _exc + cdef bytearray _partial + cdef unsigned int _state + + cdef int _opcode + cdef bint _frame_fin + cdef int _frame_opcode + cdef list _payload_fragments + cdef Py_ssize_t _frame_payload_len + + cdef bytes _tail + cdef bint _has_mask + cdef bytes _frame_mask + cdef Py_ssize_t _payload_bytes_to_read + cdef unsigned int _payload_len_flag + cdef int _compressed + cdef object _decompressobj + cdef bint _compress + + cpdef tuple feed_data(self, object data) + + @cython.locals( + is_continuation=bint, + fin=bint, + has_partial=bint, + payload_merged=bytes, + ) + cpdef void _handle_frame(self, bint fin, int opcode, object payload, int compressed) except * + + @cython.locals( + start_pos=Py_ssize_t, + data_len=Py_ssize_t, + length=Py_ssize_t, + chunk_size=Py_ssize_t, + chunk_len=Py_ssize_t, + data_len=Py_ssize_t, + data_cstr="const unsigned char *", + first_byte="unsigned char", + second_byte="unsigned char", + f_start_pos=Py_ssize_t, + f_end_pos=Py_ssize_t, + has_mask=bint, + fin=bint, + had_fragments=Py_ssize_t, + payload_bytearray=bytearray, + ) + cpdef void _feed_data(self, bytes data) except * diff --git a/py_modules/aiohttp/_websocket/reader_c.py b/py_modules/aiohttp/_websocket/reader_c.py new file mode 100644 index 0000000..ff5a5b3 --- /dev/null +++ b/py_modules/aiohttp/_websocket/reader_c.py @@ -0,0 +1,513 @@ +"""Reader for WebSocket protocol versions 13 and 8.""" + +import asyncio +import builtins +from collections import deque +from typing import Final + +from ..base_protocol import BaseProtocol +from ..compression_utils import ZLibDecompressor +from ..helpers import _EXC_SENTINEL, set_exception +from ..streams import EofStream +from .helpers import UNPACK_CLOSE_CODE, UNPACK_LEN3, websocket_mask +from .models import ( + WS_DEFLATE_TRAILING, + WebSocketError, + WSCloseCode, + WSMessage, + WSMessageTextBytes, + WSMsgType, +) + +ALLOWED_CLOSE_CODES: Final[set[int]] = {int(i) for i in WSCloseCode} + +# States for the reader, used to parse the WebSocket frame +# integer values are used so they can be cythonized +READ_HEADER = 1 +READ_PAYLOAD_LENGTH = 2 +READ_PAYLOAD_MASK = 3 +READ_PAYLOAD = 4 + +WS_MSG_TYPE_BINARY = WSMsgType.BINARY +WS_MSG_TYPE_TEXT = WSMsgType.TEXT + +# WSMsgType values unpacked so they can by cythonized to ints +OP_CODE_NOT_SET = -1 +OP_CODE_CONTINUATION = WSMsgType.CONTINUATION.value +OP_CODE_TEXT = WSMsgType.TEXT.value +OP_CODE_BINARY = WSMsgType.BINARY.value +OP_CODE_CLOSE = WSMsgType.CLOSE.value +OP_CODE_PING = WSMsgType.PING.value +OP_CODE_PONG = WSMsgType.PONG.value + +EMPTY_FRAME_ERROR = (True, b"") +EMPTY_FRAME = (False, b"") + +COMPRESSED_NOT_SET = -1 +COMPRESSED_FALSE = 0 +COMPRESSED_TRUE = 1 + +TUPLE_NEW = tuple.__new__ + +cython_int = int # Typed to int in Python, but cython with use a signed int in the pxd + + +class WebSocketDataQueue: + """WebSocketDataQueue resumes and pauses an underlying stream. + + It is a destination for WebSocket data. + """ + + def __init__( + self, protocol: BaseProtocol, limit: int, *, loop: asyncio.AbstractEventLoop + ) -> None: + self._size = 0 + self._protocol = protocol + self._limit = limit * 2 + self._loop = loop + self._eof = False + self._waiter: asyncio.Future[None] | None = None + self._exception: BaseException | None = None + self._buffer: deque[tuple[WSMessage | WSMessageTextBytes, int]] = deque() + self._get_buffer = self._buffer.popleft + self._put_buffer = self._buffer.append + + def is_eof(self) -> bool: + return self._eof + + def exception(self) -> BaseException | None: + return self._exception + + def set_exception( + self, + exc: BaseException, + exc_cause: builtins.BaseException = _EXC_SENTINEL, + ) -> None: + self._eof = True + self._exception = exc + if (waiter := self._waiter) is not None: + self._waiter = None + set_exception(waiter, exc, exc_cause) + + def _release_waiter(self) -> None: + if (waiter := self._waiter) is None: + return + self._waiter = None + if not waiter.done(): + waiter.set_result(None) + + def feed_eof(self) -> None: + self._eof = True + self._release_waiter() + self._exception = None # Break cyclic references + + def feed_data( + self, data: "WSMessage | WSMessageTextBytes", size: "cython_int" + ) -> None: + self._size += size + self._put_buffer((data, size)) + self._release_waiter() + if self._size > self._limit and not self._protocol._reading_paused: + self._protocol.pause_reading() + + async def read(self) -> WSMessage | WSMessageTextBytes: + if not self._buffer and not self._eof: + assert not self._waiter + self._waiter = self._loop.create_future() + try: + await self._waiter + except (asyncio.CancelledError, asyncio.TimeoutError): + self._waiter = None + raise + return self._read_from_buffer() + + def _read_from_buffer(self) -> WSMessage | WSMessageTextBytes: + if self._buffer: + data, size = self._get_buffer() + self._size -= size + if self._size < self._limit and self._protocol._reading_paused: + self._protocol.resume_reading() + return data + if self._exception is not None: + raise self._exception + raise EofStream + + +class WebSocketReader: + def __init__( + self, + queue: WebSocketDataQueue, + max_msg_size: int, + compress: bool, + decode_text: bool, + ) -> None: + self.queue = queue + self._max_msg_size = max_msg_size + self._decode_text = decode_text + + self._exc: Exception | None = None + self._partial = bytearray() + self._state = READ_HEADER + + self._opcode: int = OP_CODE_NOT_SET + self._frame_fin = False + self._frame_opcode: int = OP_CODE_NOT_SET + self._payload_fragments: list[bytes] = [] + self._frame_payload_len = 0 + + self._tail: bytes = b"" + self._has_mask = False + self._frame_mask: bytes | None = None + self._payload_bytes_to_read = 0 + self._payload_len_flag = 0 + self._compressed: int = COMPRESSED_NOT_SET + self._decompressobj: ZLibDecompressor | None = None + self._compress = compress + + def feed_eof(self) -> None: + self.queue.feed_eof() + + # data can be bytearray on Windows because proactor event loop uses bytearray + # and asyncio types this to Union[bytes, bytearray, memoryview] so we need + # coerce data to bytes if it is not + def feed_data(self, data: bytes | bytearray | memoryview) -> tuple[bool, bytes]: + if type(data) is not bytes: + data = bytes(data) + + if self._exc is not None: + return True, data + + try: + self._feed_data(data) + except Exception as exc: + self._exc = exc + set_exception(self.queue, exc) + return EMPTY_FRAME_ERROR + + return EMPTY_FRAME + + def _handle_frame( + self, + fin: bool, + opcode: int | cython_int, # Union intended: Cython pxd uses C int + payload: bytes | bytearray, + compressed: int | cython_int, # Union intended: Cython pxd uses C int + ) -> None: + msg: WSMessage + if opcode in {OP_CODE_TEXT, OP_CODE_BINARY, OP_CODE_CONTINUATION}: + # Validate continuation frames before processing + if opcode == OP_CODE_CONTINUATION and self._opcode == OP_CODE_NOT_SET: + raise WebSocketError( + WSCloseCode.PROTOCOL_ERROR, + "Continuation frame for non started message", + ) + + # load text/binary + if not fin: + # got partial frame payload + if opcode != OP_CODE_CONTINUATION: + self._opcode = opcode + self._partial += payload + return + + has_partial = bool(self._partial) + if opcode == OP_CODE_CONTINUATION: + opcode = self._opcode + self._opcode = OP_CODE_NOT_SET + # previous frame was non finished + # we should get continuation opcode + elif has_partial: + raise WebSocketError( + WSCloseCode.PROTOCOL_ERROR, + "The opcode in non-fin frame is expected " + f"to be zero, got {opcode!r}", + ) + + assembled_payload: bytes | bytearray + if has_partial: + assembled_payload = self._partial + payload + self._partial.clear() + else: + assembled_payload = payload + + # Decompress process must to be done after all packets + # received. + if compressed: + if not self._decompressobj: + self._decompressobj = ZLibDecompressor(suppress_deflate_header=True) + # XXX: It's possible that the zlib backend (isal is known to + # do this, maybe others too?) will return max_length bytes, + # but internally buffer more data such that the payload is + # >max_length, so we return one extra byte and if we're able + # to do that, then the message is too big. + payload_merged = self._decompressobj.decompress_sync( + assembled_payload + WS_DEFLATE_TRAILING, + ( + self._max_msg_size + 1 + if self._max_msg_size + else self._max_msg_size + ), + ) + if self._max_msg_size and len(payload_merged) > self._max_msg_size: + raise WebSocketError( + WSCloseCode.MESSAGE_TOO_BIG, + f"Decompressed message exceeds size limit {self._max_msg_size}", + ) + elif type(assembled_payload) is bytes: + payload_merged = assembled_payload + else: + payload_merged = bytes(assembled_payload) + + if opcode == OP_CODE_TEXT: + if self._decode_text: + try: + text = payload_merged.decode("utf-8") + except UnicodeDecodeError as exc: + raise WebSocketError( + WSCloseCode.INVALID_TEXT, "Invalid UTF-8 text message" + ) from exc + + # XXX: The Text and Binary messages here can be a performance + # bottleneck, so we use tuple.__new__ to improve performance. + # This is not type safe, but many tests should fail in + # test_client_ws_functional.py if this is wrong. + self.queue.feed_data( + TUPLE_NEW(WSMessage, (WS_MSG_TYPE_TEXT, text, "")), + len(payload_merged), + ) + else: + # Return raw bytes for TEXT messages when decode_text=False + self.queue.feed_data( + TUPLE_NEW( + WSMessageTextBytes, (WS_MSG_TYPE_TEXT, payload_merged, "") + ), + len(payload_merged), + ) + else: + self.queue.feed_data( + TUPLE_NEW(WSMessage, (WS_MSG_TYPE_BINARY, payload_merged, "")), + len(payload_merged), + ) + elif opcode == OP_CODE_CLOSE: + if len(payload) >= 2: + close_code = UNPACK_CLOSE_CODE(payload[:2])[0] + if close_code < 3000 and close_code not in ALLOWED_CLOSE_CODES: + raise WebSocketError( + WSCloseCode.PROTOCOL_ERROR, + f"Invalid close code: {close_code}", + ) + try: + close_message = payload[2:].decode("utf-8") + except UnicodeDecodeError as exc: + raise WebSocketError( + WSCloseCode.INVALID_TEXT, "Invalid UTF-8 text message" + ) from exc + msg = TUPLE_NEW(WSMessage, (WSMsgType.CLOSE, close_code, close_message)) + elif payload: + raise WebSocketError( + WSCloseCode.PROTOCOL_ERROR, + f"Invalid close frame: {fin} {opcode} {payload!r}", + ) + else: + msg = TUPLE_NEW(WSMessage, (WSMsgType.CLOSE, 0, "")) + + self.queue.feed_data(msg, 0) + elif opcode == OP_CODE_PING: + msg = TUPLE_NEW(WSMessage, (WSMsgType.PING, payload, "")) + self.queue.feed_data(msg, len(payload)) + elif opcode == OP_CODE_PONG: + msg = TUPLE_NEW(WSMessage, (WSMsgType.PONG, payload, "")) + self.queue.feed_data(msg, len(payload)) + else: + raise WebSocketError( + WSCloseCode.PROTOCOL_ERROR, f"Unexpected opcode={opcode!r}" + ) + + def _feed_data(self, data: bytes) -> None: + """Return the next frame from the socket.""" + if self._tail: + data, self._tail = self._tail + data, b"" + + start_pos: int = 0 + data_len = len(data) + data_cstr = data + + while True: + # read header + if self._state == READ_HEADER: + if data_len - start_pos < 2: + break + first_byte = data_cstr[start_pos] + second_byte = data_cstr[start_pos + 1] + start_pos += 2 + + fin = (first_byte >> 7) & 1 + rsv1 = (first_byte >> 6) & 1 + rsv2 = (first_byte >> 5) & 1 + rsv3 = (first_byte >> 4) & 1 + opcode = first_byte & 0xF + + # frame-fin = %x0 ; more frames of this message follow + # / %x1 ; final frame of this message + # frame-rsv1 = %x0 ; + # 1 bit, MUST be 0 unless negotiated otherwise + # frame-rsv2 = %x0 ; + # 1 bit, MUST be 0 unless negotiated otherwise + # frame-rsv3 = %x0 ; + # 1 bit, MUST be 0 unless negotiated otherwise + # + # Remove rsv1 from this test for deflate development + if rsv2 or rsv3 or (rsv1 and not self._compress): + raise WebSocketError( + WSCloseCode.PROTOCOL_ERROR, + "Received frame with non-zero reserved bits", + ) + + if opcode not in { + OP_CODE_CONTINUATION, + OP_CODE_TEXT, + OP_CODE_BINARY, + OP_CODE_CLOSE, + OP_CODE_PING, + OP_CODE_PONG, + }: + raise WebSocketError( + WSCloseCode.PROTOCOL_ERROR, + f"Unexpected opcode={opcode!r}", + ) + + if opcode > 0x7 and fin == 0: + raise WebSocketError( + WSCloseCode.PROTOCOL_ERROR, + "Received fragmented control frame", + ) + + has_mask = (second_byte >> 7) & 1 + length = second_byte & 0x7F + + # Control frames MUST have a payload + # length of 125 bytes or less + if opcode > 0x7 and length > 125: + raise WebSocketError( + WSCloseCode.PROTOCOL_ERROR, + "Control frame payload cannot be larger than 125 bytes", + ) + + # Set compress status if last package is FIN + # OR set compress status if this is first fragment + # Raise error if not first fragment with rsv1 = 0x1 + if self._frame_fin or self._compressed == COMPRESSED_NOT_SET: + self._compressed = COMPRESSED_TRUE if rsv1 else COMPRESSED_FALSE + elif rsv1: + raise WebSocketError( + WSCloseCode.PROTOCOL_ERROR, + "Received frame with non-zero reserved bits", + ) + + # Control frames (opcode > 0x7) may be interleaved between the + # fragments of a data message. + # https://datatracker.ietf.org/doc/html/rfc6455#section-5.4 + if opcode <= 0x7: + self._frame_fin = bool(fin) + self._frame_opcode = opcode + self._has_mask = bool(has_mask) + self._payload_len_flag = length + self._state = READ_PAYLOAD_LENGTH + + # read payload length + if self._state == READ_PAYLOAD_LENGTH: + len_flag = self._payload_len_flag + if len_flag == 126: + if data_len - start_pos < 2: + break + first_byte = data_cstr[start_pos] + second_byte = data_cstr[start_pos + 1] + start_pos += 2 + self._payload_bytes_to_read = first_byte << 8 | second_byte + elif len_flag > 126: + if data_len - start_pos < 8: + break + self._payload_bytes_to_read = UNPACK_LEN3(data, start_pos)[0] + start_pos += 8 + else: + self._payload_bytes_to_read = len_flag + + # Reject oversized data frames before buffering any payload + # bytes. Control frames are capped at 125 bytes (checked in + # READ_HEADER) so only text/binary/continuation need this. + if self._max_msg_size and self._frame_opcode in { + OP_CODE_TEXT, + OP_CODE_BINARY, + OP_CODE_CONTINUATION, + }: + projected_size = self._payload_bytes_to_read + len(self._partial) + if projected_size >= self._max_msg_size: + raise WebSocketError( + WSCloseCode.MESSAGE_TOO_BIG, + f"Message size {projected_size} " + f"exceeds limit {self._max_msg_size}", + ) + + self._state = READ_PAYLOAD_MASK if self._has_mask else READ_PAYLOAD + + # read payload mask + if self._state == READ_PAYLOAD_MASK: + if data_len - start_pos < 4: + break + self._frame_mask = data_cstr[start_pos : start_pos + 4] + start_pos += 4 + self._state = READ_PAYLOAD + + if self._state == READ_PAYLOAD: + chunk_len = data_len - start_pos + if self._payload_bytes_to_read >= chunk_len: + f_end_pos = data_len + self._payload_bytes_to_read -= chunk_len + else: + f_end_pos = start_pos + self._payload_bytes_to_read + self._payload_bytes_to_read = 0 + + had_fragments = self._frame_payload_len + self._frame_payload_len += f_end_pos - start_pos + f_start_pos = start_pos + start_pos = f_end_pos + + if self._payload_bytes_to_read != 0: + # If we don't have a complete frame, we need to save the + # data for the next call to feed_data. + self._payload_fragments.append(data_cstr[f_start_pos:f_end_pos]) + break + + payload: bytes | bytearray + if had_fragments: + # We have to join the payload fragments get the payload + self._payload_fragments.append(data_cstr[f_start_pos:f_end_pos]) + if self._has_mask: + assert self._frame_mask is not None + payload_bytearray = bytearray(b"".join(self._payload_fragments)) + websocket_mask(self._frame_mask, payload_bytearray) + payload = payload_bytearray + else: + payload = b"".join(self._payload_fragments) + self._payload_fragments.clear() + elif self._has_mask: + assert self._frame_mask is not None + payload_bytearray = data_cstr[f_start_pos:f_end_pos] # type: ignore[assignment] + if type(payload_bytearray) is not bytearray: # pragma: no branch + # Cython will do the conversion for us + # but we need to do it for Python and we + # will always get here in Python + payload_bytearray = bytearray(payload_bytearray) + websocket_mask(self._frame_mask, payload_bytearray) + payload = payload_bytearray + else: + payload = data_cstr[f_start_pos:f_end_pos] + + self._handle_frame( + self._frame_fin, self._frame_opcode, payload, self._compressed + ) + self._frame_payload_len = 0 + self._state = READ_HEADER + + # XXX: Cython needs slices to be bounded, so we can't omit the slice end here. + self._tail = data_cstr[start_pos:data_len] if start_pos < data_len else b"" diff --git a/py_modules/aiohttp/_websocket/reader_py.py b/py_modules/aiohttp/_websocket/reader_py.py new file mode 100644 index 0000000..ff5a5b3 --- /dev/null +++ b/py_modules/aiohttp/_websocket/reader_py.py @@ -0,0 +1,513 @@ +"""Reader for WebSocket protocol versions 13 and 8.""" + +import asyncio +import builtins +from collections import deque +from typing import Final + +from ..base_protocol import BaseProtocol +from ..compression_utils import ZLibDecompressor +from ..helpers import _EXC_SENTINEL, set_exception +from ..streams import EofStream +from .helpers import UNPACK_CLOSE_CODE, UNPACK_LEN3, websocket_mask +from .models import ( + WS_DEFLATE_TRAILING, + WebSocketError, + WSCloseCode, + WSMessage, + WSMessageTextBytes, + WSMsgType, +) + +ALLOWED_CLOSE_CODES: Final[set[int]] = {int(i) for i in WSCloseCode} + +# States for the reader, used to parse the WebSocket frame +# integer values are used so they can be cythonized +READ_HEADER = 1 +READ_PAYLOAD_LENGTH = 2 +READ_PAYLOAD_MASK = 3 +READ_PAYLOAD = 4 + +WS_MSG_TYPE_BINARY = WSMsgType.BINARY +WS_MSG_TYPE_TEXT = WSMsgType.TEXT + +# WSMsgType values unpacked so they can by cythonized to ints +OP_CODE_NOT_SET = -1 +OP_CODE_CONTINUATION = WSMsgType.CONTINUATION.value +OP_CODE_TEXT = WSMsgType.TEXT.value +OP_CODE_BINARY = WSMsgType.BINARY.value +OP_CODE_CLOSE = WSMsgType.CLOSE.value +OP_CODE_PING = WSMsgType.PING.value +OP_CODE_PONG = WSMsgType.PONG.value + +EMPTY_FRAME_ERROR = (True, b"") +EMPTY_FRAME = (False, b"") + +COMPRESSED_NOT_SET = -1 +COMPRESSED_FALSE = 0 +COMPRESSED_TRUE = 1 + +TUPLE_NEW = tuple.__new__ + +cython_int = int # Typed to int in Python, but cython with use a signed int in the pxd + + +class WebSocketDataQueue: + """WebSocketDataQueue resumes and pauses an underlying stream. + + It is a destination for WebSocket data. + """ + + def __init__( + self, protocol: BaseProtocol, limit: int, *, loop: asyncio.AbstractEventLoop + ) -> None: + self._size = 0 + self._protocol = protocol + self._limit = limit * 2 + self._loop = loop + self._eof = False + self._waiter: asyncio.Future[None] | None = None + self._exception: BaseException | None = None + self._buffer: deque[tuple[WSMessage | WSMessageTextBytes, int]] = deque() + self._get_buffer = self._buffer.popleft + self._put_buffer = self._buffer.append + + def is_eof(self) -> bool: + return self._eof + + def exception(self) -> BaseException | None: + return self._exception + + def set_exception( + self, + exc: BaseException, + exc_cause: builtins.BaseException = _EXC_SENTINEL, + ) -> None: + self._eof = True + self._exception = exc + if (waiter := self._waiter) is not None: + self._waiter = None + set_exception(waiter, exc, exc_cause) + + def _release_waiter(self) -> None: + if (waiter := self._waiter) is None: + return + self._waiter = None + if not waiter.done(): + waiter.set_result(None) + + def feed_eof(self) -> None: + self._eof = True + self._release_waiter() + self._exception = None # Break cyclic references + + def feed_data( + self, data: "WSMessage | WSMessageTextBytes", size: "cython_int" + ) -> None: + self._size += size + self._put_buffer((data, size)) + self._release_waiter() + if self._size > self._limit and not self._protocol._reading_paused: + self._protocol.pause_reading() + + async def read(self) -> WSMessage | WSMessageTextBytes: + if not self._buffer and not self._eof: + assert not self._waiter + self._waiter = self._loop.create_future() + try: + await self._waiter + except (asyncio.CancelledError, asyncio.TimeoutError): + self._waiter = None + raise + return self._read_from_buffer() + + def _read_from_buffer(self) -> WSMessage | WSMessageTextBytes: + if self._buffer: + data, size = self._get_buffer() + self._size -= size + if self._size < self._limit and self._protocol._reading_paused: + self._protocol.resume_reading() + return data + if self._exception is not None: + raise self._exception + raise EofStream + + +class WebSocketReader: + def __init__( + self, + queue: WebSocketDataQueue, + max_msg_size: int, + compress: bool, + decode_text: bool, + ) -> None: + self.queue = queue + self._max_msg_size = max_msg_size + self._decode_text = decode_text + + self._exc: Exception | None = None + self._partial = bytearray() + self._state = READ_HEADER + + self._opcode: int = OP_CODE_NOT_SET + self._frame_fin = False + self._frame_opcode: int = OP_CODE_NOT_SET + self._payload_fragments: list[bytes] = [] + self._frame_payload_len = 0 + + self._tail: bytes = b"" + self._has_mask = False + self._frame_mask: bytes | None = None + self._payload_bytes_to_read = 0 + self._payload_len_flag = 0 + self._compressed: int = COMPRESSED_NOT_SET + self._decompressobj: ZLibDecompressor | None = None + self._compress = compress + + def feed_eof(self) -> None: + self.queue.feed_eof() + + # data can be bytearray on Windows because proactor event loop uses bytearray + # and asyncio types this to Union[bytes, bytearray, memoryview] so we need + # coerce data to bytes if it is not + def feed_data(self, data: bytes | bytearray | memoryview) -> tuple[bool, bytes]: + if type(data) is not bytes: + data = bytes(data) + + if self._exc is not None: + return True, data + + try: + self._feed_data(data) + except Exception as exc: + self._exc = exc + set_exception(self.queue, exc) + return EMPTY_FRAME_ERROR + + return EMPTY_FRAME + + def _handle_frame( + self, + fin: bool, + opcode: int | cython_int, # Union intended: Cython pxd uses C int + payload: bytes | bytearray, + compressed: int | cython_int, # Union intended: Cython pxd uses C int + ) -> None: + msg: WSMessage + if opcode in {OP_CODE_TEXT, OP_CODE_BINARY, OP_CODE_CONTINUATION}: + # Validate continuation frames before processing + if opcode == OP_CODE_CONTINUATION and self._opcode == OP_CODE_NOT_SET: + raise WebSocketError( + WSCloseCode.PROTOCOL_ERROR, + "Continuation frame for non started message", + ) + + # load text/binary + if not fin: + # got partial frame payload + if opcode != OP_CODE_CONTINUATION: + self._opcode = opcode + self._partial += payload + return + + has_partial = bool(self._partial) + if opcode == OP_CODE_CONTINUATION: + opcode = self._opcode + self._opcode = OP_CODE_NOT_SET + # previous frame was non finished + # we should get continuation opcode + elif has_partial: + raise WebSocketError( + WSCloseCode.PROTOCOL_ERROR, + "The opcode in non-fin frame is expected " + f"to be zero, got {opcode!r}", + ) + + assembled_payload: bytes | bytearray + if has_partial: + assembled_payload = self._partial + payload + self._partial.clear() + else: + assembled_payload = payload + + # Decompress process must to be done after all packets + # received. + if compressed: + if not self._decompressobj: + self._decompressobj = ZLibDecompressor(suppress_deflate_header=True) + # XXX: It's possible that the zlib backend (isal is known to + # do this, maybe others too?) will return max_length bytes, + # but internally buffer more data such that the payload is + # >max_length, so we return one extra byte and if we're able + # to do that, then the message is too big. + payload_merged = self._decompressobj.decompress_sync( + assembled_payload + WS_DEFLATE_TRAILING, + ( + self._max_msg_size + 1 + if self._max_msg_size + else self._max_msg_size + ), + ) + if self._max_msg_size and len(payload_merged) > self._max_msg_size: + raise WebSocketError( + WSCloseCode.MESSAGE_TOO_BIG, + f"Decompressed message exceeds size limit {self._max_msg_size}", + ) + elif type(assembled_payload) is bytes: + payload_merged = assembled_payload + else: + payload_merged = bytes(assembled_payload) + + if opcode == OP_CODE_TEXT: + if self._decode_text: + try: + text = payload_merged.decode("utf-8") + except UnicodeDecodeError as exc: + raise WebSocketError( + WSCloseCode.INVALID_TEXT, "Invalid UTF-8 text message" + ) from exc + + # XXX: The Text and Binary messages here can be a performance + # bottleneck, so we use tuple.__new__ to improve performance. + # This is not type safe, but many tests should fail in + # test_client_ws_functional.py if this is wrong. + self.queue.feed_data( + TUPLE_NEW(WSMessage, (WS_MSG_TYPE_TEXT, text, "")), + len(payload_merged), + ) + else: + # Return raw bytes for TEXT messages when decode_text=False + self.queue.feed_data( + TUPLE_NEW( + WSMessageTextBytes, (WS_MSG_TYPE_TEXT, payload_merged, "") + ), + len(payload_merged), + ) + else: + self.queue.feed_data( + TUPLE_NEW(WSMessage, (WS_MSG_TYPE_BINARY, payload_merged, "")), + len(payload_merged), + ) + elif opcode == OP_CODE_CLOSE: + if len(payload) >= 2: + close_code = UNPACK_CLOSE_CODE(payload[:2])[0] + if close_code < 3000 and close_code not in ALLOWED_CLOSE_CODES: + raise WebSocketError( + WSCloseCode.PROTOCOL_ERROR, + f"Invalid close code: {close_code}", + ) + try: + close_message = payload[2:].decode("utf-8") + except UnicodeDecodeError as exc: + raise WebSocketError( + WSCloseCode.INVALID_TEXT, "Invalid UTF-8 text message" + ) from exc + msg = TUPLE_NEW(WSMessage, (WSMsgType.CLOSE, close_code, close_message)) + elif payload: + raise WebSocketError( + WSCloseCode.PROTOCOL_ERROR, + f"Invalid close frame: {fin} {opcode} {payload!r}", + ) + else: + msg = TUPLE_NEW(WSMessage, (WSMsgType.CLOSE, 0, "")) + + self.queue.feed_data(msg, 0) + elif opcode == OP_CODE_PING: + msg = TUPLE_NEW(WSMessage, (WSMsgType.PING, payload, "")) + self.queue.feed_data(msg, len(payload)) + elif opcode == OP_CODE_PONG: + msg = TUPLE_NEW(WSMessage, (WSMsgType.PONG, payload, "")) + self.queue.feed_data(msg, len(payload)) + else: + raise WebSocketError( + WSCloseCode.PROTOCOL_ERROR, f"Unexpected opcode={opcode!r}" + ) + + def _feed_data(self, data: bytes) -> None: + """Return the next frame from the socket.""" + if self._tail: + data, self._tail = self._tail + data, b"" + + start_pos: int = 0 + data_len = len(data) + data_cstr = data + + while True: + # read header + if self._state == READ_HEADER: + if data_len - start_pos < 2: + break + first_byte = data_cstr[start_pos] + second_byte = data_cstr[start_pos + 1] + start_pos += 2 + + fin = (first_byte >> 7) & 1 + rsv1 = (first_byte >> 6) & 1 + rsv2 = (first_byte >> 5) & 1 + rsv3 = (first_byte >> 4) & 1 + opcode = first_byte & 0xF + + # frame-fin = %x0 ; more frames of this message follow + # / %x1 ; final frame of this message + # frame-rsv1 = %x0 ; + # 1 bit, MUST be 0 unless negotiated otherwise + # frame-rsv2 = %x0 ; + # 1 bit, MUST be 0 unless negotiated otherwise + # frame-rsv3 = %x0 ; + # 1 bit, MUST be 0 unless negotiated otherwise + # + # Remove rsv1 from this test for deflate development + if rsv2 or rsv3 or (rsv1 and not self._compress): + raise WebSocketError( + WSCloseCode.PROTOCOL_ERROR, + "Received frame with non-zero reserved bits", + ) + + if opcode not in { + OP_CODE_CONTINUATION, + OP_CODE_TEXT, + OP_CODE_BINARY, + OP_CODE_CLOSE, + OP_CODE_PING, + OP_CODE_PONG, + }: + raise WebSocketError( + WSCloseCode.PROTOCOL_ERROR, + f"Unexpected opcode={opcode!r}", + ) + + if opcode > 0x7 and fin == 0: + raise WebSocketError( + WSCloseCode.PROTOCOL_ERROR, + "Received fragmented control frame", + ) + + has_mask = (second_byte >> 7) & 1 + length = second_byte & 0x7F + + # Control frames MUST have a payload + # length of 125 bytes or less + if opcode > 0x7 and length > 125: + raise WebSocketError( + WSCloseCode.PROTOCOL_ERROR, + "Control frame payload cannot be larger than 125 bytes", + ) + + # Set compress status if last package is FIN + # OR set compress status if this is first fragment + # Raise error if not first fragment with rsv1 = 0x1 + if self._frame_fin or self._compressed == COMPRESSED_NOT_SET: + self._compressed = COMPRESSED_TRUE if rsv1 else COMPRESSED_FALSE + elif rsv1: + raise WebSocketError( + WSCloseCode.PROTOCOL_ERROR, + "Received frame with non-zero reserved bits", + ) + + # Control frames (opcode > 0x7) may be interleaved between the + # fragments of a data message. + # https://datatracker.ietf.org/doc/html/rfc6455#section-5.4 + if opcode <= 0x7: + self._frame_fin = bool(fin) + self._frame_opcode = opcode + self._has_mask = bool(has_mask) + self._payload_len_flag = length + self._state = READ_PAYLOAD_LENGTH + + # read payload length + if self._state == READ_PAYLOAD_LENGTH: + len_flag = self._payload_len_flag + if len_flag == 126: + if data_len - start_pos < 2: + break + first_byte = data_cstr[start_pos] + second_byte = data_cstr[start_pos + 1] + start_pos += 2 + self._payload_bytes_to_read = first_byte << 8 | second_byte + elif len_flag > 126: + if data_len - start_pos < 8: + break + self._payload_bytes_to_read = UNPACK_LEN3(data, start_pos)[0] + start_pos += 8 + else: + self._payload_bytes_to_read = len_flag + + # Reject oversized data frames before buffering any payload + # bytes. Control frames are capped at 125 bytes (checked in + # READ_HEADER) so only text/binary/continuation need this. + if self._max_msg_size and self._frame_opcode in { + OP_CODE_TEXT, + OP_CODE_BINARY, + OP_CODE_CONTINUATION, + }: + projected_size = self._payload_bytes_to_read + len(self._partial) + if projected_size >= self._max_msg_size: + raise WebSocketError( + WSCloseCode.MESSAGE_TOO_BIG, + f"Message size {projected_size} " + f"exceeds limit {self._max_msg_size}", + ) + + self._state = READ_PAYLOAD_MASK if self._has_mask else READ_PAYLOAD + + # read payload mask + if self._state == READ_PAYLOAD_MASK: + if data_len - start_pos < 4: + break + self._frame_mask = data_cstr[start_pos : start_pos + 4] + start_pos += 4 + self._state = READ_PAYLOAD + + if self._state == READ_PAYLOAD: + chunk_len = data_len - start_pos + if self._payload_bytes_to_read >= chunk_len: + f_end_pos = data_len + self._payload_bytes_to_read -= chunk_len + else: + f_end_pos = start_pos + self._payload_bytes_to_read + self._payload_bytes_to_read = 0 + + had_fragments = self._frame_payload_len + self._frame_payload_len += f_end_pos - start_pos + f_start_pos = start_pos + start_pos = f_end_pos + + if self._payload_bytes_to_read != 0: + # If we don't have a complete frame, we need to save the + # data for the next call to feed_data. + self._payload_fragments.append(data_cstr[f_start_pos:f_end_pos]) + break + + payload: bytes | bytearray + if had_fragments: + # We have to join the payload fragments get the payload + self._payload_fragments.append(data_cstr[f_start_pos:f_end_pos]) + if self._has_mask: + assert self._frame_mask is not None + payload_bytearray = bytearray(b"".join(self._payload_fragments)) + websocket_mask(self._frame_mask, payload_bytearray) + payload = payload_bytearray + else: + payload = b"".join(self._payload_fragments) + self._payload_fragments.clear() + elif self._has_mask: + assert self._frame_mask is not None + payload_bytearray = data_cstr[f_start_pos:f_end_pos] # type: ignore[assignment] + if type(payload_bytearray) is not bytearray: # pragma: no branch + # Cython will do the conversion for us + # but we need to do it for Python and we + # will always get here in Python + payload_bytearray = bytearray(payload_bytearray) + websocket_mask(self._frame_mask, payload_bytearray) + payload = payload_bytearray + else: + payload = data_cstr[f_start_pos:f_end_pos] + + self._handle_frame( + self._frame_fin, self._frame_opcode, payload, self._compressed + ) + self._frame_payload_len = 0 + self._state = READ_HEADER + + # XXX: Cython needs slices to be bounded, so we can't omit the slice end here. + self._tail = data_cstr[start_pos:data_len] if start_pos < data_len else b"" diff --git a/py_modules/aiohttp/_websocket/writer.py b/py_modules/aiohttp/_websocket/writer.py new file mode 100644 index 0000000..7daf4fc --- /dev/null +++ b/py_modules/aiohttp/_websocket/writer.py @@ -0,0 +1,261 @@ +"""WebSocket protocol versions 13 and 8.""" + +import asyncio +import random +import sys +from functools import partial +from typing import Final, Optional, Set + +from ..base_protocol import BaseProtocol +from ..client_exceptions import ClientConnectionResetError +from ..compression_utils import ZLibBackend, ZLibCompressor +from ..helpers import DEFAULT_CHUNK_SIZE +from .helpers import ( + MASK_LEN, + MSG_SIZE, + PACK_CLOSE_CODE, + PACK_LEN1, + PACK_LEN2, + PACK_LEN3, + PACK_RANDBITS, + websocket_mask, +) +from .models import WS_DEFLATE_TRAILING, WSMsgType + +# WebSocket opcode boundary: opcodes 0-7 are data frames, 8-15 are control frames +# Control frames (ping, pong, close) are never compressed +WS_CONTROL_FRAME_OPCODE: Final[int] = 8 + +# For websockets, keeping latency low is extremely important as implementations +# generally expect to be able to send and receive messages quickly. We use a +# larger chunk size to reduce the number of executor calls and avoid task +# creation overhead, since both are significant sources of latency when chunks +# are small. A size of 16KiB was chosen as a balance between avoiding task +# overhead and not blocking the event loop too long with synchronous compression. + +WEBSOCKET_MAX_SYNC_CHUNK_SIZE = 16 * 1024 + + +class WebSocketWriter: + """WebSocket writer. + + The writer is responsible for sending messages to the client. It is + created by the protocol when a connection is established. The writer + should avoid implementing any application logic and should only be + concerned with the low-level details of the WebSocket protocol. + """ + + def __init__( + self, + protocol: BaseProtocol, + transport: asyncio.Transport, + *, + use_mask: bool = False, + limit: int = DEFAULT_CHUNK_SIZE, + random: random.Random = random.Random(), + compress: int = 0, + notakeover: bool = False, + ) -> None: + """Initialize a WebSocket writer.""" + self.protocol = protocol + self.transport = transport + self.use_mask = use_mask + self.get_random_bits = partial(random.getrandbits, 32) + self.compress = compress + self.notakeover = notakeover + self._closing = False + self._limit = limit + self._output_size = 0 + self._compressobj: Optional[ZLibCompressor] = None + self._send_lock = asyncio.Lock() + self._background_tasks: Set[asyncio.Task[None]] = set() + + async def send_frame( + self, message: bytes, opcode: int, compress: int | None = None + ) -> None: + """Send a frame over the websocket with message as its payload.""" + if self._closing and not (opcode & WSMsgType.CLOSE): + raise ClientConnectionResetError("Cannot write to closing transport") + + if not (compress or self.compress) or opcode >= WS_CONTROL_FRAME_OPCODE: + # Non-compressed frames don't need lock or shield + self._write_websocket_frame(message, opcode, 0) + elif len(message) <= WEBSOCKET_MAX_SYNC_CHUNK_SIZE: + # Small compressed payloads - compress synchronously in event loop + # We need the lock even though sync compression has no await points. + # This prevents small frames from interleaving with large frames that + # compress in the executor, avoiding compressor state corruption. + async with self._send_lock: + self._send_compressed_frame_sync(message, opcode, compress) + else: + # Large compressed frames need shield to prevent corruption + # For large compressed frames, the entire compress+send + # operation must be atomic. If cancelled after compression but + # before send, the compressor state would be advanced but data + # not sent, corrupting subsequent frames. + # Create a task to shield from cancellation + # The lock is acquired inside the shielded task so the entire + # operation (lock + compress + send) completes atomically. + # Use eager_start on Python 3.12+ to avoid scheduling overhead + loop = asyncio.get_running_loop() + coro = self._send_compressed_frame_async_locked(message, opcode, compress) + if sys.version_info >= (3, 12): + send_task = asyncio.Task(coro, loop=loop, eager_start=True) + else: + send_task = loop.create_task(coro) + # Keep a strong reference to prevent garbage collection + self._background_tasks.add(send_task) + send_task.add_done_callback(self._background_tasks.discard) + await asyncio.shield(send_task) + + # It is safe to return control to the event loop when using compression + # after this point as we have already sent or buffered all the data. + # Once we have written output_size up to the limit, we call the + # drain helper which waits for the transport to be ready to accept + # more data. This is a flow control mechanism to prevent the buffer + # from growing too large. The drain helper will return right away + # if the writer is not paused. + if self._output_size > self._limit: + self._output_size = 0 + if self.protocol._paused: + await self.protocol._drain_helper() + + def _write_websocket_frame(self, message: bytes, opcode: int, rsv: int) -> None: + """ + Write a websocket frame to the transport. + + This method handles frame header construction, masking, and writing to transport. + It does not handle compression or flow control - those are the responsibility + of the caller. + """ + msg_length = len(message) + + use_mask = self.use_mask + mask_bit = 0x80 if use_mask else 0 + + # Depending on the message length, the header is assembled differently. + # The first byte is reserved for the opcode and the RSV bits. + first_byte = 0x80 | rsv | opcode + if msg_length < 126: + header = PACK_LEN1(first_byte, msg_length | mask_bit) + header_len = 2 + elif msg_length < 65536: + header = PACK_LEN2(first_byte, 126 | mask_bit, msg_length) + header_len = 4 + else: + header = PACK_LEN3(first_byte, 127 | mask_bit, msg_length) + header_len = 10 + + if self.transport.is_closing(): + raise ClientConnectionResetError("Cannot write to closing transport") + + # https://datatracker.ietf.org/doc/html/rfc6455#section-5.3 + # If we are using a mask, we need to generate it randomly + # and apply it to the message before sending it. A mask is + # a 32-bit value that is applied to the message using a + # bitwise XOR operation. It is used to prevent certain types + # of attacks on the websocket protocol. The mask is only used + # when aiohttp is acting as a client. Servers do not use a mask. + if use_mask: + mask = PACK_RANDBITS(self.get_random_bits()) + message_arr = bytearray(message) + websocket_mask(mask, message_arr) + self.transport.write(header + mask + message_arr) + self._output_size += MASK_LEN + elif msg_length > MSG_SIZE: + self.transport.write(header) + self.transport.write(message) + else: + self.transport.write(header + message) + + self._output_size += header_len + msg_length + + def _get_compressor(self, compress: int | None) -> ZLibCompressor: + """Get or create a compressor object for the given compression level.""" + if compress: + # Do not set self._compress if compressing is for this frame + return ZLibCompressor( + level=ZLibBackend.Z_BEST_SPEED, + wbits=-compress, + max_sync_chunk_size=WEBSOCKET_MAX_SYNC_CHUNK_SIZE, + ) + if not self._compressobj: + self._compressobj = ZLibCompressor( + level=ZLibBackend.Z_BEST_SPEED, + wbits=-self.compress, + max_sync_chunk_size=WEBSOCKET_MAX_SYNC_CHUNK_SIZE, + ) + return self._compressobj + + def _send_compressed_frame_sync( + self, message: bytes, opcode: int, compress: int | None + ) -> None: + """ + Synchronous send for small compressed frames. + + This is used for small compressed payloads that compress synchronously in the event loop. + Since there are no await points, this is inherently cancellation-safe. + """ + # RSV are the reserved bits in the frame header. They are used to + # indicate that the frame is using an extension. + # https://datatracker.ietf.org/doc/html/rfc6455#section-5.2 + compressobj = self._get_compressor(compress) + # (0x40) RSV1 is set for compressed frames + # https://datatracker.ietf.org/doc/html/rfc7692#section-7.2.3.1 + self._write_websocket_frame( + ( + compressobj.compress_sync(message) + + compressobj.flush( + ZLibBackend.Z_FULL_FLUSH + if self.notakeover + else ZLibBackend.Z_SYNC_FLUSH + ) + ).removesuffix(WS_DEFLATE_TRAILING), + opcode, + 0x40, + ) + + async def _send_compressed_frame_async_locked( + self, message: bytes, opcode: int, compress: int | None + ) -> None: + """ + Async send for large compressed frames with lock. + + Acquires the lock and compresses large payloads asynchronously in + the executor. The lock is held for the entire operation to ensure + the compressor state is not corrupted by concurrent sends. + + MUST be run shielded from cancellation. If cancelled after + compression but before sending, the compressor state would be + advanced but data not sent, corrupting subsequent frames. + """ + async with self._send_lock: + # RSV are the reserved bits in the frame header. They are used to + # indicate that the frame is using an extension. + # https://datatracker.ietf.org/doc/html/rfc6455#section-5.2 + compressobj = self._get_compressor(compress) + # (0x40) RSV1 is set for compressed frames + # https://datatracker.ietf.org/doc/html/rfc7692#section-7.2.3.1 + self._write_websocket_frame( + ( + await compressobj.compress(message) + + compressobj.flush( + ZLibBackend.Z_FULL_FLUSH + if self.notakeover + else ZLibBackend.Z_SYNC_FLUSH + ) + ).removesuffix(WS_DEFLATE_TRAILING), + opcode, + 0x40, + ) + + async def close(self, code: int = 1000, message: bytes | str = b"") -> None: + """Close the websocket, sending the specified code and message.""" + if isinstance(message, str): + message = message.encode("utf-8") + try: + await self.send_frame( + PACK_CLOSE_CODE(code) + message, opcode=WSMsgType.CLOSE + ) + finally: + self._closing = True diff --git a/py_modules/aiohttp/abc.py b/py_modules/aiohttp/abc.py new file mode 100644 index 0000000..b672873 --- /dev/null +++ b/py_modules/aiohttp/abc.py @@ -0,0 +1,270 @@ +import asyncio +import logging +import socket +from abc import ABC, abstractmethod +from collections.abc import Awaitable, Callable, Generator, Iterable, Sequence, Sized +from http.cookies import BaseCookie, Morsel, SimpleCookie +from types import MappingProxyType +from typing import TYPE_CHECKING, Any, TypedDict + +from multidict import CIMultiDict +from yarl import URL + +from ._cookie_helpers import parse_set_cookie_headers +from .typedefs import LooseCookies + +if TYPE_CHECKING: + from .web_app import Application + from .web_exceptions import HTTPException + from .web_request import BaseRequest, Request + from .web_response import StreamResponse +else: + BaseRequest = Request = Application = StreamResponse = Any + HTTPException = Any + + +class AbstractRouter(ABC): + def __init__(self) -> None: + self._frozen = False + + def post_init(self, app: Application) -> None: + """Post init stage. + + Not an abstract method for sake of backward compatibility, + but if the router wants to be aware of the application + it can override this. + """ + + @property + def frozen(self) -> bool: + return self._frozen + + def freeze(self) -> None: + """Freeze router.""" + self._frozen = True + + @abstractmethod + async def resolve(self, request: Request) -> "AbstractMatchInfo": + """Return MATCH_INFO for given request""" + + +class AbstractMatchInfo(ABC): + + __slots__ = () + + @property # pragma: no branch + @abstractmethod + def handler(self) -> Callable[[Request], Awaitable[StreamResponse]]: + """Execute matched request handler""" + + @property + @abstractmethod + def expect_handler( + self, + ) -> Callable[[Request], Awaitable[StreamResponse | None]]: + """Expect handler for 100-continue processing""" + + @property # pragma: no branch + @abstractmethod + def http_exception(self) -> HTTPException | None: + """HTTPException instance raised on router's resolving, or None""" + + @abstractmethod # pragma: no branch + def get_info(self) -> dict[str, Any]: + """Return a dict with additional info useful for introspection""" + + @property # pragma: no branch + @abstractmethod + def apps(self) -> tuple[Application, ...]: + """Stack of nested applications. + + Top level application is left-most element. + + """ + + @abstractmethod + def add_app(self, app: Application) -> None: + """Add application to the nested apps stack.""" + + @abstractmethod + def freeze(self) -> None: + """Freeze the match info. + + The method is called after route resolution. + + After the call .add_app() is forbidden. + + """ + + +class AbstractView(ABC): + """Abstract class based view.""" + + def __init__(self, request: Request) -> None: + self._request = request + + @property + def request(self) -> Request: + """Request instance.""" + return self._request + + @abstractmethod + def __await__(self) -> Generator[None, None, StreamResponse]: + """Execute the view handler.""" + + +class ResolveResult(TypedDict): + """Resolve result. + + This is the result returned from an AbstractResolver's + resolve method. + + :param hostname: The hostname that was provided. + :param host: The IP address that was resolved. + :param port: The port that was resolved. + :param family: The address family that was resolved. + :param proto: The protocol that was resolved. + :param flags: The flags that were resolved. + """ + + hostname: str + host: str + port: int + family: int + proto: int + flags: int + + +class AbstractResolver(ABC): + """Abstract DNS resolver.""" + + @abstractmethod + async def resolve( + self, host: str, port: int = 0, family: socket.AddressFamily = socket.AF_INET + ) -> list[ResolveResult]: + """Return IP address for given hostname""" + + @abstractmethod + async def close(self) -> None: + """Release resolver""" + + +if TYPE_CHECKING: + IterableBase = Iterable[Morsel[str]] +else: + IterableBase = Iterable + + +ClearCookiePredicate = Callable[["Morsel[str]"], bool] + + +class AbstractCookieJar(Sized, IterableBase): + """Abstract Cookie Jar.""" + + def __init__(self, *, loop: asyncio.AbstractEventLoop | None = None) -> None: + self._loop = loop or asyncio.get_running_loop() + + @property + @abstractmethod + def unsafe(self) -> bool: + """Return True if cookies can be used with IP addresses.""" + + @property + @abstractmethod + def quote_cookie(self) -> bool: + """Return True if cookies should be quoted.""" + + @property + @abstractmethod + def cookies(self) -> MappingProxyType[tuple[str, str], SimpleCookie]: + """Return the cookies stored in this jar.""" + + @property + @abstractmethod + def host_only_cookies(self) -> frozenset[tuple[str, str]]: + """Return the host-only cookies stored in this jar.""" + + @abstractmethod + def clear(self, predicate: ClearCookiePredicate | None = None) -> None: + """Clear all cookies if no predicate is passed.""" + + @abstractmethod + def clear_domain(self, domain: str) -> None: + """Clear all cookies for domain and all subdomains.""" + + @abstractmethod + def update_cookies(self, cookies: LooseCookies, response_url: URL = URL()) -> None: + """Update cookies.""" + + def update_cookies_from_headers( + self, headers: Sequence[str], response_url: URL + ) -> None: + """Update cookies from raw Set-Cookie headers.""" + if headers and (cookies_to_update := parse_set_cookie_headers(headers)): + self.update_cookies(cookies_to_update, response_url) + + @abstractmethod + def filter_cookies(self, request_url: URL) -> "BaseCookie[str]": + """Return the jar's cookies filtered by their attributes.""" + + +class AbstractStreamWriter(ABC): + """Abstract stream writer.""" + + buffer_size: int = 0 + output_size: int = 0 + length: int | None = 0 + + @abstractmethod + async def write(self, chunk: bytes | bytearray | memoryview) -> None: + """Write chunk into stream.""" + + @abstractmethod + async def write_eof(self, chunk: bytes = b"") -> None: + """Write last chunk.""" + + @abstractmethod + async def drain(self) -> None: + """Flush the write buffer.""" + + @abstractmethod + def enable_compression( + self, encoding: str = "deflate", strategy: int | None = None + ) -> None: + """Enable HTTP body compression""" + + @abstractmethod + def enable_chunking(self) -> None: + """Enable HTTP chunked mode""" + + @abstractmethod + async def write_headers( + self, status_line: str, headers: "CIMultiDict[str]" + ) -> None: + """Write HTTP headers""" + + def send_headers(self) -> None: + """Force sending buffered headers if not already sent. + + Required only if write_headers() buffers headers instead of sending immediately. + For backwards compatibility, this method does nothing by default. + """ + + +class AbstractAccessLogger(ABC): + """Abstract writer to access log.""" + + __slots__ = ("logger", "log_format") + + def __init__(self, logger: logging.Logger, log_format: str) -> None: + self.logger = logger + self.log_format = log_format + + @abstractmethod + def log(self, request: BaseRequest, response: StreamResponse, time: float) -> None: + """Emit log to logger.""" + + @property + def enabled(self) -> bool: + """Check if logger is enabled.""" + return True diff --git a/py_modules/aiohttp/base_protocol.py b/py_modules/aiohttp/base_protocol.py new file mode 100644 index 0000000..df3f8c0 --- /dev/null +++ b/py_modules/aiohttp/base_protocol.py @@ -0,0 +1,140 @@ +import asyncio +from typing import TYPE_CHECKING, Any, cast + +from .client_exceptions import ClientConnectionResetError +from .helpers import set_exception +from .tcp_helpers import tcp_nodelay + +if TYPE_CHECKING: + from .http_parser import HttpParser + +# Raised by transport.pause_reading()/resume_reading() when the transport +# does not support flow control; safe to ignore. +# NOTE: Catch these with a plain try/except/pass, never contextlib.suppress(): +# pause/resume run on the hot read path and suppress() is ~6x slower than +# try/except here (it builds a context manager and unpacks this tuple per call). +PAUSE_RESUME_READING_ERRORS = (AttributeError, NotImplementedError, RuntimeError) + + +class BaseProtocol(asyncio.Protocol): + __slots__ = ( + "_loop", + "_paused", + "_parser", + "_drain_waiter", + "_connection_lost", + "_reading_paused", + "_upgraded", + "transport", + ) + + def __init__( + self, loop: asyncio.AbstractEventLoop, parser: "HttpParser[Any] | None" = None + ) -> None: + self._loop: asyncio.AbstractEventLoop = loop + self._paused = False + self._drain_waiter: asyncio.Future[None] | None = None + self._reading_paused = False + self._parser = parser + self._upgraded = False + + self.transport: asyncio.Transport | None = None + + @property + def connected(self) -> bool: + """Return True if the connection is open.""" + return self.transport is not None + + @property + def writing_paused(self) -> bool: + return self._paused + + def pause_writing(self) -> None: + assert not self._paused + self._paused = True + + def resume_writing(self) -> None: + assert self._paused + self._paused = False + + waiter = self._drain_waiter + if waiter is not None: + self._drain_waiter = None + if not waiter.done(): + waiter.set_result(None) + + def pause_reading(self) -> None: + self._reading_paused = True + # Parser shouldn't be paused on websockets. + if not self._upgraded: + assert self._parser is not None + self._parser.pause_reading() + if self.transport is not None: + try: + self.transport.pause_reading() + except PAUSE_RESUME_READING_ERRORS: + # Transport lacks flow control; nothing to pause. Intentionally + # ignored (see PAUSE_RESUME_READING_ERRORS; do not use suppress). + pass + + def _reading_paused_for_msg_queue(self) -> bool: + """Keep the transport paused for protocol-specific reasons (overridden).""" + return False + + def resume_reading(self, resume_parser: bool = True) -> None: + self._reading_paused = False + + # This will resume parsing any unprocessed data from the last pause. + if not self._upgraded and resume_parser: + self.data_received(b"") + + # Reading may have been paused again in the above call if there was a lot of + # compressed data still pending. + if ( + not self._reading_paused + and not self._reading_paused_for_msg_queue() + and self.transport is not None + ): + try: + self.transport.resume_reading() + except PAUSE_RESUME_READING_ERRORS: + # Transport lacks flow control; nothing to resume. Intentionally + # ignored (see PAUSE_RESUME_READING_ERRORS; do not use suppress). + pass + self._reading_paused = False + + def connection_made(self, transport: asyncio.BaseTransport) -> None: + tr = cast(asyncio.Transport, transport) + tcp_nodelay(tr, True) + self.transport = tr + + def connection_lost(self, exc: BaseException | None) -> None: + # Wake up the writer if currently paused. + self.transport = None + if not self._paused: + return + waiter = self._drain_waiter + if waiter is None: + return + self._drain_waiter = None + if waiter.done(): + return + if exc is None: + waiter.set_result(None) + else: + set_exception( + waiter, + ConnectionError("Connection lost"), + exc, + ) + + async def _drain_helper(self) -> None: + if self.transport is None: + raise ClientConnectionResetError("Connection lost") + if not self._paused: + return + waiter = self._drain_waiter + if waiter is None: + waiter = self._loop.create_future() + self._drain_waiter = waiter + await waiter diff --git a/py_modules/aiohttp/client.py b/py_modules/aiohttp/client.py new file mode 100644 index 0000000..12f236d --- /dev/null +++ b/py_modules/aiohttp/client.py @@ -0,0 +1,1818 @@ +"""HTTP Client for asyncio.""" + +import asyncio +import base64 +import hashlib +import json +import os +import sys +import traceback +import warnings +from collections.abc import ( + Awaitable, + Callable, + Coroutine, + Generator, + Iterable, + Sequence, +) +from contextlib import suppress +from types import TracebackType +from typing import ( + TYPE_CHECKING, + Any, + Final, + Generic, + Literal, + TypedDict, + TypeVar, + overload, +) + +import attr +from multidict import CIMultiDict, MultiDict, MultiDictProxy, istr +from yarl import URL + +from . import hdrs, http, payload +from ._websocket.reader import WebSocketDataQueue +from .abc import AbstractCookieJar +from .client_exceptions import ( + ClientConnectionError, + ClientConnectionResetError, + ClientConnectorCertificateError, + ClientConnectorDNSError, + ClientConnectorError, + ClientConnectorSSLError, + ClientError, + ClientHttpProxyError, + ClientOSError, + ClientPayloadError, + ClientProxyConnectionError, + ClientResponseError, + ClientSSLError, + ConnectionTimeoutError, + ContentTypeError, + InvalidURL, + InvalidUrlClientError, + InvalidUrlRedirectClientError, + NonHttpUrlClientError, + NonHttpUrlRedirectClientError, + RedirectClientError, + ServerConnectionError, + ServerDisconnectedError, + ServerFingerprintMismatch, + ServerTimeoutError, + SocketTimeoutError, + TooManyRedirects, + WSMessageTypeError, + WSServerHandshakeError, +) +from .client_middlewares import ClientMiddlewareType, build_client_middlewares +from .client_reqrep import ( + ClientRequest as ClientRequest, + ClientResponse as ClientResponse, + Fingerprint as Fingerprint, + RequestInfo as RequestInfo, + _merge_ssl_params, +) +from .client_ws import ( + DEFAULT_WS_CLIENT_TIMEOUT, + ClientWebSocketResponse as ClientWebSocketResponse, + ClientWSTimeout as ClientWSTimeout, +) +from .connector import ( + HTTP_AND_EMPTY_SCHEMA_SET, + BaseConnector as BaseConnector, + NamedPipeConnector as NamedPipeConnector, + TCPConnector as TCPConnector, + UnixConnector as UnixConnector, +) +from .cookiejar import CookieJar +from .helpers import ( + _SENTINEL, + DEBUG, + DEFAULT_CHUNK_SIZE, + EMPTY_BODY_METHODS, + BasicAuth, + TimeoutHandle, + basicauth_from_netrc, + get_env_proxy_for_url, + netrc_from_env, + sentinel, + strip_auth_from_url, +) +from .http import WS_KEY, HttpVersion, WebSocketReader, WebSocketWriter +from .http_websocket import WSHandshakeError, ws_ext_gen, ws_ext_parse +from .tracing import Trace, TraceConfig +from .typedefs import ( + JSONBytesEncoder, + JSONEncoder, + LooseCookies, + LooseHeaders, + Query, + StrOrURL, +) + +__all__ = ( + # client_exceptions + "ClientConnectionError", + "ClientConnectionResetError", + "ClientConnectorCertificateError", + "ClientConnectorDNSError", + "ClientConnectorError", + "ClientConnectorSSLError", + "ClientError", + "ClientHttpProxyError", + "ClientOSError", + "ClientPayloadError", + "ClientProxyConnectionError", + "ClientResponseError", + "ClientSSLError", + "ConnectionTimeoutError", + "ContentTypeError", + "InvalidURL", + "InvalidUrlClientError", + "RedirectClientError", + "NonHttpUrlClientError", + "InvalidUrlRedirectClientError", + "NonHttpUrlRedirectClientError", + "ServerConnectionError", + "ServerDisconnectedError", + "ServerFingerprintMismatch", + "ServerTimeoutError", + "SocketTimeoutError", + "TooManyRedirects", + "WSServerHandshakeError", + # client_reqrep + "ClientRequest", + "ClientResponse", + "Fingerprint", + "RequestInfo", + # connector + "BaseConnector", + "TCPConnector", + "UnixConnector", + "NamedPipeConnector", + # client_ws + "ClientWebSocketResponse", + # client + "ClientSession", + "ClientTimeout", + "ClientWSTimeout", + "request", + "WSMessageTypeError", +) + + +if TYPE_CHECKING: + from ssl import SSLContext +else: + SSLContext = Any + +if sys.version_info >= (3, 11) and TYPE_CHECKING: + from typing import Unpack + + +class _RequestOptions(TypedDict, total=False): + params: Query + data: Any + json: Any + cookies: LooseCookies | None + headers: LooseHeaders | None + skip_auto_headers: Iterable[str] | None + auth: BasicAuth | None + allow_redirects: bool + max_redirects: int + compress: str | bool | None + chunked: bool | None + expect100: bool + raise_for_status: None | bool | Callable[[ClientResponse], Awaitable[None]] + read_until_eof: bool + proxy: StrOrURL | None + proxy_auth: BasicAuth | None + timeout: "ClientTimeout | _SENTINEL | None" + ssl: SSLContext | bool | Fingerprint + server_hostname: str | None + proxy_headers: LooseHeaders | None + trace_request_ctx: object + read_bufsize: int | None + auto_decompress: bool | None + max_line_size: int | None + max_field_size: int | None + max_headers: int | None + middlewares: Sequence[ClientMiddlewareType] | None + + +class _WSConnectOptions(TypedDict, total=False): + method: str + protocols: Iterable[str] + timeout: "ClientWSTimeout | _SENTINEL" + receive_timeout: float | None + autoclose: bool + autoping: bool + heartbeat: float | None + auth: BasicAuth | None + origin: str | None + params: Query + headers: LooseHeaders | None + proxy: StrOrURL | None + proxy_auth: BasicAuth | None + ssl: SSLContext | bool | Fingerprint + verify_ssl: bool | None + fingerprint: bytes | None + ssl_context: SSLContext | None + server_hostname: str | None + proxy_headers: LooseHeaders | None + compress: int + max_msg_size: int + + +@attr.s(auto_attribs=True, frozen=True, slots=True) +class ClientTimeout: + total: float | None = None + connect: float | None = None + sock_read: float | None = None + sock_connect: float | None = None + ceil_threshold: float = 5 + + # pool_queue_timeout: Optional[float] = None + # dns_resolution_timeout: Optional[float] = None + # socket_connect_timeout: Optional[float] = None + # connection_acquiring_timeout: Optional[float] = None + # new_connection_timeout: Optional[float] = None + # http_header_timeout: Optional[float] = None + # response_body_timeout: Optional[float] = None + + # to create a timeout specific for a single request, either + # - create a completely new one to overwrite the default + # - or use http://www.attrs.org/en/stable/api.html#attr.evolve + # to overwrite the defaults + + +# 5 Minute default read timeout +DEFAULT_TIMEOUT: Final[ClientTimeout] = ClientTimeout(total=5 * 60, sock_connect=30) + +# https://www.rfc-editor.org/rfc/rfc9110#section-9.2.2 +IDEMPOTENT_METHODS = frozenset({"GET", "HEAD", "OPTIONS", "TRACE", "PUT", "DELETE"}) + +_RetType_co = TypeVar( + "_RetType_co", + bound="ClientResponse | ClientWebSocketResponse[bool]", + covariant=True, +) +_CharsetResolver = Callable[[ClientResponse, bytes], str] + + +class ClientSession: + """First-class interface for making HTTP requests.""" + + ATTRS = frozenset( + [ + "_base_url", + "_base_url_origin", + "_source_traceback", + "_connector", + "_loop", + "_cookie_jar", + "_connector_owner", + "_default_auth", + "_version", + "_json_serialize", + "_json_serialize_bytes", + "_requote_redirect_url", + "_timeout", + "_raise_for_status", + "_auto_decompress", + "_trust_env", + "_default_headers", + "_skip_auto_headers", + "_request_class", + "_response_class", + "_ws_response_class", + "_trace_configs", + "_read_bufsize", + "_max_line_size", + "_max_field_size", + "_max_headers", + "_resolve_charset", + "_default_proxy", + "_default_proxy_auth", + "_retry_connection", + "_middlewares", + "requote_redirect_url", + ] + ) + + _source_traceback: traceback.StackSummary | None = None + _connector: BaseConnector | None = None + + def __init__( + self, + base_url: StrOrURL | None = None, + *, + connector: BaseConnector | None = None, + loop: asyncio.AbstractEventLoop | None = None, + cookies: LooseCookies | None = None, + headers: LooseHeaders | None = None, + proxy: StrOrURL | None = None, + proxy_auth: BasicAuth | None = None, + skip_auto_headers: Iterable[str] | None = None, + auth: BasicAuth | None = None, + json_serialize: JSONEncoder = json.dumps, + json_serialize_bytes: JSONBytesEncoder | None = None, + request_class: type[ClientRequest] = ClientRequest, + response_class: type[ClientResponse] = ClientResponse, + ws_response_class: type[ClientWebSocketResponse] = ClientWebSocketResponse, + version: HttpVersion = http.HttpVersion11, + cookie_jar: AbstractCookieJar | None = None, + connector_owner: bool = True, + raise_for_status: bool | Callable[[ClientResponse], Awaitable[None]] = False, + read_timeout: float | _SENTINEL = sentinel, + conn_timeout: float | None = None, + timeout: object | ClientTimeout = sentinel, + auto_decompress: bool = True, + trust_env: bool = False, + requote_redirect_url: bool = True, + trace_configs: list[TraceConfig] | None = None, + read_bufsize: int = DEFAULT_CHUNK_SIZE, + max_line_size: int = 8190, + max_field_size: int = 8190, + max_headers: int = 128, + fallback_charset_resolver: _CharsetResolver = lambda r, b: "utf-8", + middlewares: Sequence[ClientMiddlewareType] = (), + ssl_shutdown_timeout: _SENTINEL | None | float = sentinel, + ) -> None: + # We initialise _connector to None immediately, as it's referenced in __del__() + # and could cause issues if an exception occurs during initialisation. + self._connector: BaseConnector | None = None + + if loop is None: + if connector is not None: + loop = connector._loop + + loop = loop or asyncio.get_running_loop() + + if base_url is None or isinstance(base_url, URL): + self._base_url: URL | None = base_url + self._base_url_origin = None if base_url is None else base_url.origin() + else: + self._base_url = URL(base_url) + self._base_url_origin = self._base_url.origin() + assert self._base_url.absolute, "Only absolute URLs are supported" + if self._base_url is not None and not self._base_url.path.endswith("/"): + raise ValueError("base_url must have a trailing '/'") + + if timeout is sentinel or timeout is None: + self._timeout = DEFAULT_TIMEOUT + if read_timeout is not sentinel: + warnings.warn( + "read_timeout is deprecated, use timeout argument instead", + DeprecationWarning, + stacklevel=2, + ) + self._timeout = attr.evolve(self._timeout, total=read_timeout) + if conn_timeout is not None: + self._timeout = attr.evolve(self._timeout, connect=conn_timeout) + warnings.warn( + "conn_timeout is deprecated, use timeout argument instead", + DeprecationWarning, + stacklevel=2, + ) + else: + if not isinstance(timeout, ClientTimeout): + raise ValueError( + f"timeout parameter cannot be of {type(timeout)} type, " + "please use 'timeout=ClientTimeout(...)'", + ) + self._timeout = timeout + if read_timeout is not sentinel: + raise ValueError( + "read_timeout and timeout parameters " + "conflict, please setup " + "timeout.read" + ) + if conn_timeout is not None: + raise ValueError( + "conn_timeout and timeout parameters " + "conflict, please setup " + "timeout.connect" + ) + + if ssl_shutdown_timeout is not sentinel: + warnings.warn( + "The ssl_shutdown_timeout parameter is deprecated and will be removed in aiohttp 4.0", + DeprecationWarning, + stacklevel=2, + ) + + if connector is None: + connector = TCPConnector( + loop=loop, ssl_shutdown_timeout=ssl_shutdown_timeout + ) + + if connector._loop is not loop: + raise RuntimeError("Session and connector has to use same event loop") + + self._loop = loop + + if loop.get_debug(): + self._source_traceback = traceback.extract_stack(sys._getframe(1)) + + if cookie_jar is None: + cookie_jar = CookieJar(loop=loop) + self._cookie_jar = cookie_jar + + if cookies: + self._cookie_jar.update_cookies(cookies) + + if auth is not None: + warnings.warn( + "The 'auth' parameter is deprecated and will be removed in v4;" + " pass headers={'Authorization': " + "aiohttp.encode_basic_auth(login, password)} instead", + DeprecationWarning, + stacklevel=2, + ) + if proxy_auth is not None: + warnings.warn( + "The 'proxy_auth' parameter is deprecated and will be removed in v4;" + " pass proxy_headers={'Proxy-Authorization': " + "aiohttp.encode_basic_auth(login, password)} instead", + DeprecationWarning, + stacklevel=2, + ) + self._connector = connector + self._connector_owner = connector_owner + self._default_auth = auth + self._version = version + self._json_serialize = json_serialize + self._json_serialize_bytes = json_serialize_bytes + self._raise_for_status = raise_for_status + self._auto_decompress = auto_decompress + self._trust_env = trust_env + self._requote_redirect_url = requote_redirect_url + self._read_bufsize = read_bufsize + self._max_line_size = max_line_size + self._max_field_size = max_field_size + self._max_headers = max_headers + + # Convert to list of tuples + if headers: + real_headers: CIMultiDict[str] = CIMultiDict(headers) + else: + real_headers = CIMultiDict() + self._default_headers: CIMultiDict[str] = real_headers + if skip_auto_headers is not None: + self._skip_auto_headers = frozenset(istr(i) for i in skip_auto_headers) + else: + self._skip_auto_headers = frozenset() + + self._request_class = request_class + self._response_class = response_class + self._ws_response_class = ws_response_class + + self._trace_configs = trace_configs or [] + for trace_config in self._trace_configs: + trace_config.freeze() + + self._resolve_charset = fallback_charset_resolver + + self._default_proxy = proxy + self._default_proxy_auth = proxy_auth + self._retry_connection: bool = True + self._middlewares = middlewares + + def __init_subclass__(cls: type["ClientSession"]) -> None: + warnings.warn( + f"Inheritance class {cls.__name__} from ClientSession is discouraged", + DeprecationWarning, + stacklevel=2, + ) + + if DEBUG: + + def __setattr__(self, name: str, val: Any) -> None: + if name not in self.ATTRS: + warnings.warn( + f"Setting custom ClientSession.{name} attribute is discouraged", + DeprecationWarning, + stacklevel=2, + ) + super().__setattr__(name, val) + + def __del__(self, _warnings: Any = warnings) -> None: + if not self.closed: + kwargs = {"source": self} + _warnings.warn( + f"Unclosed client session {self!r}", ResourceWarning, **kwargs + ) + context = {"client_session": self, "message": "Unclosed client session"} + if self._source_traceback is not None: + context["source_traceback"] = self._source_traceback + self._loop.call_exception_handler(context) + + if sys.version_info >= (3, 11) and TYPE_CHECKING: + + def request( + self, + method: str, + url: StrOrURL, + **kwargs: Unpack[_RequestOptions], + ) -> "_RequestContextManager": ... + + else: + + def request( + self, method: str, url: StrOrURL, **kwargs: Any + ) -> "_RequestContextManager": + """Perform HTTP request.""" + return _RequestContextManager(self._request(method, url, **kwargs)) + + def _build_url(self, str_or_url: StrOrURL) -> URL: + url = URL(str_or_url) + if self._base_url and not url.absolute: + return self._base_url.join(url) + return url + + async def _request( + self, + method: str, + str_or_url: StrOrURL, + *, + params: Query = None, + data: Any = None, + json: Any = None, + cookies: LooseCookies | None = None, + headers: LooseHeaders | None = None, + skip_auto_headers: Iterable[str] | None = None, + auth: BasicAuth | None = None, + allow_redirects: bool = True, + max_redirects: int = 10, + compress: str | bool | None = None, + chunked: bool | None = None, + expect100: bool = False, + raise_for_status: ( + None | bool | Callable[[ClientResponse], Awaitable[None]] + ) = None, + read_until_eof: bool = True, + proxy: StrOrURL | None = None, + proxy_auth: BasicAuth | None = None, + timeout: ClientTimeout | _SENTINEL = sentinel, + verify_ssl: bool | None = None, + fingerprint: bytes | None = None, + ssl_context: SSLContext | None = None, + ssl: SSLContext | bool | Fingerprint = True, + server_hostname: str | None = None, + proxy_headers: LooseHeaders | None = None, + trace_request_ctx: object = None, + read_bufsize: int | None = None, + auto_decompress: bool | None = None, + max_line_size: int | None = None, + max_field_size: int | None = None, + max_headers: int | None = None, + middlewares: Sequence[ClientMiddlewareType] | None = None, + ) -> ClientResponse: + + # NOTE: timeout clamps existing connect and read timeouts. We cannot + # set the default to None because we need to detect if the user wants + # to use the existing timeouts by setting timeout to None. + + if self.closed: + raise RuntimeError("Session is closed") + + method = method.upper() + ssl = _merge_ssl_params(ssl, verify_ssl, ssl_context, fingerprint) + + if auth is not None: + warnings.warn( + "The 'auth' parameter is deprecated and will be removed in v4;" + " pass headers={'Authorization': " + "aiohttp.encode_basic_auth(login, password)} instead", + DeprecationWarning, + stacklevel=3, + ) + if proxy_auth is not None: + warnings.warn( + "The 'proxy_auth' parameter is deprecated and will be removed in v4;" + " pass proxy_headers={'Proxy-Authorization': " + "aiohttp.encode_basic_auth(login, password)} instead", + DeprecationWarning, + stacklevel=3, + ) + + if data is not None and json is not None: + raise ValueError( + "data and json parameters can not be used at the same time" + ) + elif json is not None: + if self._json_serialize_bytes is not None: + data = payload.JsonBytesPayload(json, dumps=self._json_serialize_bytes) + else: + data = payload.JsonPayload(json, dumps=self._json_serialize) + + if not isinstance(chunked, bool) and chunked is not None: + warnings.warn("Chunk size is deprecated #1615", DeprecationWarning) + + redirects = 0 + history: list[ClientResponse] = [] + version = self._version + params = params or {} + + # Merge with default headers and transform to CIMultiDict + headers = self._prepare_headers(headers) + + try: + url = self._build_url(str_or_url) + except ValueError as e: + raise InvalidUrlClientError(str_or_url) from e + + assert self._connector is not None + if url.scheme not in self._connector.allowed_protocol_schema_set: + raise NonHttpUrlClientError(url) + + skip_headers: Iterable[istr] | None + if skip_auto_headers is not None: + skip_headers = { + istr(i) for i in skip_auto_headers + } | self._skip_auto_headers + elif self._skip_auto_headers: + skip_headers = self._skip_auto_headers + else: + skip_headers = None + + if proxy is None: + proxy = self._default_proxy + if proxy_auth is None: + proxy_auth = self._default_proxy_auth + + if proxy is None: + proxy_headers = None + else: + proxy_headers = self._prepare_headers(proxy_headers) + try: + proxy = URL(proxy) + except ValueError as e: + raise InvalidURL(proxy) from e + + if timeout is sentinel: + real_timeout: ClientTimeout = self._timeout + else: + if not isinstance(timeout, ClientTimeout): + real_timeout = ClientTimeout(total=timeout) + else: + real_timeout = timeout + # timeout is cumulative for all request operations + # (request, redirects, responses, data consuming) + tm = TimeoutHandle( + self._loop, real_timeout.total, ceil_threshold=real_timeout.ceil_threshold + ) + handle = tm.start() + + if read_bufsize is None: + read_bufsize = self._read_bufsize + + if auto_decompress is None: + auto_decompress = self._auto_decompress + + if max_line_size is None: + max_line_size = self._max_line_size + + if max_field_size is None: + max_field_size = self._max_field_size + + if max_headers is None: + max_headers = self._max_headers + + traces = [ + Trace( + self, + trace_config, + trace_config.trace_config_ctx(trace_request_ctx=trace_request_ctx), + ) + for trace_config in self._trace_configs + ] + + for trace in traces: + await trace.send_request_start(method, url.update_query(params), headers) + + timer = tm.timer() + req: ClientRequest | None = None + try: + with timer: + # https://www.rfc-editor.org/rfc/rfc9112.html#name-retrying-requests + retry_persistent_connection = ( + self._retry_connection and method in IDEMPOTENT_METHODS + ) + while True: + url, auth_from_url = strip_auth_from_url(url) + if not url.raw_host: + # NOTE: Bail early, otherwise, causes `InvalidURL` through + # NOTE: `self._request_class()` below. + err_exc_cls = ( + InvalidUrlRedirectClientError + if redirects + else InvalidUrlClientError + ) + raise err_exc_cls(url) + # If `auth` was passed for an already authenticated URL, + # disallow only if this is the initial URL; this is to avoid issues + # with sketchy redirects that are not the caller's responsibility + if not history and (auth and auth_from_url): + raise ValueError( + "Cannot combine AUTH argument with " + "credentials encoded in URL" + ) + + # Override the auth with the one from the URL only if we + # have no auth, or if we got an auth from a redirect URL + if auth is None or (history and auth_from_url is not None): + auth = auth_from_url + + if ( + auth is None + and self._default_auth + and ( + not self._base_url or self._base_url_origin == url.origin() + ) + ): + auth = self._default_auth + + # Try netrc if auth is still None and trust_env is enabled. + if auth is None and self._trust_env and url.host is not None: + auth = await self._loop.run_in_executor( + None, self._get_netrc_auth, url.host + ) + + # It would be confusing if we support explicit + # Authorization header with auth argument + if ( + headers is not None + and auth is not None + and hdrs.AUTHORIZATION in headers + ): + raise ValueError( + "Cannot combine AUTHORIZATION header " + "with AUTH argument or credentials " + "encoded in URL" + ) + + all_cookies = self._cookie_jar.filter_cookies(url) + + if cookies is not None: + tmp_cookie_jar = CookieJar( + unsafe=self._cookie_jar.unsafe, + quote_cookie=self._cookie_jar.quote_cookie, + ) + tmp_cookie_jar.update_cookies(cookies) + req_cookies = tmp_cookie_jar.filter_cookies(url) + if req_cookies: + all_cookies.load(req_cookies) + + proxy_: URL | None = None + if proxy is not None: + proxy_ = URL(proxy) + elif self._trust_env: + with suppress(LookupError): + proxy_, proxy_auth = await asyncio.to_thread( + get_env_proxy_for_url, url + ) + + req = self._request_class( + method, + url, + params=params, + headers=headers, + skip_auto_headers=skip_headers, + data=data, + cookies=all_cookies, + auth=auth, + version=version, + compress=compress, + chunked=chunked, + expect100=expect100, + loop=self._loop, + response_class=self._response_class, + proxy=proxy_, + proxy_auth=proxy_auth, + timer=timer, + session=self, + ssl=ssl if ssl is not None else True, + server_hostname=server_hostname, + proxy_headers=proxy_headers, + traces=traces, + trust_env=self.trust_env, + ) + + async def _connect_and_send_request( + req: ClientRequest, + ) -> ClientResponse: + # connection timeout + assert self._connector is not None + try: + conn = await self._connector.connect( + req, traces=traces, timeout=real_timeout + ) + except asyncio.TimeoutError as exc: + raise ConnectionTimeoutError( + f"Connection timeout to host {req.url}" + ) from exc + + assert conn.protocol is not None + conn.protocol.set_response_params( + timer=timer, + skip_payload=req.method in EMPTY_BODY_METHODS, + read_until_eof=read_until_eof, + auto_decompress=auto_decompress, + read_timeout=real_timeout.sock_read, + read_bufsize=read_bufsize, + timeout_ceil_threshold=self._connector._timeout_ceil_threshold, + max_line_size=max_line_size, + max_field_size=max_field_size, + max_headers=max_headers, + ) + try: + resp = await req.send(conn) + try: + await resp.start(conn) + except BaseException: + resp.close() + raise + except BaseException: + conn.close() + raise + return resp + + # Apply middleware (if any) - per-request middleware overrides session middleware + effective_middlewares = ( + self._middlewares if middlewares is None else middlewares + ) + + if effective_middlewares: + handler = build_client_middlewares( + _connect_and_send_request, effective_middlewares + ) + else: + handler = _connect_and_send_request + + try: + resp = await handler(req) + # Client connector errors should not be retried + except ( + ConnectionTimeoutError, + ClientConnectorError, + ClientConnectorCertificateError, + ClientConnectorSSLError, + ): + raise + except (ClientOSError, ServerDisconnectedError): + if retry_persistent_connection: + retry_persistent_connection = False + continue + raise + except ClientError: + raise + except OSError as exc: + if exc.errno is None and isinstance(exc, asyncio.TimeoutError): + raise + raise ClientOSError(*exc.args) from exc + + # Update cookies from raw headers to preserve duplicates + if resp._raw_cookie_headers: + self._cookie_jar.update_cookies_from_headers( + resp._raw_cookie_headers, resp.url + ) + + # redirects + if resp.status in (301, 302, 303, 307, 308) and allow_redirects: + + for trace in traces: + await trace.send_request_redirect( + method, url.update_query(params), headers, resp + ) + + redirects += 1 + history.append(resp) + if max_redirects and redirects >= max_redirects: + if req._body is not None: + await req._body.close() + resp.close() + raise TooManyRedirects( + history[0].request_info, tuple(history) + ) + + # For 301 and 302, mimic IE, now changed in RFC + # https://github.com/kennethreitz/requests/pull/269 + if (resp.status == 303 and resp.method != hdrs.METH_HEAD) or ( + resp.status in (301, 302) and resp.method == hdrs.METH_POST + ): + method = hdrs.METH_GET + data = None + if headers.get(hdrs.CONTENT_LENGTH): + headers.pop(hdrs.CONTENT_LENGTH) + else: + # For 307/308, always preserve the request body + # For 301/302 with non-POST methods, preserve the request body + # https://www.rfc-editor.org/rfc/rfc9110#section-15.4.3-3.1 + # Use the existing payload to avoid recreating it from + # a potentially consumed file. + # + # If the payload is already consumed and cannot be replayed, + # fail fast instead of silently sending an empty body. + if req._body is not None and req._body.consumed: + resp.close() + raise ClientPayloadError( + "Cannot follow redirect with a consumed request " + "body. Use bytes, a seekable file-like object, " + "or set allow_redirects=False." + ) + data = req._body + + r_url = resp.headers.get(hdrs.LOCATION) or resp.headers.get( + hdrs.URI + ) + if r_url is None: + # see github.com/aio-libs/aiohttp/issues/2022 + break + else: + # reading from correct redirection + # response is forbidden + resp.release() + + try: + parsed_redirect_url = URL( + r_url, encoded=not self._requote_redirect_url + ) + except ValueError as e: + if req._body is not None: + await req._body.close() + resp.close() + raise InvalidUrlRedirectClientError( + r_url, + "Server attempted redirecting to a location that does not look like a URL", + ) from e + + scheme = parsed_redirect_url.scheme + if scheme not in HTTP_AND_EMPTY_SCHEMA_SET: + if req._body is not None: + await req._body.close() + resp.close() + raise NonHttpUrlRedirectClientError(r_url) + elif not scheme: + parsed_redirect_url = url.join(parsed_redirect_url) + + try: + redirect_origin = parsed_redirect_url.origin() + except ValueError as origin_val_err: + if req._body is not None: + await req._body.close() + resp.close() + raise InvalidUrlRedirectClientError( + parsed_redirect_url, + "Invalid redirect URL origin", + ) from origin_val_err + + if url.origin() != redirect_origin: + auth = None + cookies = None + headers.popall(hdrs.AUTHORIZATION, None) + headers.popall(hdrs.COOKIE, None) + headers.popall(hdrs.PROXY_AUTHORIZATION, None) + + url = parsed_redirect_url + params = {} + resp.release() + continue + + break + + if req._body is not None: + await req._body.close() + # check response status + if raise_for_status is None: + raise_for_status = self._raise_for_status + + if raise_for_status is None: + pass + elif callable(raise_for_status): + await raise_for_status(resp) + elif raise_for_status: + resp.raise_for_status() + + # register connection + if handle is not None: + if resp.connection is not None: + resp.connection.add_callback(handle.cancel) + else: + handle.cancel() + + resp._history = tuple(history) + + for trace in traces: + await trace.send_request_end( + method, url.update_query(params), headers, resp + ) + return resp + + except BaseException as e: + # cleanup timer + tm.close() + if handle: + handle.cancel() + handle = None + + if req is not None and req._body is not None: + await req._body.close() + + for trace in traces: + await trace.send_request_exception( + method, url.update_query(params), headers, e + ) + raise + + if sys.version_info >= (3, 11) and TYPE_CHECKING: + + @overload + def ws_connect( + self, + url: StrOrURL, + *, + decode_text: Literal[True] = ..., + **kwargs: Unpack[_WSConnectOptions], + ) -> "_BaseRequestContextManager[ClientWebSocketResponse[Literal[True]]]": ... + + @overload + def ws_connect( + self, + url: StrOrURL, + *, + decode_text: Literal[False], + **kwargs: Unpack[_WSConnectOptions], + ) -> "_BaseRequestContextManager[ClientWebSocketResponse[Literal[False]]]": ... + + @overload + def ws_connect( + self, + url: StrOrURL, + *, + decode_text: bool = ..., + **kwargs: Unpack[_WSConnectOptions], + ) -> "_BaseRequestContextManager[ClientWebSocketResponse[bool]]": ... + + def ws_connect( + self, + url: StrOrURL, + *, + method: str = hdrs.METH_GET, + protocols: Iterable[str] = (), + timeout: ClientWSTimeout | _SENTINEL = sentinel, + receive_timeout: float | None = None, + autoclose: bool = True, + autoping: bool = True, + heartbeat: float | None = None, + auth: BasicAuth | None = None, + origin: str | None = None, + params: Query = None, + headers: LooseHeaders | None = None, + proxy: StrOrURL | None = None, + proxy_auth: BasicAuth | None = None, + ssl: SSLContext | bool | Fingerprint = True, + verify_ssl: bool | None = None, + fingerprint: bytes | None = None, + ssl_context: SSLContext | None = None, + server_hostname: str | None = None, + proxy_headers: LooseHeaders | None = None, + compress: int = 0, + max_msg_size: int = 4 * 1024 * 1024, + decode_text: bool = True, + ) -> "_BaseRequestContextManager[ClientWebSocketResponse[bool]]": + """Initiate websocket connection.""" + return _WSRequestContextManager( + self._ws_connect( + url, + method=method, + protocols=protocols, + timeout=timeout, + receive_timeout=receive_timeout, + autoclose=autoclose, + autoping=autoping, + heartbeat=heartbeat, + auth=auth, + origin=origin, + params=params, + headers=headers, + proxy=proxy, + proxy_auth=proxy_auth, + ssl=ssl, + verify_ssl=verify_ssl, + fingerprint=fingerprint, + ssl_context=ssl_context, + server_hostname=server_hostname, + proxy_headers=proxy_headers, + compress=compress, + max_msg_size=max_msg_size, + decode_text=decode_text, + ) + ) + + if sys.version_info >= (3, 11) and TYPE_CHECKING: + + @overload + async def _ws_connect( + self, + url: StrOrURL, + *, + decode_text: Literal[True] = ..., + **kwargs: Unpack[_WSConnectOptions], + ) -> "ClientWebSocketResponse[Literal[True]]": ... + + @overload + async def _ws_connect( + self, + url: StrOrURL, + *, + decode_text: Literal[False], + **kwargs: Unpack[_WSConnectOptions], + ) -> "ClientWebSocketResponse[Literal[False]]": ... + + @overload + async def _ws_connect( + self, + url: StrOrURL, + *, + decode_text: bool = ..., + **kwargs: Unpack[_WSConnectOptions], + ) -> "ClientWebSocketResponse[bool]": ... + + async def _ws_connect( + self, + url: StrOrURL, + *, + method: str = hdrs.METH_GET, + protocols: Iterable[str] = (), + timeout: ClientWSTimeout | _SENTINEL = sentinel, + receive_timeout: float | None = None, + autoclose: bool = True, + autoping: bool = True, + heartbeat: float | None = None, + auth: BasicAuth | None = None, + origin: str | None = None, + params: Query = None, + headers: LooseHeaders | None = None, + proxy: StrOrURL | None = None, + proxy_auth: BasicAuth | None = None, + ssl: SSLContext | bool | Fingerprint = True, + verify_ssl: bool | None = None, + fingerprint: bytes | None = None, + ssl_context: SSLContext | None = None, + server_hostname: str | None = None, + proxy_headers: LooseHeaders | None = None, + compress: int = 0, + max_msg_size: int = 4 * 1024 * 1024, + decode_text: bool = True, + ) -> "ClientWebSocketResponse[bool]": + if auth is not None: + warnings.warn( + "The 'auth' parameter is deprecated and will be removed in v4;" + " pass headers={'Authorization': " + "aiohttp.encode_basic_auth(login, password)} instead", + DeprecationWarning, + stacklevel=3, + ) + if proxy_auth is not None: + warnings.warn( + "The 'proxy_auth' parameter is deprecated and will be removed in v4;" + " pass proxy_headers={'Proxy-Authorization': " + "aiohttp.encode_basic_auth(login, password)} instead", + DeprecationWarning, + stacklevel=3, + ) + if timeout is not sentinel: + if isinstance(timeout, ClientWSTimeout): + ws_timeout = timeout + else: + warnings.warn( + "parameter 'timeout' of type 'float' " + "is deprecated, please use " + "'timeout=ClientWSTimeout(ws_close=...)'", + DeprecationWarning, + stacklevel=2, + ) + ws_timeout = ClientWSTimeout(ws_close=timeout) + else: + ws_timeout = DEFAULT_WS_CLIENT_TIMEOUT + if receive_timeout is not None: + warnings.warn( + "float parameter 'receive_timeout' " + "is deprecated, please use parameter " + "'timeout=ClientWSTimeout(ws_receive=...)'", + DeprecationWarning, + stacklevel=2, + ) + ws_timeout = attr.evolve(ws_timeout, ws_receive=receive_timeout) + + if headers is None: + real_headers: CIMultiDict[str] = CIMultiDict() + else: + real_headers = CIMultiDict(headers) + + default_headers = { + hdrs.UPGRADE: "websocket", + hdrs.CONNECTION: "Upgrade", + hdrs.SEC_WEBSOCKET_VERSION: "13", + } + + for key, value in default_headers.items(): + real_headers.setdefault(key, value) + + sec_key = base64.b64encode(os.urandom(16)) + real_headers[hdrs.SEC_WEBSOCKET_KEY] = sec_key.decode() + + if protocols: + real_headers[hdrs.SEC_WEBSOCKET_PROTOCOL] = ",".join(protocols) + if origin is not None: + real_headers[hdrs.ORIGIN] = origin + if compress: + extstr = ws_ext_gen(compress=compress) + real_headers[hdrs.SEC_WEBSOCKET_EXTENSIONS] = extstr + + # For the sake of backward compatibility, if user passes in None, convert it to True + if ssl is None: + warnings.warn( + "ssl=None is deprecated, please use ssl=True", + DeprecationWarning, + stacklevel=2, + ) + ssl = True + ssl = _merge_ssl_params(ssl, verify_ssl, ssl_context, fingerprint) + + # send request + resp = await self.request( + method, + url, + params=params, + headers=real_headers, + read_until_eof=False, + auth=auth, + proxy=proxy, + proxy_auth=proxy_auth, + ssl=ssl, + server_hostname=server_hostname, + proxy_headers=proxy_headers, + ) + + try: + # check handshake + if resp.status != 101: + raise WSServerHandshakeError( + resp.request_info, + resp.history, + message="Invalid response status", + status=resp.status, + headers=resp.headers, + ) + + if resp.headers.get(hdrs.UPGRADE, "").lower() != "websocket": + raise WSServerHandshakeError( + resp.request_info, + resp.history, + message="Invalid upgrade header", + status=resp.status, + headers=resp.headers, + ) + + if resp.headers.get(hdrs.CONNECTION, "").lower() != "upgrade": + raise WSServerHandshakeError( + resp.request_info, + resp.history, + message="Invalid connection header", + status=resp.status, + headers=resp.headers, + ) + + # key calculation + r_key = resp.headers.get(hdrs.SEC_WEBSOCKET_ACCEPT, "") + match = base64.b64encode(hashlib.sha1(sec_key + WS_KEY).digest()).decode() + if r_key != match: + raise WSServerHandshakeError( + resp.request_info, + resp.history, + message="Invalid challenge response", + status=resp.status, + headers=resp.headers, + ) + + # websocket protocol + protocol = None + if protocols and hdrs.SEC_WEBSOCKET_PROTOCOL in resp.headers: + resp_protocols = [ + proto.strip() + for proto in resp.headers[hdrs.SEC_WEBSOCKET_PROTOCOL].split(",") + ] + + for proto in resp_protocols: + if proto in protocols: + protocol = proto + break + + # websocket compress + notakeover = False + if compress: + compress_hdrs = resp.headers.get(hdrs.SEC_WEBSOCKET_EXTENSIONS) + if compress_hdrs: + try: + compress, notakeover = ws_ext_parse(compress_hdrs) + except WSHandshakeError as exc: + raise WSServerHandshakeError( + resp.request_info, + resp.history, + message=exc.args[0], + status=resp.status, + headers=resp.headers, + ) from exc + else: + compress = 0 + notakeover = False + + conn = resp.connection + assert conn is not None + conn_proto = conn.protocol + assert conn_proto is not None + + # For WS connection the read_timeout must be either receive_timeout or greater + # None == no timeout, i.e. infinite timeout, so None is the max timeout possible + if ws_timeout.ws_receive is None: + # Reset regardless + conn_proto.read_timeout = None + elif conn_proto.read_timeout is not None: + conn_proto.read_timeout = max( + ws_timeout.ws_receive, conn_proto.read_timeout + ) + + transport = conn.transport + assert transport is not None + reader = WebSocketDataQueue(conn_proto, DEFAULT_CHUNK_SIZE, loop=self._loop) + writer = WebSocketWriter( + conn_proto, + transport, + use_mask=True, + compress=compress, + notakeover=notakeover, + ) + except BaseException: + resp.close() + raise + else: + ws_resp = self._ws_response_class( + reader, + writer, + protocol, + resp, + ws_timeout, + autoclose, + autoping, + self._loop, + heartbeat=heartbeat, + compress=compress, + client_notakeover=notakeover, + ) + parser = WebSocketReader( + reader, + max_msg_size, + compress=bool(compress), + decode_text=decode_text, + ) + cb = None if heartbeat is None else ws_resp._on_data_received + conn_proto.set_parser(parser, reader, data_received_cb=cb) + return ws_resp + + def _prepare_headers(self, headers: LooseHeaders | None) -> "CIMultiDict[str]": + """Add default headers and transform it to CIMultiDict""" + # Convert headers to MultiDict + result = CIMultiDict(self._default_headers) + if headers: + if not isinstance(headers, (MultiDictProxy, MultiDict)): + headers = CIMultiDict(headers) + added_names: set[str] = set() + for key, value in headers.items(): + if key in added_names: + result.add(key, value) + else: + result[key] = value + added_names.add(key) + return result + + def _get_netrc_auth(self, host: str) -> BasicAuth | None: + """ + Get auth from netrc for the given host. + + This method is designed to be called in an executor to avoid + blocking I/O in the event loop. + """ + netrc_obj = netrc_from_env() + try: + return basicauth_from_netrc(netrc_obj, host) + except LookupError: + return None + + if sys.version_info >= (3, 11) and TYPE_CHECKING: + + def get( + self, + url: StrOrURL, + **kwargs: Unpack[_RequestOptions], + ) -> "_RequestContextManager": ... + + def options( + self, + url: StrOrURL, + **kwargs: Unpack[_RequestOptions], + ) -> "_RequestContextManager": ... + + def head( + self, + url: StrOrURL, + **kwargs: Unpack[_RequestOptions], + ) -> "_RequestContextManager": ... + + def post( + self, + url: StrOrURL, + **kwargs: Unpack[_RequestOptions], + ) -> "_RequestContextManager": ... + + def put( + self, + url: StrOrURL, + **kwargs: Unpack[_RequestOptions], + ) -> "_RequestContextManager": ... + + def patch( + self, + url: StrOrURL, + **kwargs: Unpack[_RequestOptions], + ) -> "_RequestContextManager": ... + + def delete( + self, + url: StrOrURL, + **kwargs: Unpack[_RequestOptions], + ) -> "_RequestContextManager": ... + + else: + + def get( + self, url: StrOrURL, *, allow_redirects: bool = True, **kwargs: Any + ) -> "_RequestContextManager": + """Perform HTTP GET request.""" + return _RequestContextManager( + self._request( + hdrs.METH_GET, url, allow_redirects=allow_redirects, **kwargs + ) + ) + + def options( + self, url: StrOrURL, *, allow_redirects: bool = True, **kwargs: Any + ) -> "_RequestContextManager": + """Perform HTTP OPTIONS request.""" + return _RequestContextManager( + self._request( + hdrs.METH_OPTIONS, url, allow_redirects=allow_redirects, **kwargs + ) + ) + + def head( + self, url: StrOrURL, *, allow_redirects: bool = False, **kwargs: Any + ) -> "_RequestContextManager": + """Perform HTTP HEAD request.""" + return _RequestContextManager( + self._request( + hdrs.METH_HEAD, url, allow_redirects=allow_redirects, **kwargs + ) + ) + + def post( + self, url: StrOrURL, *, data: Any = None, **kwargs: Any + ) -> "_RequestContextManager": + """Perform HTTP POST request.""" + return _RequestContextManager( + self._request(hdrs.METH_POST, url, data=data, **kwargs) + ) + + def put( + self, url: StrOrURL, *, data: Any = None, **kwargs: Any + ) -> "_RequestContextManager": + """Perform HTTP PUT request.""" + return _RequestContextManager( + self._request(hdrs.METH_PUT, url, data=data, **kwargs) + ) + + def patch( + self, url: StrOrURL, *, data: Any = None, **kwargs: Any + ) -> "_RequestContextManager": + """Perform HTTP PATCH request.""" + return _RequestContextManager( + self._request(hdrs.METH_PATCH, url, data=data, **kwargs) + ) + + def delete(self, url: StrOrURL, **kwargs: Any) -> "_RequestContextManager": + """Perform HTTP DELETE request.""" + return _RequestContextManager( + self._request(hdrs.METH_DELETE, url, **kwargs) + ) + + async def close(self) -> None: + """Close underlying connector. + + Release all acquired resources. + """ + if not self.closed: + if self._connector is not None and self._connector_owner: + await self._connector.close() + self._connector = None + + @property + def closed(self) -> bool: + """Is client session closed. + + A readonly property. + """ + return self._connector is None or self._connector.closed + + @property + def connector(self) -> BaseConnector | None: + """Connector instance used for the session.""" + return self._connector + + @property + def cookie_jar(self) -> AbstractCookieJar: + """The session cookies.""" + return self._cookie_jar + + @property + def version(self) -> tuple[int, int]: + """The session HTTP protocol version.""" + return self._version + + @property + def requote_redirect_url(self) -> bool: + """Do URL requoting on redirection handling.""" + return self._requote_redirect_url + + @requote_redirect_url.setter + def requote_redirect_url(self, val: bool) -> None: + """Do URL requoting on redirection handling.""" + warnings.warn( + "session.requote_redirect_url modification is deprecated #2778", + DeprecationWarning, + stacklevel=2, + ) + self._requote_redirect_url = val + + @property + def loop(self) -> asyncio.AbstractEventLoop: + """Session's loop.""" + warnings.warn( + "client.loop property is deprecated", DeprecationWarning, stacklevel=2 + ) + return self._loop + + @property + def timeout(self) -> ClientTimeout: + """Timeout for the session.""" + return self._timeout + + @property + def headers(self) -> "CIMultiDict[str]": + """The default headers of the client session.""" + return self._default_headers + + @property + def skip_auto_headers(self) -> frozenset[istr]: + """Headers for which autogeneration should be skipped""" + return self._skip_auto_headers + + @property + def auth(self) -> BasicAuth | None: + """An object that represents HTTP Basic Authorization""" + return self._default_auth + + @property + def json_serialize(self) -> JSONEncoder: + """Json serializer callable""" + return self._json_serialize + + @property + def connector_owner(self) -> bool: + """Should connector be closed on session closing""" + return self._connector_owner + + @property + def raise_for_status( + self, + ) -> bool | Callable[[ClientResponse], Awaitable[None]]: + """Should `ClientResponse.raise_for_status()` be called for each response.""" + return self._raise_for_status + + @property + def auto_decompress(self) -> bool: + """Should the body response be automatically decompressed.""" + return self._auto_decompress + + @property + def trust_env(self) -> bool: + """ + Should proxies information from environment or netrc be trusted. + + Information is from HTTP_PROXY / HTTPS_PROXY environment variables + or ~/.netrc file if present. + """ + return self._trust_env + + @property + def trace_configs(self) -> list[TraceConfig]: + """A list of TraceConfig instances used for client tracing""" + return self._trace_configs + + def detach(self) -> None: + """Detach connector from session without closing the former. + + Session is switched to closed state anyway. + """ + self._connector = None + + def __enter__(self) -> None: + raise TypeError("Use async with instead") + + def __exit__( + self, + exc_type: type[BaseException] | None, + exc_val: BaseException | None, + exc_tb: TracebackType | None, + ) -> None: + # __exit__ should exist in pair with __enter__ but never executed + pass # pragma: no cover + + async def __aenter__(self) -> "ClientSession": + return self + + async def __aexit__( + self, + exc_type: type[BaseException] | None, + exc_val: BaseException | None, + exc_tb: TracebackType | None, + ) -> None: + await self.close() + + +class _BaseRequestContextManager( + Coroutine[Any, Any, _RetType_co], Generic[_RetType_co] +): + + __slots__ = ("_coro", "_resp") + + def __init__(self, coro: Coroutine[asyncio.Future[Any], None, _RetType_co]) -> None: + self._coro: Coroutine[asyncio.Future[Any], None, _RetType_co] = coro + + def send(self, arg: None) -> asyncio.Future[Any]: + return self._coro.send(arg) + + def throw(self, *args: Any, **kwargs: Any) -> asyncio.Future[Any]: + return self._coro.throw(*args, **kwargs) + + def close(self) -> None: + return self._coro.close() + + def __await__(self) -> Generator[Any, None, _RetType_co]: + ret = self._coro.__await__() + return ret + + def __iter__(self) -> Generator[Any, None, _RetType_co]: + return self.__await__() + + async def __aenter__(self) -> _RetType_co: + self._resp: _RetType_co = await self._coro + return await self._resp.__aenter__() # type: ignore[return-value] + + async def __aexit__( + self, + exc_type: type[BaseException] | None, + exc: BaseException | None, + tb: TracebackType | None, + ) -> None: + await self._resp.__aexit__(exc_type, exc, tb) + + +_RequestContextManager = _BaseRequestContextManager[ClientResponse] +_WSRequestContextManager = _BaseRequestContextManager[ClientWebSocketResponse[bool]] + + +class _SessionRequestContextManager: + + __slots__ = ("_coro", "_resp", "_session") + + def __init__( + self, + coro: Coroutine[asyncio.Future[Any], None, ClientResponse], + session: ClientSession, + ) -> None: + self._coro = coro + self._resp: ClientResponse | None = None + self._session = session + + async def __aenter__(self) -> ClientResponse: + try: + self._resp = await self._coro + except BaseException: + await self._session.close() + raise + else: + return self._resp + + async def __aexit__( + self, + exc_type: type[BaseException] | None, + exc: BaseException | None, + tb: TracebackType | None, + ) -> None: + assert self._resp is not None + self._resp.close() + await self._session.close() + + +if sys.version_info >= (3, 11) and TYPE_CHECKING: + + def request( + method: str, + url: StrOrURL, + *, + version: HttpVersion = http.HttpVersion11, + connector: BaseConnector | None = None, + loop: asyncio.AbstractEventLoop | None = None, + **kwargs: Unpack[_RequestOptions], + ) -> _SessionRequestContextManager: ... + +else: + + def request( + method: str, + url: StrOrURL, + *, + version: HttpVersion = http.HttpVersion11, + connector: BaseConnector | None = None, + loop: asyncio.AbstractEventLoop | None = None, + **kwargs: Any, + ) -> _SessionRequestContextManager: + """Constructs and sends a request. + + Returns response object. + method - HTTP method + url - request url + params - (optional) Dictionary or bytes to be sent in the query + string of the new request + data - (optional) Dictionary, bytes, or file-like object to + send in the body of the request + json - (optional) Any json compatible python object + headers - (optional) Dictionary of HTTP Headers to send with + the request + cookies - (optional) Dict object to send with the request + auth - (optional) BasicAuth named tuple represent HTTP Basic Auth + auth - aiohttp.helpers.BasicAuth + allow_redirects - (optional) If set to False, do not follow + redirects + version - Request HTTP version. + compress - Set to True if request has to be compressed + with deflate encoding. + chunked - Set to chunk size for chunked transfer encoding. + expect100 - Expect 100-continue response from server. + connector - BaseConnector sub-class instance to support + connection pooling. + read_until_eof - Read response until eof if response + does not have Content-Length header. + loop - Optional event loop. + timeout - Optional ClientTimeout settings structure, 5min + total timeout by default. + Usage:: + >>> import aiohttp + >>> async with aiohttp.request('GET', 'http://python.org/') as resp: + ... print(resp) + ... data = await resp.read() + + """ + connector_owner = False + if connector is None: + connector_owner = True + connector = TCPConnector(loop=loop, force_close=True) + + session = ClientSession( + loop=loop, + cookies=kwargs.pop("cookies", None), + version=version, + timeout=kwargs.pop("timeout", sentinel), + connector=connector, + connector_owner=connector_owner, + ) + + return _SessionRequestContextManager( + session._request(method, url, **kwargs), + session, + ) diff --git a/py_modules/aiohttp/client_exceptions.py b/py_modules/aiohttp/client_exceptions.py new file mode 100644 index 0000000..e3e503b --- /dev/null +++ b/py_modules/aiohttp/client_exceptions.py @@ -0,0 +1,426 @@ +"""HTTP related errors.""" + +import asyncio +import warnings +from typing import TYPE_CHECKING, Union + +from multidict import MultiMapping + +from .typedefs import StrOrURL + +if TYPE_CHECKING: + import ssl + + SSLContext = ssl.SSLContext +else: + try: + import ssl + + SSLContext = ssl.SSLContext + except ImportError: # pragma: no cover + ssl = SSLContext = None # type: ignore[assignment] + +if TYPE_CHECKING: + from .client_reqrep import ClientResponse, ConnectionKey, Fingerprint, RequestInfo + from .http_parser import RawResponseMessage +else: + RequestInfo = ClientResponse = ConnectionKey = RawResponseMessage = None + +__all__ = ( + "ClientError", + "ClientConnectionError", + "ClientConnectionResetError", + "ClientOSError", + "ClientConnectorError", + "ClientProxyConnectionError", + "ClientSSLError", + "ClientConnectorDNSError", + "ClientConnectorSSLError", + "ClientConnectorCertificateError", + "ConnectionTimeoutError", + "SocketTimeoutError", + "ServerConnectionError", + "ServerTimeoutError", + "ServerDisconnectedError", + "ServerFingerprintMismatch", + "ClientResponseError", + "ClientHttpProxyError", + "WSServerHandshakeError", + "ContentTypeError", + "ClientPayloadError", + "InvalidURL", + "InvalidUrlClientError", + "RedirectClientError", + "NonHttpUrlClientError", + "InvalidUrlRedirectClientError", + "NonHttpUrlRedirectClientError", + "WSMessageTypeError", +) + + +class ClientError(Exception): + """Base class for client connection errors.""" + + +class ClientResponseError(ClientError): + """Base class for exceptions that occur after getting a response. + + request_info: An instance of RequestInfo. + history: A sequence of responses, if redirects occurred. + status: HTTP status code. + message: Error message. + headers: Response headers. + """ + + def __init__( + self, + request_info: RequestInfo, + history: tuple[ClientResponse, ...], + *, + code: int | None = None, + status: int | None = None, + message: str = "", + headers: MultiMapping[str] | None = None, + ) -> None: + self.request_info = request_info + if code is not None: + if status is not None: + raise ValueError( + "Both code and status arguments are provided; " + "code is deprecated, use status instead" + ) + warnings.warn( + "code argument is deprecated, use status instead", + DeprecationWarning, + stacklevel=2, + ) + if status is not None: + self.status = status + elif code is not None: + self.status = code + else: + self.status = 0 + self.message = message + self.headers = headers + self.history = history + self.args = (request_info, history) + + def __str__(self) -> str: + return f"{self.status}, message={self.message!r}, url={str(self.request_info.real_url)!r}" + + def __repr__(self) -> str: + args = f"{self.request_info!r}, {self.history!r}" + if self.status != 0: + args += f", status={self.status!r}" + if self.message != "": + args += f", message={self.message!r}" + if self.headers is not None: + args += f", headers={self.headers!r}" + return f"{type(self).__name__}({args})" + + @property + def code(self) -> int: + warnings.warn( + "code property is deprecated, use status instead", + DeprecationWarning, + stacklevel=2, + ) + return self.status + + @code.setter + def code(self, value: int) -> None: + warnings.warn( + "code property is deprecated, use status instead", + DeprecationWarning, + stacklevel=2, + ) + self.status = value + + +class ContentTypeError(ClientResponseError): + """ContentType found is not valid.""" + + +class WSServerHandshakeError(ClientResponseError): + """websocket server handshake error.""" + + +class ClientHttpProxyError(ClientResponseError): + """HTTP proxy error. + + Raised in :class:`aiohttp.connector.TCPConnector` if + proxy responds with status other than ``200 OK`` + on ``CONNECT`` request. + """ + + +class TooManyRedirects(ClientResponseError): + """Client was redirected too many times.""" + + +class ClientConnectionError(ClientError): + """Base class for client socket errors.""" + + +class ClientConnectionResetError(ClientConnectionError, ConnectionResetError): + """ConnectionResetError""" + + +class ClientOSError(ClientConnectionError, OSError): + """OSError error.""" + + +class ClientConnectorError(ClientOSError): + """Client connector error. + + Raised in :class:`aiohttp.connector.TCPConnector` if + a connection can not be established. + """ + + def __init__(self, connection_key: ConnectionKey, os_error: OSError) -> None: + self._conn_key = connection_key + self._os_error = os_error + super().__init__(os_error.errno, os_error.strerror) + self.args = (connection_key, os_error) + + @property + def os_error(self) -> OSError: + return self._os_error + + @property + def host(self) -> str: + return self._conn_key.host + + @property + def port(self) -> int | None: + return self._conn_key.port + + @property + def ssl(self) -> Union[SSLContext, bool, "Fingerprint"]: + return self._conn_key.ssl + + def __str__(self) -> str: + return "Cannot connect to host {0.host}:{0.port} ssl:{1} [{2}]".format( + self, "default" if self.ssl is True else self.ssl, self.strerror + ) + + # OSError.__reduce__ does too much black magick + __reduce__ = BaseException.__reduce__ + + +class ClientConnectorDNSError(ClientConnectorError): + """DNS resolution failed during client connection. + + Raised in :class:`aiohttp.connector.TCPConnector` if + DNS resolution fails. + """ + + +class ClientProxyConnectionError(ClientConnectorError): + """Proxy connection error. + + Raised in :class:`aiohttp.connector.TCPConnector` if + connection to proxy can not be established. + """ + + +class UnixClientConnectorError(ClientConnectorError): + """Unix connector error. + + Raised in :py:class:`aiohttp.connector.UnixConnector` + if connection to unix socket can not be established. + """ + + def __init__( + self, path: str, connection_key: ConnectionKey, os_error: OSError + ) -> None: + self._path = path + super().__init__(connection_key, os_error) + + @property + def path(self) -> str: + return self._path + + def __str__(self) -> str: + return "Cannot connect to unix socket {0.path} ssl:{1} [{2}]".format( + self, "default" if self.ssl is True else self.ssl, self.strerror + ) + + +class ServerConnectionError(ClientConnectionError): + """Server connection errors.""" + + +class ServerDisconnectedError(ServerConnectionError): + """Server disconnected.""" + + def __init__(self, message: RawResponseMessage | str | None = None) -> None: + if message is None: + message = "Server disconnected" + + self.args = (message,) + self.message = message + + +class ServerTimeoutError(ServerConnectionError, asyncio.TimeoutError): + """Server timeout error.""" + + +class ConnectionTimeoutError(ServerTimeoutError): + """Connection timeout error.""" + + +class SocketTimeoutError(ServerTimeoutError): + """Socket timeout error.""" + + +class ServerFingerprintMismatch(ServerConnectionError): + """SSL certificate does not match expected fingerprint.""" + + def __init__(self, expected: bytes, got: bytes, host: str, port: int) -> None: + self.expected = expected + self.got = got + self.host = host + self.port = port + self.args = (expected, got, host, port) + + def __repr__(self) -> str: + return f"<{self.__class__.__name__} expected={self.expected!r} got={self.got!r} host={self.host!r} port={self.port!r}>" + + +class ClientPayloadError(ClientError): + """Response payload error.""" + + +class InvalidURL(ClientError, ValueError): + """Invalid URL. + + URL used for fetching is malformed, e.g. it doesn't contains host + part. + """ + + # Derive from ValueError for backward compatibility + + def __init__(self, url: StrOrURL, description: str | None = None) -> None: + # The type of url is not yarl.URL because the exception can be raised + # on URL(url) call + self._url = url + self._description = description + + if description: + super().__init__(url, description) + else: + super().__init__(url) + + @property + def url(self) -> StrOrURL: + return self._url + + @property + def description(self) -> "str | None": + return self._description + + def __repr__(self) -> str: + return f"<{self.__class__.__name__} {self}>" + + def __str__(self) -> str: + if self._description: + return f"{self._url} - {self._description}" + return str(self._url) + + +class InvalidUrlClientError(InvalidURL): + """Invalid URL client error.""" + + +class RedirectClientError(ClientError): + """Client redirect error.""" + + +class NonHttpUrlClientError(ClientError): + """Non http URL client error.""" + + +class InvalidUrlRedirectClientError(InvalidUrlClientError, RedirectClientError): + """Invalid URL redirect client error.""" + + +class NonHttpUrlRedirectClientError(NonHttpUrlClientError, RedirectClientError): + """Non http URL redirect client error.""" + + +class ClientSSLError(ClientConnectorError): + """Base error for ssl.*Errors.""" + + +if ssl is not None: + cert_errors = (ssl.CertificateError,) + cert_errors_bases = ( + ClientSSLError, + ssl.CertificateError, + ) + + ssl_errors = (ssl.SSLError,) + ssl_error_bases = (ClientSSLError, ssl.SSLError) +else: # pragma: no cover + cert_errors = tuple() + cert_errors_bases = ( + ClientSSLError, + ValueError, + ) + + ssl_errors = tuple() + ssl_error_bases = (ClientSSLError,) + + +class ClientConnectorSSLError(*ssl_error_bases): # type: ignore[misc] + """Response ssl error.""" + + +class ClientConnectorCertificateError(*cert_errors_bases): # type: ignore[misc] + """Response certificate error.""" + + _conn_key: ConnectionKey + + def __init__( + # TODO: If we require ssl in future, this can become ssl.CertificateError + self, + connection_key: ConnectionKey, + certificate_error: Exception, + ) -> None: + if isinstance(certificate_error, cert_errors + (OSError,)): + # ssl.CertificateError has errno and strerror, so we should be fine + os_error = certificate_error + else: + os_error = OSError() + + super().__init__(connection_key, os_error) + self._certificate_error = certificate_error + self.args = (connection_key, certificate_error) + + @property + def certificate_error(self) -> Exception: + return self._certificate_error + + @property + def host(self) -> str: + return self._conn_key.host + + @property + def port(self) -> int | None: + return self._conn_key.port + + @property + def ssl(self) -> bool: + return self._conn_key.is_ssl + + def __str__(self) -> str: + return ( + f"Cannot connect to host {self.host}:{self.port} ssl:{self.ssl} " + f"[{self.certificate_error.__class__.__name__}: " + f"{self.certificate_error.args}]" + ) + + +class WSMessageTypeError(TypeError): + """WebSocket message type is not valid.""" diff --git a/py_modules/aiohttp/client_middleware_digest_auth.py b/py_modules/aiohttp/client_middleware_digest_auth.py new file mode 100644 index 0000000..fbcdf54 --- /dev/null +++ b/py_modules/aiohttp/client_middleware_digest_auth.py @@ -0,0 +1,507 @@ +""" +Digest authentication middleware for aiohttp client. + +This middleware implements HTTP Digest Authentication according to RFC 7616, +providing a more secure alternative to Basic Authentication. It supports all +standard hash algorithms including MD5, SHA, SHA-256, SHA-512 and their session +variants, as well as both 'auth' and 'auth-int' quality of protection (qop) options. +""" + +import hashlib +import os +import re +import sys +import time +from collections.abc import Callable +from typing import Final, Literal, TypedDict + +from yarl import URL + +from . import hdrs +from .client_exceptions import ClientError +from .client_middlewares import ClientHandlerType +from .client_reqrep import ClientRequest, ClientResponse +from .payload import Payload + + +class DigestAuthChallenge(TypedDict, total=False): + realm: str + nonce: str + qop: str + algorithm: str + opaque: str + domain: str + stale: str + + +DigestFunctions: dict[str, Callable[[bytes], "hashlib._Hash"]] = { + "MD5": hashlib.md5, + "MD5-SESS": hashlib.md5, + "SHA": hashlib.sha1, + "SHA-SESS": hashlib.sha1, + "SHA256": hashlib.sha256, + "SHA256-SESS": hashlib.sha256, + "SHA-256": hashlib.sha256, + "SHA-256-SESS": hashlib.sha256, + "SHA512": hashlib.sha512, + "SHA512-SESS": hashlib.sha512, + "SHA-512": hashlib.sha512, + "SHA-512-SESS": hashlib.sha512, +} + + +# Compile the regex pattern once at module level for performance +_HEADER_PAIRS_PATTERN = re.compile( + r'(?:^|\s|,\s*)(\w+)(?:\s*=\s*(?:"((?:[^"\\]|\\.)*)"|([^\s,]+)))?' + if sys.version_info < (3, 11) + else r'(?:^|\s|,\s*)((?>\w+))(?:\s*=\s*(?:"((?:[^"\\]|\\.)*)"|([^\s,]+)))?' + # +------------|--------|--|--||--|--|----|------|---|---||-----|-> Match valid start/sep + # +--------|--|--||--|--|----|------|---|---||-----|-> alphanumeric key (atomic group reduces backtracking) + # +--|--||--|--|----|------|---|---||-----|-> optional value; absent => bare auth-scheme token + # +--||--|--|----|------|---|---||-----|-> maybe whitespace + # +|--|--|----|------|---|---||-----|-> = (delimiter) + # +--|--|----|------|---|---||-----|-> maybe whitespace + # +--|----|------|---|---||-----|-> group quoted or unquoted + # +----|------|---|---||-----|-> if quoted... + # +------|---|---||-----|-> anything but " or \ + # +---|---||-----|-> escaped characters allowed + # +---||-----|-> or can be empty string + # +|-----|-> if unquoted... + # +-----|-> anything but , or + # +-> at least one char req'd +) + + +# RFC 7616: Challenge parameters to extract +CHALLENGE_FIELDS: Final[ + tuple[ + Literal["realm", "nonce", "qop", "algorithm", "opaque", "domain", "stale"], ... + ] +] = ( + "realm", + "nonce", + "qop", + "algorithm", + "opaque", + "domain", + "stale", +) + +# Supported digest authentication algorithms +# Use a tuple of sorted keys for predictable documentation and error messages +SUPPORTED_ALGORITHMS: Final[tuple[str, ...]] = tuple(sorted(DigestFunctions.keys())) + +# RFC 7616: Fields that require quoting in the Digest auth header +# These fields must be enclosed in double quotes in the Authorization header. +# Algorithm, qop, and nc are never quoted per RFC specifications. +# This frozen set is used by the template-based header construction to +# automatically determine which fields need quotes. +QUOTED_AUTH_FIELDS: Final[frozenset[str]] = frozenset( + {"username", "realm", "nonce", "uri", "response", "opaque", "cnonce"} +) + + +def escape_quotes(value: str) -> str: + """Escape backslashes and double quotes for HTTP quoted-strings.""" + return value.replace("\\", "\\\\").replace('"', '\\"') + + +def unescape_quotes(value: str) -> str: + """Unescape backslashes and double quotes in HTTP quoted-strings.""" + return value.replace('\\"', '"').replace("\\\\", "\\") + + +def parse_header_pairs(header: str) -> dict[str, str]: + """ + Parse key-value pairs from the first challenge of a WWW-Authenticate header. + + This function handles the complex format of WWW-Authenticate header values, + supporting both quoted and unquoted values, proper handling of commas in + quoted values, and whitespace variations per RFC 7616. + + A single header may carry several challenges + (https://www.rfc-editor.org/rfc/rfc7235#section-4.1). Parsing + stops at the next auth-scheme token so a later challenge's parameters cannot + overwrite the first challenge's values; a leading scheme token is skipped. + + Examples of supported formats: + - key1="value1", key2=value2 + - key1 = "value1" , key2="value, with, commas" + - key1=value1,key2="value2" + - realm="example.com", nonce="12345", qop="auth" + + Args: + header: The header value string to parse + + Returns: + Dictionary mapping parameter names to their values + """ + pairs: dict[str, str] = {} + for match in _HEADER_PAIRS_PATTERN.finditer(header): + key = match.group(1) + quoted_val, unquoted_val = match.group(2), match.group(3) + if quoted_val is None and unquoted_val is None: + # Bare token with no "=value": an auth-scheme name, not a parameter. + # Skip a leading scheme; once parameters exist, a new scheme marks + # the start of the next challenge, so stop here. + if pairs: + break + continue + pairs[key] = ( + unescape_quotes(quoted_val) if quoted_val is not None else unquoted_val + ) + return pairs + + +class DigestAuthMiddleware: + """ + HTTP digest authentication middleware for aiohttp client. + + This middleware intercepts 401 Unauthorized responses containing a Digest + authentication challenge, calculates the appropriate digest credentials, + and automatically retries the request with the proper Authorization header. + + Features: + - Handles all aspects of Digest authentication handshake automatically + - Supports all standard hash algorithms: + - MD5, MD5-SESS + - SHA, SHA-SESS + - SHA256, SHA256-SESS, SHA-256, SHA-256-SESS + - SHA512, SHA512-SESS, SHA-512, SHA-512-SESS + - Supports 'auth' and 'auth-int' quality of protection modes + - Properly handles quoted strings and parameter parsing + - Includes replay attack protection with client nonce count tracking + - Supports preemptive authentication per RFC 7616 Section 3.6 + + Origin scoping: + The credentials are scoped to the origin of the first request the + middleware handles. A request to a different origin is passed through + untouched, so it never receives a digest response computed from those + credentials, unless that origin falls within a protection space the + anchor origin advertised through the RFC 7616 ``domain`` directive. Make + the first request through the middleware against the intended origin, as + the anchor is pinned to it and not reset for the life of the instance. + + Standards compliance: + - RFC 7616: HTTP Digest Access Authentication (primary reference) + - RFC 2617: HTTP Authentication (deprecated by RFC 7616) + - RFC 1945: Section 11.1 (username restrictions) + + Implementation notes: + The core digest calculation is inspired by the implementation in + https://github.com/requests/requests/blob/v2.18.4/requests/auth.py + with added support for modern digest auth features and error handling. + """ + + def __init__( + self, + login: str, + password: str, + preemptive: bool = True, + ) -> None: + if login is None: + raise ValueError("None is not allowed as login value") + + if password is None: + raise ValueError("None is not allowed as password value") + + if ":" in login: + raise ValueError('A ":" is not allowed in username (RFC 1945#section-11.1)') + + self._login_str: Final[str] = login + self._login_bytes: Final[bytes] = login.encode("utf-8") + self._password_bytes: Final[bytes] = password.encode("utf-8") + + self._last_nonce_bytes = b"" + self._nonce_count = 0 + self._challenge: DigestAuthChallenge = {} + self._preemptive: bool = preemptive + # Set of URLs defining the protection space + self._protection_space: list[str] = [] + # Origin the credentials are scoped to; set on the first request. + self._origin: URL | None = None + + async def _encode(self, method: str, url: URL, body: Payload | Literal[b""]) -> str: + """ + Build digest authorization header for the current challenge. + + Args: + method: The HTTP method (GET, POST, etc.) + url: The request URL + body: The request body (used for qop=auth-int) + + Returns: + A fully formatted Digest authorization header string + + Raises: + ClientError: If the challenge is missing required parameters or + contains unsupported values + + """ + challenge = self._challenge + if "realm" not in challenge: + raise ClientError( + "Malformed Digest auth challenge: Missing 'realm' parameter" + ) + + if "nonce" not in challenge: + raise ClientError( + "Malformed Digest auth challenge: Missing 'nonce' parameter" + ) + + # Empty realm values are allowed per RFC 7616 (SHOULD, not MUST, contain host name) + realm = challenge["realm"] + nonce = challenge["nonce"] + + # Empty nonce values are not allowed as they are security-critical for replay protection + if not nonce: + raise ClientError( + "Security issue: Digest auth challenge contains empty 'nonce' value" + ) + + qop_raw = challenge.get("qop", "") + # Preserve original algorithm case for response while using uppercase for processing + algorithm_original = challenge.get("algorithm", "MD5") + algorithm = algorithm_original.upper() + opaque = challenge.get("opaque", "") + + # Convert string values to bytes once + nonce_bytes = nonce.encode("utf-8") + realm_bytes = realm.encode("utf-8") + # Use the encoded request-target (raw_path_qs) since that is what is + # transmitted on the wire and what the server signs against. Using the + # decoded form would cause digest verification to fail when the path + # or query string contains percent-encoded reserved characters. + path = URL(url).raw_path_qs + + # Process QoP + qop = "" + qop_bytes = b"" + if qop_raw: + valid_qops = {"auth", "auth-int"}.intersection( + {q.strip() for q in qop_raw.split(",") if q.strip()} + ) + if not valid_qops: + raise ClientError( + f"Digest auth error: Unsupported Quality of Protection (qop) value(s): {qop_raw}" + ) + + qop = "auth-int" if "auth-int" in valid_qops else "auth" + qop_bytes = qop.encode("utf-8") + + if algorithm not in DigestFunctions: + raise ClientError( + f"Digest auth error: Unsupported hash algorithm: {algorithm}. " + f"Supported algorithms: {', '.join(SUPPORTED_ALGORITHMS)}" + ) + hash_fn: Final = DigestFunctions[algorithm] + + def H(x: bytes) -> bytes: + """RFC 7616 Section 3: Hash function H(data) = hex(hash(data)).""" + return hash_fn(x).hexdigest().encode() + + def KD(s: bytes, d: bytes) -> bytes: + """RFC 7616 Section 3: KD(secret, data) = H(concat(secret, ":", data)).""" + return H(b":".join((s, d))) + + # Calculate A1 and A2 + A1 = b":".join((self._login_bytes, realm_bytes, self._password_bytes)) + A2 = f"{method.upper()}:{path}".encode() + if qop == "auth-int": + if isinstance(body, Payload): # will always be empty bytes unless Payload + entity_bytes = await body.as_bytes() # Get bytes from Payload + else: + entity_bytes = body + entity_hash = H(entity_bytes) + A2 = b":".join((A2, entity_hash)) + + HA1 = H(A1) + HA2 = H(A2) + + # Nonce count handling + if nonce_bytes == self._last_nonce_bytes: + self._nonce_count += 1 + else: + self._nonce_count = 1 + + self._last_nonce_bytes = nonce_bytes + ncvalue = f"{self._nonce_count:08x}" + ncvalue_bytes = ncvalue.encode("utf-8") + + # Generate client nonce + cnonce = hashlib.sha1( + b"".join( + [ + str(self._nonce_count).encode("utf-8"), + nonce_bytes, + time.ctime().encode("utf-8"), + os.urandom(8), + ] + ) + ).hexdigest()[:16] + cnonce_bytes = cnonce.encode("utf-8") + + # Special handling for session-based algorithms + if algorithm.upper().endswith("-SESS"): + HA1 = H(b":".join((HA1, nonce_bytes, cnonce_bytes))) + + # Calculate the response digest + if qop: + noncebit = b":".join( + (nonce_bytes, ncvalue_bytes, cnonce_bytes, qop_bytes, HA2) + ) + response_digest = KD(HA1, noncebit) + else: + response_digest = KD(HA1, b":".join((nonce_bytes, HA2))) + + # Define a dict mapping of header fields to their values + # Group fields into always-present, optional, and qop-dependent + header_fields = { + # Always present fields + "username": escape_quotes(self._login_str), + "realm": escape_quotes(realm), + "nonce": escape_quotes(nonce), + "uri": path, + "response": response_digest.decode(), + "algorithm": algorithm_original, + } + + # Optional fields + if opaque: + header_fields["opaque"] = escape_quotes(opaque) + + # QoP-dependent fields + if qop: + header_fields["qop"] = qop + header_fields["nc"] = ncvalue + header_fields["cnonce"] = cnonce + + # Build header using templates for each field type + pairs: list[str] = [] + for field, value in header_fields.items(): + if field in QUOTED_AUTH_FIELDS: + pairs.append(f'{field}="{value}"') + else: + pairs.append(f"{field}={value}") + + return f"Digest {', '.join(pairs)}" + + def _in_protection_space(self, url: URL) -> bool: + """ + Check if the given URL is within the current protection space. + + According to RFC 7616, a URI is in the protection space if any URI + in the protection space is a prefix of it (after both have been made absolute). + """ + request_str = str(url) + for space_str in self._protection_space: + # Check if request starts with space URL + if not request_str.startswith(space_str): + continue + # Exact match or space ends with / (proper directory prefix) + if len(request_str) == len(space_str) or space_str[-1] == "/": + return True + # Check next char is / to ensure proper path boundary + if request_str[len(space_str)] == "/": + return True + return False + + def _authenticate(self, response: ClientResponse) -> bool: + """ + Takes the given response and tries digest-auth, if needed. + + Returns true if the original request must be resent. + """ + if response.status != 401: + return False + + auth_header = response.headers.get("www-authenticate", "") + if not auth_header: + return False # No authentication header present + + method, sep, headers = auth_header.partition(" ") + if not sep: + # No space found in www-authenticate header + return False # Malformed auth header, missing scheme separator + + if method.lower() != "digest": + # Not a digest auth challenge (could be Basic, Bearer, etc.) + return False + + if not headers: + # We have a digest scheme but no parameters + return False # Malformed digest header, missing parameters + + # We have a digest auth header with content + if not (header_pairs := parse_header_pairs(headers)): + # Failed to parse any key-value pairs + return False # Malformed digest header, no valid parameters + + # Extract challenge parameters + self._challenge = {} + for field in CHALLENGE_FIELDS: + if (value := header_pairs.get(field)) is not None: + self._challenge[field] = value + + # Update protection space based on domain parameter or default to origin + origin = response.url.origin() + self._protection_space = [] + + if domain := self._challenge.get("domain"): + # Parse space-separated list of URIs + for uri in domain.split(): + # Remove quotes if present + uri = uri.strip('"') + if not uri: + continue + if uri.startswith("/"): + # Path-absolute, relative to origin + self._protection_space.append(str(origin.join(URL(uri)))) + else: + # Absolute URI + self._protection_space.append(str(URL(uri))) + + if not self._protection_space: + self._protection_space = [str(origin)] + + # Return True only if we found at least one challenge parameter + return bool(self._challenge) + + async def __call__( + self, request: ClientRequest, handler: ClientHandlerType + ) -> ClientResponse: + """Run the digest auth middleware.""" + # Credentials are scoped to the first request's origin. Other origins + # pass through untouched unless a challenge from the anchor origin + # advertised them via RFC 7616 domain; mirrors aiohttp stripping + # Authorization on cross-origin redirects. + origin = request.url.origin() + if self._origin is None: + self._origin = origin + elif origin != self._origin and not self._in_protection_space(request.url): + return await handler(request) + + response = None + for retry_count in range(2): + # Apply authorization header if: + # 1. This is a retry after 401 (retry_count > 0), OR + # 2. Preemptive auth is enabled AND we have a challenge AND the URL is in protection space + if retry_count > 0 or ( + self._preemptive + and self._challenge + and self._in_protection_space(request.url) + ): + request.headers[hdrs.AUTHORIZATION] = await self._encode( + request.method, request.url, request.body + ) + + # Send the request + response = await handler(request) + + # Check if we need to authenticate + if not self._authenticate(response): + break + + # At this point, response is guaranteed to be defined + assert response is not None + return response diff --git a/py_modules/aiohttp/client_middlewares.py b/py_modules/aiohttp/client_middlewares.py new file mode 100644 index 0000000..3ca2cb2 --- /dev/null +++ b/py_modules/aiohttp/client_middlewares.py @@ -0,0 +1,55 @@ +"""Client middleware support.""" + +from collections.abc import Awaitable, Callable, Sequence + +from .client_reqrep import ClientRequest, ClientResponse + +__all__ = ("ClientMiddlewareType", "ClientHandlerType", "build_client_middlewares") + +# Type alias for client request handlers - functions that process requests and return responses +ClientHandlerType = Callable[[ClientRequest], Awaitable[ClientResponse]] + +# Type for client middleware - similar to server but uses ClientRequest/ClientResponse +ClientMiddlewareType = Callable[ + [ClientRequest, ClientHandlerType], Awaitable[ClientResponse] +] + + +def build_client_middlewares( + handler: ClientHandlerType, + middlewares: Sequence[ClientMiddlewareType], +) -> ClientHandlerType: + """ + Apply middlewares to request handler. + + The middlewares are applied in reverse order, so the first middleware + in the list wraps all subsequent middlewares and the handler. + + This implementation avoids using partial/update_wrapper to minimize overhead + and doesn't cache to avoid holding references to stateful middleware. + """ + # Optimize for single middleware case + if len(middlewares) == 1: + middleware = middlewares[0] + + async def single_middleware_handler(req: ClientRequest) -> ClientResponse: + return await middleware(req, handler) + + return single_middleware_handler + + # Build the chain for multiple middlewares + current_handler = handler + + for middleware in reversed(middlewares): + # Create a new closure that captures the current state + def make_wrapper( + mw: ClientMiddlewareType, next_h: ClientHandlerType + ) -> ClientHandlerType: + async def wrapped(req: ClientRequest) -> ClientResponse: + return await mw(req, next_h) + + return wrapped + + current_handler = make_wrapper(middleware, current_handler) + + return current_handler diff --git a/py_modules/aiohttp/client_proto.py b/py_modules/aiohttp/client_proto.py new file mode 100644 index 0000000..2bac26b --- /dev/null +++ b/py_modules/aiohttp/client_proto.py @@ -0,0 +1,372 @@ +import asyncio +from contextlib import suppress +from typing import Any, Callable + +from .base_protocol import BaseProtocol +from .client_exceptions import ( + ClientConnectionError, + ClientOSError, + ClientPayloadError, + ServerDisconnectedError, + SocketTimeoutError, +) +from .helpers import ( + _EXC_SENTINEL, + DEFAULT_CHUNK_SIZE, + EMPTY_BODY_STATUS_CODES, + BaseTimerContext, + set_exception, + set_result, +) +from .http import HttpResponseParser, RawResponseMessage +from .http_exceptions import HttpProcessingError +from .streams import EMPTY_PAYLOAD, DataQueue, StreamReader + + +class ResponseHandler(BaseProtocol, DataQueue[tuple[RawResponseMessage, StreamReader]]): + """Helper class to adapt between Protocol and StreamReader.""" + + def __init__(self, loop: asyncio.AbstractEventLoop) -> None: + BaseProtocol.__init__(self, loop=loop, parser=None) + DataQueue.__init__(self, loop) + + self._should_close = False + + self._payload: StreamReader | None = None + self._skip_payload = False + self._payload_parser = None + self._data_received_cb: Callable[[], None] | None = None + + self._timer = None + self._tail = b"" + + self._read_timeout: float | None = None + self._read_timeout_handle: asyncio.TimerHandle | None = None + + self._timeout_ceil_threshold: float | None = 5 + + self._closed: None | asyncio.Future[None] = None + self._connection_lost_called = False + + @property + def closed(self) -> None | asyncio.Future[None]: + """Future that is set when the connection is closed. + + This property returns a Future that will be completed when the connection + is closed. The Future is created lazily on first access to avoid creating + futures that will never be awaited. + + Returns: + - A Future[None] if the connection is still open or was closed after + this property was accessed + - None if connection_lost() was already called before this property + was ever accessed (indicating no one is waiting for the closure) + """ + if self._closed is None and not self._connection_lost_called: + self._closed = self._loop.create_future() + return self._closed + + @property + def upgraded(self) -> bool: + return self._upgraded + + @property + def should_close(self) -> bool: + return bool( + self._should_close + or (self._payload is not None and not self._payload.is_eof()) + or self._upgraded + or self._exception is not None + or self._payload_parser is not None + or self._buffer + or self._tail + ) + + def force_close(self) -> None: + self._should_close = True + + def close(self) -> None: + self._exception = None # Break cyclic references + transport = self.transport + if transport is not None: + transport.close() + self.transport = None + self._payload = None + self._drop_timeout() + + def abort(self) -> None: + self._exception = None # Break cyclic references + transport = self.transport + if transport is not None: + transport.abort() + self.transport = None + self._payload = None + self._drop_timeout() + + def is_connected(self) -> bool: + return self.transport is not None and not self.transport.is_closing() + + def connection_lost(self, exc: BaseException | None) -> None: + self._connection_lost_called = True + self._drop_timeout() + + original_connection_error = exc + reraised_exc = original_connection_error + + connection_closed_cleanly = original_connection_error is None + + if self._closed is not None: + # If someone is waiting for the closed future, + # we should set it to None or an exception. If + # self._closed is None, it means that + # connection_lost() was called already + # or nobody is waiting for it. + if connection_closed_cleanly: + set_result(self._closed, None) + else: + assert original_connection_error is not None + set_exception( + self._closed, + ClientConnectionError( + f"Connection lost: {original_connection_error !s}", + ), + original_connection_error, + ) + + if self._payload_parser is not None: + with suppress(Exception): # FIXME: log this somehow? + self._payload_parser.feed_eof() + + uncompleted = None + if self._parser is not None: + try: + uncompleted = self._parser.feed_eof() + except Exception as underlying_exc: + if self._payload is not None: + client_payload_exc_msg = ( + f"Response payload is not completed: {underlying_exc !r}" + ) + if not connection_closed_cleanly: + client_payload_exc_msg = ( + f"{client_payload_exc_msg !s}. " + f"{original_connection_error !r}" + ) + set_exception( + self._payload, + ClientPayloadError(client_payload_exc_msg), + underlying_exc, + ) + + if not self.is_eof(): + if isinstance(original_connection_error, OSError): + reraised_exc = ClientOSError(*original_connection_error.args) + if connection_closed_cleanly: + reraised_exc = ServerDisconnectedError(uncompleted) + # assigns self._should_close to True as side effect, + # we do it anyway below + underlying_non_eof_exc = ( + _EXC_SENTINEL + if connection_closed_cleanly + else original_connection_error + ) + assert underlying_non_eof_exc is not None + assert reraised_exc is not None + self.set_exception(reraised_exc, underlying_non_eof_exc) + + self._should_close = True + self._parser = None + self._payload = None + self._payload_parser = None + self._reading_paused = False + + super().connection_lost(reraised_exc) + + def eof_received(self) -> None: + # should call parser.feed_eof() most likely + self._drop_timeout() + + def pause_reading(self) -> None: + super().pause_reading() + self._drop_timeout() + + def resume_reading(self, resume_parser: bool = True) -> None: + was_paused = self._reading_paused + super().resume_reading(resume_parser) + if was_paused: + self._reschedule_timeout() + + def set_exception( + self, + exc: BaseException, + exc_cause: BaseException = _EXC_SENTINEL, + ) -> None: + self._should_close = True + self._drop_timeout() + super().set_exception(exc, exc_cause) + + def set_parser( + self, + parser: Any, + payload: Any, + data_received_cb: Callable[[], None] | None = None, + ) -> None: + # TODO: actual types are: + # parser: WebSocketReader + # payload: WebSocketDataQueue + # but they are not generi enough + # Need an ABC for both types + self._payload = payload + self._payload_parser = parser + self._data_received_cb = data_received_cb + + self._drop_timeout() + + if self._tail: + data, self._tail = self._tail, b"" + self.data_received(data) + + def set_response_params( + self, + *, + timer: BaseTimerContext | None = None, + skip_payload: bool = False, + read_until_eof: bool = False, + auto_decompress: bool = True, + read_timeout: float | None = None, + read_bufsize: int = DEFAULT_CHUNK_SIZE, + timeout_ceil_threshold: float = 5, + max_line_size: int = 8190, + max_field_size: int = 8190, + max_headers: int = 128, + ) -> None: + self._skip_payload = skip_payload + + self._read_timeout = read_timeout + + self._timeout_ceil_threshold = timeout_ceil_threshold + + self._parser = HttpResponseParser( + self, + self._loop, + read_bufsize, + timer=timer, + payload_exception=ClientPayloadError, + response_with_body=not skip_payload, + read_until_eof=read_until_eof, + auto_decompress=auto_decompress, + max_line_size=max_line_size, + max_field_size=max_field_size, + max_headers=max_headers, + ) + + if self._tail: + data, self._tail = self._tail, b"" + self.data_received(data) + + def _drop_timeout(self) -> None: + if self._read_timeout_handle is not None: + self._read_timeout_handle.cancel() + self._read_timeout_handle = None + + def _reschedule_timeout(self) -> None: + timeout = self._read_timeout + if self._read_timeout_handle is not None: + self._read_timeout_handle.cancel() + + if timeout: + self._read_timeout_handle = self._loop.call_later( + timeout, self._on_read_timeout + ) + else: + self._read_timeout_handle = None + + def start_timeout(self) -> None: + self._reschedule_timeout() + + @property + def read_timeout(self) -> float | None: + return self._read_timeout + + @read_timeout.setter + def read_timeout(self, read_timeout: float | None) -> None: + self._read_timeout = read_timeout + + def _on_read_timeout(self) -> None: + exc = SocketTimeoutError("Timeout on reading data from socket") + self.set_exception(exc) + if self._payload is not None: + set_exception(self._payload, exc) + + def data_received(self, data: bytes) -> None: + # If no data, then we are resuming decompression. We haven't received + # data from the socket, so we can avoid the reschedule overhead. + if data: + self._reschedule_timeout() + + # custom payload parser - currently always WebSocketReader + if self._payload_parser is not None: + if self._data_received_cb is not None: + self._data_received_cb() + eof, tail = self._payload_parser.feed_data(data) + if eof: + self._payload = None + self._payload_parser = None + + if tail: + self.data_received(tail) + return + + if self._upgraded or self._parser is None: + # i.e. websocket connection, websocket parser is not set yet + self._tail += data + return + + # parse http messages + try: + messages, upgraded, tail = self._parser.feed_data(data) + except BaseException as underlying_exc: + if self.transport is not None: + # connection.release() could be called BEFORE + # data_received(), the transport is already + # closed in this case + self.transport.close() + if not isinstance(underlying_exc, Exception): + raise + # should_close is True after the call + if isinstance(underlying_exc, HttpProcessingError): + exc = HttpProcessingError( + code=underlying_exc.code, + message=underlying_exc.message, + headers=underlying_exc.headers, + ) + else: + exc = HttpProcessingError() + self.set_exception(exc, underlying_exc) + return + + self._upgraded = upgraded + + payload: StreamReader | None = None + for message, payload in messages: + if message.should_close: + self._should_close = True + + self._payload = payload + + if self._skip_payload or message.code in EMPTY_BODY_STATUS_CODES: + self.feed_data((message, EMPTY_PAYLOAD), 0) + else: + self.feed_data((message, payload), 0) + + if payload is not None: + # new message(s) was processed + # register timeout handler unsubscribing + # either on end-of-stream or immediately for + # EMPTY_PAYLOAD + if payload is not EMPTY_PAYLOAD: + payload.on_eof(self._drop_timeout) + else: + self._drop_timeout() + + if upgraded and tail: + self.data_received(tail) diff --git a/py_modules/aiohttp/client_reqrep.py b/py_modules/aiohttp/client_reqrep.py new file mode 100644 index 0000000..c77213e --- /dev/null +++ b/py_modules/aiohttp/client_reqrep.py @@ -0,0 +1,1557 @@ +import asyncio +import codecs +import contextlib +import functools +import io +import re +import sys +import traceback +import warnings +from collections.abc import Callable, Iterable, Mapping +from hashlib import md5, sha1, sha256 +from http.cookies import Morsel, SimpleCookie +from types import MappingProxyType, TracebackType +from typing import TYPE_CHECKING, Any, Literal, NamedTuple, Optional, Union + +import attr +from multidict import CIMultiDict, CIMultiDictProxy, MultiDict, MultiDictProxy +from yarl import URL + +from . import hdrs, helpers, http, multipart, payload +from ._cookie_helpers import ( + parse_cookie_header, + parse_set_cookie_headers, + preserve_morsel_with_coded_value, +) +from .abc import AbstractStreamWriter +from .client_exceptions import ( + ClientConnectionError, + ClientOSError, + ClientResponseError, + ContentTypeError, + InvalidURL, + ServerFingerprintMismatch, +) +from .compression_utils import HAS_BROTLI, HAS_ZSTD +from .formdata import FormData +from .helpers import ( + _SENTINEL, + BaseTimerContext, + BasicAuth, + HeadersMixin, + TimerNoop, + _basic_auth_no_warn, + noop, + reify, + sentinel, + set_exception, + set_result, +) +from .http import ( + SERVER_SOFTWARE, + HttpVersion, + HttpVersion10, + HttpVersion11, + StreamWriter, +) +from .streams import StreamReader +from .typedefs import ( + DEFAULT_JSON_DECODER, + JSONDecoder, + LooseCookies, + LooseHeaders, + Query, + RawHeaders, +) + +if TYPE_CHECKING: + import ssl + from ssl import SSLContext +else: + try: + import ssl + from ssl import SSLContext + except ImportError: # pragma: no cover + ssl = None # type: ignore[assignment] + SSLContext = object # type: ignore[misc,assignment] + + +__all__ = ("ClientRequest", "ClientResponse", "RequestInfo", "Fingerprint") + + +if TYPE_CHECKING: + from .client import ClientSession + from .connector import Connection + from .tracing import Trace + + +_CONNECTION_CLOSED_EXCEPTION = ClientConnectionError("Connection closed") +_CONTAINS_CONTROL_CHAR_RE = re.compile(r"[^-!#$%&'*+.^_`|~0-9a-zA-Z]") +json_re = re.compile(r"^application/(?:[\w.+-]+?\+)?json") +_DIGITS_RE = re.compile(r"\d+", re.ASCII) + + +def _gen_default_accept_encoding() -> str: + encodings = [ + "gzip", + "deflate", + ] + if HAS_BROTLI: + encodings.append("br") + if HAS_ZSTD: + encodings.append("zstd") + return ", ".join(encodings) + + +@attr.s(auto_attribs=True, frozen=True, slots=True) +class ContentDisposition: + type: str | None + parameters: "MappingProxyType[str, str]" + filename: str | None + + +class _RequestInfo(NamedTuple): + url: URL + method: str + headers: "CIMultiDictProxy[str]" + real_url: URL + + +class RequestInfo(_RequestInfo): + + def __new__( + cls, + url: URL, + method: str, + headers: "CIMultiDictProxy[str]", + real_url: URL | _SENTINEL = sentinel, + ) -> "RequestInfo": + """Create a new RequestInfo instance. + + For backwards compatibility, the real_url parameter is optional. + """ + return tuple.__new__( + cls, (url, method, headers, url if real_url is sentinel else real_url) + ) + + +class Fingerprint: + HASHFUNC_BY_DIGESTLEN = { + 16: md5, + 20: sha1, + 32: sha256, + } + + def __init__(self, fingerprint: bytes) -> None: + digestlen = len(fingerprint) + hashfunc = self.HASHFUNC_BY_DIGESTLEN.get(digestlen) + if not hashfunc: + raise ValueError("fingerprint has invalid length") + elif hashfunc is md5 or hashfunc is sha1: + raise ValueError("md5 and sha1 are insecure and not supported. Use sha256.") + self._hashfunc = hashfunc + self._fingerprint = fingerprint + + @property + def fingerprint(self) -> bytes: + return self._fingerprint + + def check(self, transport: asyncio.Transport) -> None: + if not transport.get_extra_info("sslcontext"): + return + sslobj = transport.get_extra_info("ssl_object") + cert = sslobj.getpeercert(binary_form=True) + got = self._hashfunc(cert).digest() + if got != self._fingerprint: + host, port, *_ = transport.get_extra_info("peername") + raise ServerFingerprintMismatch(self._fingerprint, got, host, port) + + +if ssl is not None: + SSL_ALLOWED_TYPES = (ssl.SSLContext, bool, Fingerprint, type(None)) +else: # pragma: no cover + SSL_ALLOWED_TYPES = (bool, type(None)) + + +def _merge_ssl_params( + ssl: Union["SSLContext", bool, Fingerprint], + verify_ssl: bool | None, + ssl_context: Optional["SSLContext"], + fingerprint: bytes | None, +) -> Union["SSLContext", bool, Fingerprint]: + if ssl is None: + ssl = True # Double check for backwards compatibility + if verify_ssl is not None and not verify_ssl: + warnings.warn( + "verify_ssl is deprecated, use ssl=False instead", + DeprecationWarning, + stacklevel=3, + ) + if ssl is not True: + raise ValueError( + "verify_ssl, ssl_context, fingerprint and ssl " + "parameters are mutually exclusive" + ) + else: + ssl = False + if ssl_context is not None: + warnings.warn( + "ssl_context is deprecated, use ssl=context instead", + DeprecationWarning, + stacklevel=3, + ) + if ssl is not True: + raise ValueError( + "verify_ssl, ssl_context, fingerprint and ssl " + "parameters are mutually exclusive" + ) + else: + ssl = ssl_context + if fingerprint is not None: + warnings.warn( + "fingerprint is deprecated, use ssl=Fingerprint(fingerprint) instead", + DeprecationWarning, + stacklevel=3, + ) + if ssl is not True: + raise ValueError( + "verify_ssl, ssl_context, fingerprint and ssl " + "parameters are mutually exclusive" + ) + else: + ssl = Fingerprint(fingerprint) + if not isinstance(ssl, SSL_ALLOWED_TYPES): + raise TypeError( + "ssl should be SSLContext, bool, Fingerprint or None, " + f"got {ssl!r} instead." + ) + return ssl + + +_SSL_SCHEMES = frozenset(("https", "wss")) + + +# ConnectionKey is a NamedTuple because it is used as a key in a dict +# and a set in the connector. Since a NamedTuple is a tuple it uses +# the fast native tuple __hash__ and __eq__ implementation in CPython. +class ConnectionKey(NamedTuple): + # the key should contain an information about used proxy / TLS + # to prevent reusing wrong connections from a pool + host: str + port: int | None + is_ssl: bool + ssl: SSLContext | bool | Fingerprint + proxy: URL | None + proxy_auth: BasicAuth | None + proxy_headers_hash: int | None # hash(CIMultiDict) + server_hostname: str | None = None + + +def _is_expected_content_type( + response_content_type: str, expected_content_type: str +) -> bool: + if expected_content_type == "application/json": + return json_re.match(response_content_type) is not None + return expected_content_type in response_content_type + + +def _warn_if_unclosed_payload(payload: payload.Payload, stacklevel: int = 2) -> None: + """Warn if the payload is not closed. + + Callers must check that the body is a Payload before calling this method. + + Args: + payload: The payload to check + stacklevel: Stack level for the warning (default 2 for direct callers) + """ + if not payload.autoclose and not payload.consumed: + warnings.warn( + "The previous request body contains unclosed resources. " + "Use await request.update_body() instead of setting request.body " + "directly to properly close resources and avoid leaks.", + ResourceWarning, + stacklevel=stacklevel, + ) + + +class ClientResponse(HeadersMixin): + + # Some of these attributes are None when created, + # but will be set by the start() method. + # As the end user will likely never see the None values, we cheat the types below. + # from the Status-Line of the response + version: HttpVersion | None = None # HTTP-Version + status: int = None # type: ignore[assignment] # Status-Code + reason: str | None = None # Reason-Phrase + + content: StreamReader = None # type: ignore[assignment] # Payload stream + _body: bytes | None = None + _headers: CIMultiDictProxy[str] = None # type: ignore[assignment] + _history: tuple["ClientResponse", ...] = () + _raw_headers: RawHeaders = None # type: ignore[assignment] + + _connection: Optional["Connection"] = None # current connection + _cookies: SimpleCookie | None = None + _raw_cookie_headers: tuple[str, ...] | None = None + _continue: Optional["asyncio.Future[bool]"] = None + _source_traceback: traceback.StackSummary | None = None + _session: Optional["ClientSession"] = None + # set up by ClientRequest after ClientResponse object creation + # post-init stage allows to not change ctor signature + _closed = True # to allow __del__ for non-initialized properly response + _released = False + _in_context = False + + _resolve_charset: Callable[["ClientResponse", bytes], str] = lambda *_: "utf-8" + + __writer: Optional["asyncio.Task[None]"] = None + _stream_writer: Optional[AbstractStreamWriter] = None + _output_size: int = 0 + _upload_complete: Optional[asyncio.Future[None]] = None + + def __init__( + self, + method: str, + url: URL, + *, + writer: "asyncio.Task[None] | None", + continue100: Optional["asyncio.Future[bool]"], + timer: BaseTimerContext, + request_info: RequestInfo, + traces: list["Trace"], + loop: asyncio.AbstractEventLoop, + session: "ClientSession", + stream_writer: AbstractStreamWriter, + ) -> None: + # URL forbids subclasses, so a simple type check is enough. + assert type(url) is URL + + self.method = method + + self._real_url = url + self._url = url.with_fragment(None) if url.raw_fragment else url + if writer is None: # Request already sent + self._output_size = stream_writer.output_size + else: + self._stream_writer = stream_writer + self._writer = writer + if continue100 is not None: + self._continue = continue100 + self._request_info = request_info + self._timer = timer if timer is not None else TimerNoop() + self._cache: dict[str, Any] = {} + self._traces = traces + self._loop = loop + # Save reference to _resolve_charset, so that get_encoding() will still + # work after the response has finished reading the body. + # TODO: Fix session=None in tests (see ClientRequest.__init__). + if session is not None: + # store a reference to session #1985 + self._session = session + self._resolve_charset = session._resolve_charset + if loop.get_debug(): + self._source_traceback = traceback.extract_stack(sys._getframe(1)) + + def __reset_writer(self, _: object = None) -> None: + self.__writer = None + if self._stream_writer is not None: + self._output_size = self._stream_writer.output_size + self._stream_writer = None + if self._upload_complete is not None and not self._upload_complete.done(): + self._upload_complete.set_result(None) + + @property + def _writer(self) -> Optional["asyncio.Task[None]"]: + """The writer task for streaming data. + + _writer is only provided for backwards compatibility + for subclasses that may need to access it. + """ + return self.__writer + + @_writer.setter + def _writer(self, writer: Optional["asyncio.Task[None]"]) -> None: + """Set the writer task for streaming data.""" + if self.__writer is not None: + self.__writer.remove_done_callback(self.__reset_writer) + self.__writer = writer + if writer is None: + return + if writer.done(): + # The writer is already done, so we can clear it immediately. + self.__reset_writer() + else: + writer.add_done_callback(self.__reset_writer) + + @property + def output_size(self) -> int: + """Number of bytes sent for this request.""" + if self._stream_writer is not None: + return self._stream_writer.output_size + return self._output_size + + @property + def upload_complete(self) -> "asyncio.Future[None]": + """Future set when the request body has been fully sent. + + Already done when the request had no body or was written eagerly. + """ + if self._upload_complete is None: + self._upload_complete = self._loop.create_future() + if self._stream_writer is None: # upload already finished + self._upload_complete.set_result(None) + return self._upload_complete + + @property + def cookies(self) -> SimpleCookie: + if self._cookies is None: + if self._raw_cookie_headers is not None: + # Parse cookies for response.cookies (SimpleCookie for backward compatibility) + cookies = SimpleCookie() + # Use parse_set_cookie_headers for more lenient parsing that handles + # malformed cookies better than SimpleCookie.load + cookies.update(parse_set_cookie_headers(self._raw_cookie_headers)) + self._cookies = cookies + else: + self._cookies = SimpleCookie() + return self._cookies + + @cookies.setter + def cookies(self, cookies: SimpleCookie) -> None: + self._cookies = cookies + # Generate raw cookie headers from the SimpleCookie + if cookies: + self._raw_cookie_headers = tuple( + morsel.OutputString() for morsel in cookies.values() + ) + else: + self._raw_cookie_headers = None + + @reify + def url(self) -> URL: + return self._url + + @reify + def url_obj(self) -> URL: + warnings.warn("Deprecated, use .url #1654", DeprecationWarning, stacklevel=2) + return self._url + + @reify + def real_url(self) -> URL: + return self._real_url + + @reify + def host(self) -> str: + assert self._url.host is not None + return self._url.host + + @reify + def headers(self) -> "CIMultiDictProxy[str]": + return self._headers + + @reify + def raw_headers(self) -> RawHeaders: + return self._raw_headers + + @reify + def request_info(self) -> RequestInfo: + return self._request_info + + @reify + def content_disposition(self) -> ContentDisposition | None: + raw = self._headers.get(hdrs.CONTENT_DISPOSITION) + if raw is None: + return None + disposition_type, params_dct = multipart.parse_content_disposition(raw) + params = MappingProxyType(params_dct) + filename = multipart.content_disposition_filename(params) + return ContentDisposition(disposition_type, params, filename) + + def __del__(self, _warnings: Any = warnings) -> None: + if self._closed: + return + + if self._connection is not None: + self._connection.release() + self._cleanup_writer() + + if self._loop.get_debug(): + kwargs = {"source": self} + _warnings.warn(f"Unclosed response {self!r}", ResourceWarning, **kwargs) + context = {"client_response": self, "message": "Unclosed response"} + if self._source_traceback: + context["source_traceback"] = self._source_traceback + self._loop.call_exception_handler(context) + + def __repr__(self) -> str: + out = io.StringIO() + ascii_encodable_url = str(self.url) + if self.reason: + ascii_encodable_reason = self.reason.encode( + "ascii", "backslashreplace" + ).decode("ascii") + else: + ascii_encodable_reason = "None" + print( + f"", + file=out, + ) + print(self.headers, file=out) + return out.getvalue() + + @property + def connection(self) -> Optional["Connection"]: + return self._connection + + @reify + def history(self) -> tuple["ClientResponse", ...]: + """A sequence of of responses, if redirects occurred.""" + return self._history + + @reify + def links(self) -> "MultiDictProxy[MultiDictProxy[str | URL]]": + links_str = ", ".join(self.headers.getall("link", [])) + + if not links_str: + return MultiDictProxy(MultiDict()) + + links: MultiDict[MultiDictProxy[str | URL]] = MultiDict() + + for val in re.split(r",(?=\s*<)", links_str): + match = re.match(r"\s*<(.*)>(.*)", val) + if match is None: # pragma: no cover + # the check exists to suppress mypy error + continue + url, params_str = match.groups() + params = params_str.split(";")[1:] + + link: MultiDict[str | URL] = MultiDict() + + for param in params: + match = re.match(r"^\s*(\S*)\s*=\s*(['\"]?)(.*?)(\2)\s*$", param, re.M) + if match is None: # pragma: no cover + # the check exists to suppress mypy error + continue + key, _, value, _ = match.groups() + + link.add(key, value) + + key = link.get("rel", url) + + link.add("url", self.url.join(URL(url))) + + links.add(str(key), MultiDictProxy(link)) + + return MultiDictProxy(links) + + async def start(self, connection: "Connection") -> "ClientResponse": + """Start response processing.""" + self._closed = False + self._protocol = connection.protocol + self._connection = connection + + with self._timer: + while True: + # read response + try: + protocol = self._protocol + message, payload = await protocol.read() # type: ignore[union-attr] + except http.HttpProcessingError as exc: + raise ClientResponseError( + self.request_info, + self.history, + status=exc.code, + message=exc.message, + headers=exc.headers, + ) from exc + + if message.code < 100 or message.code > 199 or message.code == 101: + break + + if self._continue is not None: + set_result(self._continue, True) + self._continue = None + + # payload eof handler + payload.on_eof(self._response_eof) + + # response status + self.version = message.version + self.status = message.code + self.reason = message.reason + + # headers + self._headers = message.headers # type is CIMultiDictProxy + self._raw_headers = message.raw_headers # type is Tuple[bytes, bytes] + + # payload + self.content = payload + + # cookies + if cookie_hdrs := self.headers.getall(hdrs.SET_COOKIE, ()): + # Store raw cookie headers for CookieJar + self._raw_cookie_headers = tuple(cookie_hdrs) + return self + + def _response_eof(self) -> None: + if self._closed: + return + + # protocol could be None because connection could be detached + protocol = self._connection and self._connection.protocol + if protocol is not None and protocol.upgraded: + return + + self._closed = True + self._cleanup_writer() + self._release_connection() + + @property + def closed(self) -> bool: + return self._closed + + def close(self) -> None: + if not self._released: + self._notify_content() + + self._closed = True + if self._loop is None or self._loop.is_closed(): + return + + self._cleanup_writer() + if self._connection is not None: + self._connection.close() + self._connection = None + + def release(self) -> Any: + if not self._released: + self._notify_content() + + self._closed = True + + self._cleanup_writer() + self._release_connection() + return noop() + + @property + def ok(self) -> bool: + """Returns ``True`` if ``status`` is less than ``400``, ``False`` if not. + + This is **not** a check for ``200 OK`` but a check that the response + status is under 400. + """ + return 400 > self.status + + def raise_for_status(self) -> None: + if not self.ok: + # reason should always be not None for a started response + assert self.reason is not None + + # If we're in a context we can rely on __aexit__() to release as the + # exception propagates. + if not self._in_context: + self.release() + + raise ClientResponseError( + self.request_info, + self.history, + status=self.status, + message=self.reason, + headers=self.headers, + ) + + def _release_connection(self) -> None: + if self._connection is not None: + if self.__writer is None: + self._connection.release() + self._connection = None + else: + self.__writer.add_done_callback(lambda f: self._release_connection()) + + async def _wait_released(self) -> None: + if self.__writer is not None: + try: + await self.__writer + except asyncio.CancelledError: + if ( + sys.version_info >= (3, 11) + and (task := asyncio.current_task()) + and task.cancelling() + ): + raise + self._release_connection() + + def _cleanup_writer(self) -> None: + if self.__writer is not None: + self.__writer.cancel() + if self._stream_writer is not None: + self._output_size = self._stream_writer.output_size + self._stream_writer = None + self._session = None + + def _notify_content(self) -> None: + content = self.content + if content and content.exception() is None: + set_exception(content, _CONNECTION_CLOSED_EXCEPTION) + self._released = True + + async def wait_for_close(self) -> None: + if self.__writer is not None: + try: + await self.__writer + except asyncio.CancelledError: + if ( + sys.version_info >= (3, 11) + and (task := asyncio.current_task()) + and task.cancelling() + ): + raise + self.release() + + async def read(self) -> bytes: + """Read response payload.""" + if self._body is None: + try: + self._body = await self.content.read() + for trace in self._traces: + await trace.send_response_chunk_received( + self.method, self.url, self._body + ) + except BaseException: + self.close() + raise + elif self._released: # Response explicitly released + raise ClientConnectionError("Connection closed") + + protocol = self._connection and self._connection.protocol + if protocol is None or not protocol.upgraded: + await self._wait_released() # Underlying connection released + return self._body + + def get_encoding(self) -> str: + ctype = self.headers.get(hdrs.CONTENT_TYPE, "").lower() + mimetype = helpers.parse_mimetype(ctype) + + encoding = mimetype.parameters.get("charset") + if encoding: + with contextlib.suppress(LookupError, ValueError): + return codecs.lookup(encoding).name + + if mimetype.type == "application" and ( + mimetype.subtype == "json" or mimetype.subtype == "rdap" + ): + # RFC 7159 states that the default encoding is UTF-8. + # RFC 7483 defines application/rdap+json + return "utf-8" + + if self._body is None: + raise RuntimeError( + "Cannot compute fallback encoding of a not yet read body" + ) + + return self._resolve_charset(self, self._body) + + async def text(self, encoding: str | None = None, errors: str = "strict") -> str: + """Read response payload and decode.""" + if self._body is None: + await self.read() + + if encoding is None: + encoding = self.get_encoding() + + return self._body.decode(encoding, errors=errors) # type: ignore[union-attr] + + async def json( + self, + *, + encoding: str | None = None, + loads: JSONDecoder = DEFAULT_JSON_DECODER, + content_type: str | None = "application/json", + ) -> Any: + """Read and decodes JSON response.""" + if self._body is None: + await self.read() + + if content_type: + ctype = self.headers.get(hdrs.CONTENT_TYPE, "").lower() + if not _is_expected_content_type(ctype, content_type): + raise ContentTypeError( + self.request_info, + self.history, + status=self.status, + message=( + "Attempt to decode JSON with unexpected mimetype: %s" % ctype + ), + headers=self.headers, + ) + + stripped = self._body.strip() # type: ignore[union-attr] + if not stripped: + return None + + if encoding is None: + encoding = self.get_encoding() + + return loads(stripped.decode(encoding)) + + async def __aenter__(self) -> "ClientResponse": + self._in_context = True + return self + + async def __aexit__( + self, + exc_type: type[BaseException] | None, + exc_val: BaseException | None, + exc_tb: TracebackType | None, + ) -> None: + self._in_context = False + # similar to _RequestContextManager, we do not need to check + # for exceptions, response object can close connection + # if state is broken + self.release() + await self.wait_for_close() + + +class ClientRequest: + GET_METHODS = { + hdrs.METH_GET, + hdrs.METH_HEAD, + hdrs.METH_OPTIONS, + hdrs.METH_TRACE, + } + POST_METHODS = {hdrs.METH_PATCH, hdrs.METH_POST, hdrs.METH_PUT} + ALL_METHODS = GET_METHODS.union(POST_METHODS).union({hdrs.METH_DELETE}) + + DEFAULT_HEADERS = { + hdrs.ACCEPT: "*/*", + hdrs.ACCEPT_ENCODING: _gen_default_accept_encoding(), + } + + # Type of body depends on PAYLOAD_REGISTRY, which is dynamic. + _body: None | payload.Payload = None + auth = None + response = None + + __writer: Optional["asyncio.Task[None]"] = None # async task for streaming data + + # These class defaults help create_autospec() work correctly. + # If autospec is improved in future, maybe these can be removed. + url = URL() + method = "GET" + + _continue = None # waiter future for '100 Continue' response + + _skip_auto_headers: Optional["CIMultiDict[None]"] = None + + # N.B. + # Adding __del__ method with self._writer closing doesn't make sense + # because _writer is instance method, thus it keeps a reference to self. + # Until writer has finished finalizer will not be called. + + def __init__( + self, + method: str, + url: URL, + *, + params: Query = None, + headers: LooseHeaders | None = None, + skip_auto_headers: Iterable[str] | None = None, + data: Any = None, + cookies: LooseCookies | None = None, + auth: BasicAuth | None = None, + version: http.HttpVersion = http.HttpVersion11, + compress: str | bool | None = None, + chunked: bool | None = None, + expect100: bool = False, + loop: asyncio.AbstractEventLoop | None = None, + response_class: type["ClientResponse"] | None = None, + proxy: URL | None = None, + proxy_auth: BasicAuth | None = None, + timer: BaseTimerContext | None = None, + session: Optional["ClientSession"] = None, + ssl: SSLContext | bool | Fingerprint = True, + proxy_headers: LooseHeaders | None = None, + traces: list["Trace"] | None = None, + trust_env: bool = False, + server_hostname: str | None = None, + ): + if loop is None: + loop = asyncio.get_event_loop() + if match := _CONTAINS_CONTROL_CHAR_RE.search(method): + raise ValueError( + f"Method cannot contain non-token characters {method!r} " + f"(found at least {match.group()!r})" + ) + # URL forbids subclasses, so a simple type check is enough. + assert type(url) is URL, url + if proxy is not None: + assert type(proxy) is URL, proxy + # FIXME: session is None in tests only, need to fix tests + # assert session is not None + if TYPE_CHECKING: + assert session is not None + self._session = session + if params: + url = url.extend_query(params) + self.original_url = url + self.url = url.with_fragment(None) if url.raw_fragment else url + self.method = method.upper() + self.chunked = chunked + self.compress = compress + self.loop = loop + self.length = None + if response_class is None: + real_response_class = ClientResponse + else: + real_response_class = response_class + self.response_class: type[ClientResponse] = real_response_class + self._timer = timer if timer is not None else TimerNoop() + self._ssl = ssl if ssl is not None else True + self.server_hostname = server_hostname + + if loop.get_debug(): + self._source_traceback = traceback.extract_stack(sys._getframe(1)) + + self.update_version(version) + self.update_host(url) + self.update_headers(headers) + self.update_auto_headers(skip_auto_headers) + self.update_cookies(cookies) + self.update_content_encoding(data) + self.update_auth(auth, trust_env) + self.update_proxy(proxy, proxy_auth, proxy_headers) + + self.update_body_from_data(data) + if data is not None or self.method not in self.GET_METHODS: + self.update_transfer_encoding() + self.update_expect_continue(expect100) + self._traces = [] if traces is None else traces + + def __reset_writer(self, _: object = None) -> None: + self.__writer = None + + def _get_content_length(self) -> int | None: + """Extract and validate Content-Length header value. + + Returns parsed Content-Length value or None if not set. + Raises ValueError if header exists but cannot be parsed as an integer. + """ + if hdrs.CONTENT_LENGTH not in self.headers: + return None + + content_length_hdr = self.headers[hdrs.CONTENT_LENGTH] + if not _DIGITS_RE.fullmatch(content_length_hdr): + raise ValueError(f"Invalid Content-Length header: {content_length_hdr!r}") + return int(content_length_hdr) + + @property + def skip_auto_headers(self) -> CIMultiDict[None]: + return self._skip_auto_headers or CIMultiDict() + + @property + def _writer(self) -> Optional["asyncio.Task[None]"]: + return self.__writer + + @_writer.setter + def _writer(self, writer: "asyncio.Task[None]") -> None: + if self.__writer is not None: + self.__writer.remove_done_callback(self.__reset_writer) + self.__writer = writer + writer.add_done_callback(self.__reset_writer) + + def is_ssl(self) -> bool: + return self.url.scheme in _SSL_SCHEMES + + @property + def ssl(self) -> Union["SSLContext", bool, Fingerprint]: + return self._ssl + + @property + def connection_key(self) -> ConnectionKey: + if proxy_headers := self.proxy_headers: + h: int | None = hash(tuple(proxy_headers.items())) + else: + h = None + url = self.url + return tuple.__new__( + ConnectionKey, + ( + url.raw_host or "", + url.port, + url.scheme in _SSL_SCHEMES, + self._ssl, + self.proxy, + self.proxy_auth, + h, + self.server_hostname, + ), + ) + + @property + def host(self) -> str: + ret = self.url.raw_host + assert ret is not None + return ret + + @property + def port(self) -> int | None: + return self.url.port + + @property + def body(self) -> payload.Payload | Literal[b""]: + """Request body.""" + # empty body is represented as bytes for backwards compatibility + return self._body or b"" + + @body.setter + def body(self, value: Any) -> None: + """Set request body with warning for non-autoclose payloads. + + WARNING: This setter must be called from within an event loop and is not + thread-safe. Setting body outside of an event loop may raise RuntimeError + when closing file-based payloads. + + DEPRECATED: Direct assignment to body is deprecated and will be removed + in a future version. Use await update_body() instead for proper resource + management. + """ + # Close existing payload if present + if self._body is not None: + # Warn if the payload needs manual closing + # stacklevel=3: user code -> body setter -> _warn_if_unclosed_payload + _warn_if_unclosed_payload(self._body, stacklevel=3) + # NOTE: In the future, when we remove sync close support, + # this setter will need to be removed and only the async + # update_body() method will be available. For now, we call + # _close() for backwards compatibility. + self._body._close() + self._update_body(value) + + @property + def request_info(self) -> RequestInfo: + headers: CIMultiDictProxy[str] = CIMultiDictProxy(self.headers) + # These are created on every request, so we use a NamedTuple + # for performance reasons. We don't use the RequestInfo.__new__ + # method because it has a different signature which is provided + # for backwards compatibility only. + return tuple.__new__( + RequestInfo, (self.url, self.method, headers, self.original_url) + ) + + @property + def session(self) -> "ClientSession": + """Return the ClientSession instance. + + This property provides access to the ClientSession that initiated + this request, allowing middleware to make additional requests + using the same session. + """ + return self._session + + def update_host(self, url: URL) -> None: + """Update destination host, port and connection type (ssl).""" + # get host/port + if not url.raw_host: + raise InvalidURL(url) + + # basic auth info + if url.raw_user or url.raw_password: + self.auth = _basic_auth_no_warn(url.user or "", url.password or "") + + def update_version(self, version: http.HttpVersion | str) -> None: + """Convert request version to two elements tuple. + + parser HTTP version '1.1' => (1, 1) + """ + if isinstance(version, str): + v = [part.strip() for part in version.split(".", 1)] + try: + version = http.HttpVersion(int(v[0]), int(v[1])) + except ValueError: + raise ValueError( + f"Can not parse http version number: {version}" + ) from None + self.version = version + + def update_headers(self, headers: LooseHeaders | None) -> None: + """Update request headers.""" + self.headers: CIMultiDict[str] = CIMultiDict() + + # Build the host header + host = self.url.host_port_subcomponent + + # host_port_subcomponent is None when the URL is a relative URL. + # but we know we do not have a relative URL here. + assert host is not None + self.headers[hdrs.HOST] = host + + if not headers: + return + + if isinstance(headers, (dict, MultiDictProxy, MultiDict)): + headers = headers.items() + + for key, value in headers: # type: ignore[str-unpack] + # A special case for Host header + if key in hdrs.HOST_ALL: + self.headers[key] = value + else: + self.headers.add(key, value) + + def update_auto_headers(self, skip_auto_headers: Iterable[str] | None) -> None: + if skip_auto_headers is not None: + self._skip_auto_headers = CIMultiDict( + (hdr, None) for hdr in sorted(skip_auto_headers) + ) + used_headers = self.headers.copy() + used_headers.extend(self._skip_auto_headers) # type: ignore[arg-type] + else: + # Fast path when there are no headers to skip + # which is the most common case. + used_headers = self.headers + + for hdr, val in self.DEFAULT_HEADERS.items(): + if hdr not in used_headers: + self.headers[hdr] = val + + if hdrs.USER_AGENT not in used_headers: + self.headers[hdrs.USER_AGENT] = SERVER_SOFTWARE + + def update_cookies(self, cookies: LooseCookies | None) -> None: + """Update request cookies header.""" + if not cookies: + return + + c = SimpleCookie() + if hdrs.COOKIE in self.headers: + # parse_cookie_header for RFC 6265 compliant Cookie header parsing + c.update(parse_cookie_header(self.headers.get(hdrs.COOKIE, ""))) + del self.headers[hdrs.COOKIE] + + if isinstance(cookies, Mapping): + iter_cookies = cookies.items() + else: + iter_cookies = cookies # type: ignore[assignment] + for name, value in iter_cookies: + if isinstance(value, Morsel): + # Use helper to preserve coded_value exactly as sent by server + c[name] = preserve_morsel_with_coded_value(value) + else: + c[name] = value # type: ignore[assignment] + + self.headers[hdrs.COOKIE] = c.output(header="", sep=";").strip() + + def update_content_encoding(self, data: Any) -> None: + """Set request content encoding.""" + if not data: + # Don't compress an empty body. + self.compress = None + return + + if self.headers.get(hdrs.CONTENT_ENCODING): + if self.compress: + raise ValueError( + "compress can not be set if Content-Encoding header is set" + ) + elif self.compress: + if not isinstance(self.compress, str): + self.compress = "deflate" + self.headers[hdrs.CONTENT_ENCODING] = self.compress + self.chunked = True # enable chunked, no need to deal with length + + def update_transfer_encoding(self) -> None: + """Analyze transfer-encoding header.""" + te = self.headers.get(hdrs.TRANSFER_ENCODING, "").lower() + + if "chunked" in te: + if self.chunked: + raise ValueError( + "chunked can not be set " + 'if "Transfer-Encoding: chunked" header is set' + ) + + elif self.chunked: + if hdrs.CONTENT_LENGTH in self.headers: + raise ValueError( + "chunked can not be set if Content-Length header is set" + ) + + self.headers[hdrs.TRANSFER_ENCODING] = "chunked" + + def update_auth(self, auth: BasicAuth | None, trust_env: bool = False) -> None: + """Set basic auth.""" + if auth is None: + auth = self.auth + if auth is None: + return + + if not isinstance(auth, helpers.BasicAuth): + raise TypeError("BasicAuth() tuple is required instead") + + self.headers[hdrs.AUTHORIZATION] = auth.encode() + + def update_body_from_data(self, body: Any, _stacklevel: int = 3) -> None: + """Update request body from data.""" + if self._body is not None: + _warn_if_unclosed_payload(self._body, stacklevel=_stacklevel) + + if body is None: + self._body = None + # Set Content-Length to 0 when body is None for methods that expect a body + if ( + self.method not in self.GET_METHODS + and not self.chunked + and hdrs.CONTENT_LENGTH not in self.headers + ): + self.headers[hdrs.CONTENT_LENGTH] = "0" + return + + # FormData + maybe_payload = body() if isinstance(body, FormData) else body + + try: + body_payload = payload.PAYLOAD_REGISTRY.get(maybe_payload, disposition=None) + except payload.LookupError: + body_payload = FormData(maybe_payload)() # type: ignore[arg-type] + + self._body = body_payload + # enable chunked encoding if needed + if not self.chunked and hdrs.CONTENT_LENGTH not in self.headers: + if (size := body_payload.size) is not None: + self.headers[hdrs.CONTENT_LENGTH] = str(size) + else: + self.chunked = True + + # copy payload headers + assert body_payload.headers + headers = self.headers + skip_headers = self._skip_auto_headers + for key, value in body_payload.headers.items(): + if key in headers or (skip_headers is not None and key in skip_headers): + continue + headers[key] = value + + def _update_body(self, body: Any) -> None: + """Update request body after its already been set.""" + # Remove existing Content-Length header since body is changing + if hdrs.CONTENT_LENGTH in self.headers: + del self.headers[hdrs.CONTENT_LENGTH] + + # Remove existing Transfer-Encoding header to avoid conflicts + if self.chunked and hdrs.TRANSFER_ENCODING in self.headers: + del self.headers[hdrs.TRANSFER_ENCODING] + + # Now update the body using the existing method + # Called from _update_body, add 1 to stacklevel from caller + self.update_body_from_data(body, _stacklevel=4) + + # Update transfer encoding headers if needed (same logic as __init__) + if body is not None or self.method not in self.GET_METHODS: + self.update_transfer_encoding() + + async def update_body(self, body: Any) -> None: + """ + Update request body and close previous payload if needed. + + This method safely updates the request body by first closing any existing + payload to prevent resource leaks, then setting the new body. + + IMPORTANT: Always use this method instead of setting request.body directly. + Direct assignment to request.body will leak resources if the previous body + contains file handles, streams, or other resources that need cleanup. + + Args: + body: The new body content. Can be: + - bytes/bytearray: Raw binary data + - str: Text data (will be encoded using charset from Content-Type) + - FormData: Form data that will be encoded as multipart/form-data + - Payload: A pre-configured payload object + - AsyncIterable: An async iterable of bytes chunks + - File-like object: Will be read and sent as binary data + - None: Clears the body + + Usage: + # CORRECT: Use update_body + await request.update_body(b"new request data") + + # WRONG: Don't set body directly + # request.body = b"new request data" # This will leak resources! + + # Update with form data + form_data = FormData() + form_data.add_field('field', 'value') + await request.update_body(form_data) + + # Clear body + await request.update_body(None) + + Note: + This method is async because it may need to close file handles or + other resources associated with the previous payload. Always await + this method to ensure proper cleanup. + + Warning: + Setting request.body directly is highly discouraged and can lead to: + - Resource leaks (unclosed file handles, streams) + - Memory leaks (unreleased buffers) + - Unexpected behavior with streaming payloads + + It is not recommended to change the payload type in middleware. If the + body was already set (e.g., as bytes), it's best to keep the same type + rather than converting it (e.g., to str) as this may result in unexpected + behavior. + + See Also: + - update_body_from_data: Synchronous body update without cleanup + - body property: Direct body access (STRONGLY DISCOURAGED) + + """ + # Close existing payload if it exists and needs closing + if self._body is not None: + await self._body.close() + self._update_body(body) + + def update_expect_continue(self, expect: bool = False) -> None: + if expect: + self.headers[hdrs.EXPECT] = "100-continue" + elif ( + hdrs.EXPECT in self.headers + and self.headers[hdrs.EXPECT].lower() == "100-continue" + ): + expect = True + + if expect: + self._continue = self.loop.create_future() + + def update_proxy( + self, + proxy: URL | None, + proxy_auth: BasicAuth | None, + proxy_headers: LooseHeaders | None, + ) -> None: + self.proxy = proxy + if proxy is None: + self.proxy_auth = None + self.proxy_headers = None + return + + if proxy_auth and not isinstance(proxy_auth, helpers.BasicAuth): + raise ValueError("proxy_auth must be None or BasicAuth() tuple") + self.proxy_auth = proxy_auth + + if proxy_headers is not None and not isinstance( + proxy_headers, (MultiDict, MultiDictProxy) + ): + proxy_headers = CIMultiDict(proxy_headers) + self.proxy_headers = proxy_headers + + async def write_bytes( + self, + writer: AbstractStreamWriter, + conn: "Connection", + content_length: int | None = None, + ) -> None: + """ + Write the request body to the connection stream. + + This method handles writing different types of request bodies: + 1. Payload objects (using their specialized write_with_length method) + 2. Bytes/bytearray objects + 3. Iterable body content + + Args: + writer: The stream writer to write the body to + conn: The connection being used for this request + content_length: Optional maximum number of bytes to write from the body + (None means write the entire body) + + The method properly handles: + - Waiting for 100-Continue responses if required + - Content length constraints for chunked encoding + - Error handling for network issues, cancellation, and other exceptions + - Signaling EOF and timeout management + + Raises: + ClientOSError: When there's an OS-level error writing the body + ClientConnectionError: When there's a general connection error + asyncio.CancelledError: When the operation is cancelled + + """ + # 100 response + if self._continue is not None: + # Force headers to be sent before waiting for 100-continue + writer.send_headers() + await writer.drain() + await self._continue + + protocol = conn.protocol + assert protocol is not None + try: + # This should be a rare case but the + # self._body can be set to None while + # the task is being started or we wait above + # for the 100-continue response. + # The more likely case is we have an empty + # payload, but 100-continue is still expected. + if self._body is not None: + await self._body.write_with_length(writer, content_length) + except OSError as underlying_exc: + reraised_exc = underlying_exc + + # Distinguish between timeout and other OS errors for better error reporting + exc_is_not_timeout = underlying_exc.errno is not None or not isinstance( + underlying_exc, asyncio.TimeoutError + ) + if exc_is_not_timeout: + reraised_exc = ClientOSError( + underlying_exc.errno, + f"Can not write request body for {self.url !s}", + ) + + set_exception(protocol, reraised_exc, underlying_exc) + except asyncio.CancelledError: + # Body hasn't been fully sent, so connection can't be reused + conn.close() + raise + except Exception as underlying_exc: + set_exception( + protocol, + ClientConnectionError( + "Failed to send bytes into the underlying connection " + f"{conn !s}: {underlying_exc!r}", + ), + underlying_exc, + ) + else: + # Successfully wrote the body, signal EOF and start response timeout + await writer.write_eof() + protocol.start_timeout() + + async def send(self, conn: "Connection") -> "ClientResponse": + # Specify request target: + # - CONNECT request must send authority form URI + # - not CONNECT proxy must send absolute form URI + # - most common is origin form URI + if self.method == hdrs.METH_CONNECT: + connect_host = self.url.host_subcomponent + assert connect_host is not None + path = f"{connect_host}:{self.url.port}" + elif self.proxy and not self.is_ssl(): + path = str(self.url) + else: + path = self.url.raw_path_qs + + protocol = conn.protocol + assert protocol is not None + writer = StreamWriter( + protocol, + self.loop, + on_chunk_sent=( + functools.partial(self._on_chunk_request_sent, self.method, self.url) + if self._traces + else None + ), + on_headers_sent=( + functools.partial(self._on_headers_request_sent, self.method, self.url) + if self._traces + else None + ), + ) + + if self.compress: + writer.enable_compression(self.compress) # type: ignore[arg-type] + + if self.chunked is not None: + writer.enable_chunking() + + # set default content-type + if ( + self.method in self.POST_METHODS + and ( + self._skip_auto_headers is None + or hdrs.CONTENT_TYPE not in self._skip_auto_headers + ) + and hdrs.CONTENT_TYPE not in self.headers + ): + self.headers[hdrs.CONTENT_TYPE] = "application/octet-stream" + + v = self.version + if hdrs.CONNECTION not in self.headers: + if conn._connector.force_close: + if v == HttpVersion11: + self.headers[hdrs.CONNECTION] = "close" + elif v == HttpVersion10: + self.headers[hdrs.CONNECTION] = "keep-alive" + + # status + headers + status_line = f"{self.method} {path} HTTP/{v.major}.{v.minor}" + + # Buffer headers for potential coalescing with body + await writer.write_headers(status_line, self.headers) + + task: asyncio.Task[None] | None + if self._body or self._continue is not None or protocol.writing_paused: + coro = self.write_bytes(writer, conn, self._get_content_length()) + if sys.version_info >= (3, 12): + # Optimization for Python 3.12, try to write + # bytes immediately to avoid having to schedule + # the task on the event loop. + task = asyncio.Task(coro, loop=self.loop, eager_start=True) + else: + task = self.loop.create_task(coro) + if task.done(): + task = None + else: + self._writer = task + else: + # We have nothing to write because + # - there is no body + # - the protocol does not have writing paused + # - we are not waiting for a 100-continue response + protocol.start_timeout() + writer.set_eof() + task = None + response_class = self.response_class + assert response_class is not None + self.response = response_class( + self.method, + self.original_url, + writer=task, + continue100=self._continue, + timer=self._timer, + request_info=self.request_info, + traces=self._traces, + loop=self.loop, + session=self._session, + stream_writer=writer, + ) + return self.response + + async def close(self) -> None: + if self.__writer is not None: + try: + await self.__writer + except asyncio.CancelledError: + if ( + sys.version_info >= (3, 11) + and (task := asyncio.current_task()) + and task.cancelling() + ): + raise + + def terminate(self) -> None: + if self.__writer is not None: + if not self.loop.is_closed(): + self.__writer.cancel() + self.__writer.remove_done_callback(self.__reset_writer) + self.__writer = None + + async def _on_chunk_request_sent(self, method: str, url: URL, chunk: bytes) -> None: + for trace in self._traces: + await trace.send_request_chunk_sent(method, url, chunk) + + async def _on_headers_request_sent( + self, method: str, url: URL, headers: "CIMultiDict[str]" + ) -> None: + for trace in self._traces: + await trace.send_request_headers(method, url, headers) diff --git a/py_modules/aiohttp/client_ws.py b/py_modules/aiohttp/client_ws.py new file mode 100644 index 0000000..02f560f --- /dev/null +++ b/py_modules/aiohttp/client_ws.py @@ -0,0 +1,560 @@ +"""WebSocket client for asyncio.""" + +import asyncio +import sys +from collections.abc import Callable +from types import TracebackType +from typing import Any, Generic, Literal, Optional, cast, overload + +import attr + +from ._websocket.reader import WebSocketDataQueue +from .client_exceptions import ClientError, ServerTimeoutError, WSMessageTypeError +from .client_reqrep import ClientResponse +from .helpers import calculate_timeout_when, set_result +from .http import ( + WS_CLOSED_MESSAGE, + WS_CLOSING_MESSAGE, + WebSocketError, + WSCloseCode, + WSMessage, + WSMessageDecodeText, + WSMessageNoDecodeText, + WSMsgType, +) +from .http_websocket import _INTERNAL_RECEIVE_TYPES, WebSocketWriter +from .streams import EofStream +from .typedefs import ( + DEFAULT_JSON_DECODER, + DEFAULT_JSON_ENCODER, + JSONBytesEncoder, + JSONDecoder, + JSONEncoder, +) + +if sys.version_info >= (3, 13): + from typing import TypeVar +else: + from typing_extensions import TypeVar + +if sys.version_info >= (3, 11): + import asyncio as async_timeout + from typing import Self +else: + import async_timeout + from typing_extensions import Self + +# TypeVar for whether text messages are decoded to str (True) or kept as bytes (False) +# Covariant because it only affects return types, not input types +_DecodeText = TypeVar("_DecodeText", bound=bool, covariant=True, default=Literal[True]) + + +@attr.s(frozen=True, slots=True) +class ClientWSTimeout: + ws_receive = attr.ib(type=Optional[float], default=None) + ws_close = attr.ib(type=Optional[float], default=None) + + +DEFAULT_WS_CLIENT_TIMEOUT = ClientWSTimeout(ws_receive=None, ws_close=10.0) + + +class ClientWebSocketResponse(Generic[_DecodeText]): + def __init__( + self, + reader: WebSocketDataQueue, + writer: WebSocketWriter, + protocol: str | None, + response: ClientResponse, + timeout: ClientWSTimeout, + autoclose: bool, + autoping: bool, + loop: asyncio.AbstractEventLoop, + *, + heartbeat: float | None = None, + compress: int = 0, + client_notakeover: bool = False, + ) -> None: + self._response = response + self._conn = response.connection + + self._writer = writer + self._reader = reader + self._protocol = protocol + self._closed = False + self._closing = False + self._close_code: int | None = None + self._timeout = timeout + self._autoclose = autoclose + self._autoping = autoping + self._heartbeat = heartbeat + self._heartbeat_cb: asyncio.TimerHandle | None = None + self._heartbeat_when: float = 0.0 + if heartbeat is not None: + self._pong_heartbeat = heartbeat / 2.0 + self._pong_response_cb: asyncio.TimerHandle | None = None + self._loop = loop + self._waiting: bool = False + self._close_wait: asyncio.Future[None] | None = None + self._exception: BaseException | None = None + self._compress = compress + self._client_notakeover = client_notakeover + self._ping_task: asyncio.Task[None] | None = None + self._need_heartbeat_reset = False + self._heartbeat_reset_handle: asyncio.Handle | None = None + + self._reset_heartbeat() + + def _cancel_heartbeat(self) -> None: + self._cancel_pong_response_cb() + if self._heartbeat_reset_handle is not None: + self._heartbeat_reset_handle.cancel() + self._heartbeat_reset_handle = None + self._need_heartbeat_reset = False + if self._heartbeat_cb is not None: + self._heartbeat_cb.cancel() + self._heartbeat_cb = None + if self._ping_task is not None: + self._ping_task.cancel() + self._ping_task = None + + def _cancel_pong_response_cb(self) -> None: + if self._pong_response_cb is not None: + self._pong_response_cb.cancel() + self._pong_response_cb = None + + def _on_data_received(self) -> None: + if self._heartbeat is None or self._need_heartbeat_reset: + return + loop = self._loop + assert loop is not None + # Coalesce multiple chunks received in the same loop tick into a single + # heartbeat reset. Resetting immediately per chunk increases timer churn. + self._need_heartbeat_reset = True + self._heartbeat_reset_handle = loop.call_soon(self._flush_heartbeat_reset) + + def _flush_heartbeat_reset(self) -> None: + self._heartbeat_reset_handle = None + if not self._need_heartbeat_reset: + return + self._reset_heartbeat() + self._need_heartbeat_reset = False + + def _reset_heartbeat(self) -> None: + if self._heartbeat is None: + return + self._cancel_pong_response_cb() + loop = self._loop + assert loop is not None + conn = self._conn + timeout_ceil_threshold = ( + conn._connector._timeout_ceil_threshold if conn is not None else 5 + ) + now = loop.time() + when = calculate_timeout_when(now, self._heartbeat, timeout_ceil_threshold) + self._heartbeat_when = when + if self._heartbeat_cb is None: + # We do not cancel the previous heartbeat_cb here because + # it generates a significant amount of TimerHandle churn + # which causes asyncio to rebuild the heap frequently. + # Instead _send_heartbeat() will reschedule the next + # heartbeat if it fires too early. + self._heartbeat_cb = loop.call_at(when, self._send_heartbeat) + + def _send_heartbeat(self) -> None: + self._heartbeat_cb = None + + # If heartbeat reset is pending (data is being received), skip sending + # the ping and let the reset callback handle rescheduling the heartbeat. + if self._need_heartbeat_reset: + return + + loop = self._loop + now = loop.time() + if now < self._heartbeat_when: + # Heartbeat fired too early, reschedule + self._heartbeat_cb = loop.call_at( + self._heartbeat_when, self._send_heartbeat + ) + return + + conn = self._conn + timeout_ceil_threshold = ( + conn._connector._timeout_ceil_threshold if conn is not None else 5 + ) + when = calculate_timeout_when(now, self._pong_heartbeat, timeout_ceil_threshold) + self._cancel_pong_response_cb() + self._pong_response_cb = loop.call_at(when, self._pong_not_received) + + coro = self._writer.send_frame(b"", WSMsgType.PING) + if sys.version_info >= (3, 12): + # Optimization for Python 3.12, try to send the ping + # immediately to avoid having to schedule + # the task on the event loop. + ping_task = asyncio.Task(coro, loop=loop, eager_start=True) + else: + ping_task = loop.create_task(coro) + + if not ping_task.done(): + self._ping_task = ping_task + ping_task.add_done_callback(self._ping_task_done) + else: + self._ping_task_done(ping_task) + + def _ping_task_done(self, task: "asyncio.Task[None]") -> None: + """Callback for when the ping task completes.""" + if not task.cancelled() and (exc := task.exception()): + self._handle_ping_pong_exception(exc) + self._ping_task = None + + def _pong_not_received(self) -> None: + self._handle_ping_pong_exception( + ServerTimeoutError(f"No PONG received after {self._pong_heartbeat} seconds") + ) + + def _handle_ping_pong_exception(self, exc: BaseException) -> None: + """Handle exceptions raised during ping/pong processing.""" + if self._closed: + return + self._set_closed() + self._close_code = WSCloseCode.ABNORMAL_CLOSURE + self._exception = exc + self._response.close() + if self._waiting and not self._closing: + self._reader.feed_data(WSMessage(WSMsgType.ERROR, exc, None), 0) + + def _set_closed(self) -> None: + """Set the connection to closed. + + Cancel any heartbeat timers and set the closed flag. + """ + self._closed = True + self._cancel_heartbeat() + + def _set_closing(self) -> None: + """Set the connection to closing. + + Cancel any heartbeat timers and set the closing flag. + """ + self._closing = True + self._cancel_heartbeat() + + @property + def closed(self) -> bool: + return self._closed + + @property + def close_code(self) -> int | None: + return self._close_code + + @property + def protocol(self) -> str | None: + return self._protocol + + @property + def compress(self) -> int: + return self._compress + + @property + def client_notakeover(self) -> bool: + return self._client_notakeover + + def get_extra_info(self, name: str, default: Any = None) -> Any: + """extra info from connection transport""" + conn = self._response.connection + if conn is None: + return default + transport = conn.transport + if transport is None: + return default + return transport.get_extra_info(name, default) + + def exception(self) -> BaseException | None: + return self._exception + + async def ping(self, message: bytes = b"") -> None: + await self._writer.send_frame(message, WSMsgType.PING) + + async def pong(self, message: bytes = b"") -> None: + await self._writer.send_frame(message, WSMsgType.PONG) + + async def send_frame( + self, message: bytes, opcode: WSMsgType, compress: int | None = None + ) -> None: + """Send a frame over the websocket.""" + await self._writer.send_frame(message, opcode, compress) + + async def send_str(self, data: str, compress: int | None = None) -> None: + if not isinstance(data, str): + raise TypeError("data argument must be str (%r)" % type(data)) + await self._writer.send_frame( + data.encode("utf-8"), WSMsgType.TEXT, compress=compress + ) + + async def send_bytes(self, data: bytes, compress: int | None = None) -> None: + if not isinstance(data, (bytes, bytearray, memoryview)): + raise TypeError("data argument must be byte-ish (%r)" % type(data)) + await self._writer.send_frame(data, WSMsgType.BINARY, compress=compress) + + async def send_json( + self, + data: Any, + compress: int | None = None, + *, + dumps: JSONEncoder = DEFAULT_JSON_ENCODER, + ) -> None: + await self.send_str(dumps(data), compress=compress) + + async def send_json_bytes( + self, + data: Any, + compress: int | None = None, + *, + dumps: JSONBytesEncoder, + ) -> None: + """Send JSON data using a bytes-returning encoder as a binary frame. + + Use this when your JSON encoder (like orjson) returns bytes + instead of str, avoiding the encode/decode overhead. + """ + await self.send_bytes(dumps(data), compress=compress) + + async def close(self, *, code: int = WSCloseCode.OK, message: bytes = b"") -> bool: + # we need to break `receive()` cycle first, + # `close()` may be called from different task + if self._waiting and not self._closing: + assert self._loop is not None + self._close_wait = self._loop.create_future() + self._set_closing() + self._reader.feed_data(WS_CLOSING_MESSAGE, 0) + await self._close_wait + + if self._closed: + return False + + self._set_closed() + try: + await self._writer.close(code, message) + except asyncio.CancelledError: + self._close_code = WSCloseCode.ABNORMAL_CLOSURE + self._response.close() + raise + except Exception as exc: + self._close_code = WSCloseCode.ABNORMAL_CLOSURE + self._exception = exc + self._response.close() + return True + + if self._close_code: + self._response.close() + return True + + while True: + try: + async with async_timeout.timeout(self._timeout.ws_close): + msg = await self._reader.read() + except asyncio.CancelledError: + self._close_code = WSCloseCode.ABNORMAL_CLOSURE + self._response.close() + raise + except Exception as exc: + self._close_code = WSCloseCode.ABNORMAL_CLOSURE + self._exception = exc + self._response.close() + return True + + if msg.type is WSMsgType.CLOSE: + self._close_code = msg.data + self._response.close() + return True + + @overload + async def receive( + self: "ClientWebSocketResponse[Literal[True]]", timeout: float | None = None + ) -> WSMessageDecodeText: ... + + @overload + async def receive( + self: "ClientWebSocketResponse[Literal[False]]", timeout: float | None = None + ) -> WSMessageNoDecodeText: ... + + @overload + async def receive( + self: "ClientWebSocketResponse[_DecodeText]", timeout: float | None = None + ) -> WSMessageDecodeText | WSMessageNoDecodeText: ... + + async def receive( + self, timeout: float | None = None + ) -> WSMessageDecodeText | WSMessageNoDecodeText: + receive_timeout = timeout or self._timeout.ws_receive + + while True: + if self._waiting: + raise RuntimeError("Concurrent call to receive() is not allowed") + + if self._closed: + return WS_CLOSED_MESSAGE + elif self._closing: + await self.close() + return WS_CLOSED_MESSAGE + + try: + self._waiting = True + try: + if receive_timeout: + # Entering the context manager and creating + # Timeout() object can take almost 50% of the + # run time in this loop so we avoid it if + # there is no read timeout. + async with async_timeout.timeout(receive_timeout): + msg = await self._reader.read() + else: + msg = await self._reader.read() + finally: + self._waiting = False + if self._close_wait: + set_result(self._close_wait, None) + except (asyncio.CancelledError, asyncio.TimeoutError): + self._close_code = WSCloseCode.ABNORMAL_CLOSURE + raise + except EofStream: + self._close_code = WSCloseCode.OK + await self.close() + return WSMessage(WSMsgType.CLOSED, None, None) + except ClientError: + # Likely ServerDisconnectedError when connection is lost + self._set_closed() + self._close_code = WSCloseCode.ABNORMAL_CLOSURE + return WS_CLOSED_MESSAGE + except WebSocketError as exc: + self._close_code = exc.code + await self.close(code=exc.code) + return WSMessage(WSMsgType.ERROR, exc, None) + except Exception as exc: + self._exception = exc + self._set_closing() + self._close_code = WSCloseCode.ABNORMAL_CLOSURE + await self.close() + return WSMessage(WSMsgType.ERROR, exc, None) + + if msg.type not in _INTERNAL_RECEIVE_TYPES: + # If its not a close/closing/ping/pong message + # we can return it immediately + return msg + + if msg.type is WSMsgType.CLOSE: + self._set_closing() + self._close_code = msg.data + if not self._closed and self._autoclose: + await self.close() + elif msg.type is WSMsgType.CLOSING: + self._set_closing() + elif msg.type is WSMsgType.PING and self._autoping: + await self.pong(msg.data) + continue + elif msg.type is WSMsgType.PONG and self._autoping: + continue + + return msg + + @overload + async def receive_str( + self: "ClientWebSocketResponse[Literal[True]]", *, timeout: float | None = None + ) -> str: ... + + @overload + async def receive_str( + self: "ClientWebSocketResponse[Literal[False]]", *, timeout: float | None = None + ) -> bytes: ... + + @overload + async def receive_str( + self: "ClientWebSocketResponse[_DecodeText]", *, timeout: float | None = None + ) -> str | bytes: ... + + async def receive_str(self, *, timeout: float | None = None) -> str | bytes: + """Receive TEXT message. + + Returns str when decode_text=True (default), bytes when decode_text=False. + """ + msg = await self.receive(timeout) + if msg.type is not WSMsgType.TEXT: + raise WSMessageTypeError( + f"Received message {msg.type}:{msg.data!r} is not WSMsgType.TEXT" + ) + return cast(str, msg.data) + + async def receive_bytes(self, *, timeout: float | None = None) -> bytes: + msg = await self.receive(timeout) + if msg.type is not WSMsgType.BINARY: + raise WSMessageTypeError( + f"Received message {msg.type}:{msg.data!r} is not WSMsgType.BINARY" + ) + return cast(bytes, msg.data) + + @overload + async def receive_json( + self: "ClientWebSocketResponse[Literal[True]]", + *, + loads: JSONDecoder = ..., + timeout: float | None = None, + ) -> Any: ... + + @overload + async def receive_json( + self: "ClientWebSocketResponse[Literal[False]]", + *, + loads: Callable[[bytes], Any] = ..., + timeout: float | None = None, + ) -> Any: ... + + @overload + async def receive_json( + self: "ClientWebSocketResponse[_DecodeText]", + *, + loads: JSONDecoder | Callable[[bytes], Any] = ..., + timeout: float | None = None, + ) -> Any: ... + + async def receive_json( + self, + *, + loads: JSONDecoder | Callable[[bytes], Any] = DEFAULT_JSON_DECODER, + timeout: float | None = None, + ) -> Any: + data = await self.receive_str(timeout=timeout) + return loads(data) # type: ignore[arg-type] + + def __aiter__(self) -> Self: + return self + + @overload + async def __anext__( + self: "ClientWebSocketResponse[Literal[True]]", + ) -> WSMessageDecodeText: ... + + @overload + async def __anext__( + self: "ClientWebSocketResponse[Literal[False]]", + ) -> WSMessageNoDecodeText: ... + + @overload + async def __anext__( + self: "ClientWebSocketResponse[_DecodeText]", + ) -> WSMessageDecodeText | WSMessageNoDecodeText: ... + + async def __anext__(self) -> WSMessageDecodeText | WSMessageNoDecodeText: + msg = await self.receive() + if msg.type in (WSMsgType.CLOSE, WSMsgType.CLOSING, WSMsgType.CLOSED): + raise StopAsyncIteration + return msg + + async def __aenter__(self) -> Self: + return self + + async def __aexit__( + self, + exc_type: type[BaseException] | None, + exc_val: BaseException | None, + exc_tb: TracebackType | None, + ) -> None: + await self.close() diff --git a/py_modules/aiohttp/compression_utils.py b/py_modules/aiohttp/compression_utils.py new file mode 100644 index 0000000..9313d21 --- /dev/null +++ b/py_modules/aiohttp/compression_utils.py @@ -0,0 +1,447 @@ +import asyncio +import sys +import zlib +from abc import ABC, abstractmethod +from concurrent.futures import Executor +from typing import Any, Final, Protocol, TypedDict, cast + +if sys.version_info >= (3, 12): + from collections.abc import Buffer +else: + from typing import Union + + Buffer = Union[bytes, bytearray, "memoryview[int]", "memoryview[bytes]"] + +try: + try: + import brotlicffi as brotli + except ImportError: + import brotli + + HAS_BROTLI = True +except ImportError: # pragma: no cover + HAS_BROTLI = False + +try: + if sys.version_info >= (3, 14): + from compression.zstd import ZstdDecompressor # noqa: I900 + else: # TODO(PY314): Remove mentions of backports.zstd across codebase + from backports.zstd import ZstdDecompressor + + HAS_ZSTD = True +except ImportError: + HAS_ZSTD = False + + +MAX_SYNC_CHUNK_SIZE = 4096 + +# Unlimited decompression constants - different libraries use different conventions +ZLIB_MAX_LENGTH_UNLIMITED = 0 # zlib uses 0 to mean unlimited +ZSTD_MAX_LENGTH_UNLIMITED = -1 # zstd uses -1 to mean unlimited + + +class ZLibCompressObjProtocol(Protocol): + def compress(self, data: Buffer) -> bytes: ... + def flush(self, mode: int = ..., /) -> bytes: ... + + +class ZLibDecompressObjProtocol(Protocol): + def decompress(self, data: Buffer, max_length: int = ...) -> bytes: ... + def flush(self, length: int = ..., /) -> bytes: ... + + @property + def eof(self) -> bool: ... + + @property + def unconsumed_tail(self) -> bytes: ... + + @property + def unused_data(self) -> bytes: ... + + +class ZLibBackendProtocol(Protocol): + MAX_WBITS: int + Z_FULL_FLUSH: int + Z_SYNC_FLUSH: int + Z_BEST_SPEED: int + Z_FINISH: int + + def compressobj( + self, + level: int = ..., + method: int = ..., + wbits: int = ..., + memLevel: int = ..., + strategy: int = ..., + zdict: Buffer | None = ..., + ) -> ZLibCompressObjProtocol: ... + def decompressobj( + self, wbits: int = ..., zdict: Buffer = ... + ) -> ZLibDecompressObjProtocol: ... + + def compress( + self, data: Buffer, /, level: int = ..., wbits: int = ... + ) -> bytes: ... + def decompress( + self, data: Buffer, /, wbits: int = ..., bufsize: int = ... + ) -> bytes: ... + + +class CompressObjArgs(TypedDict, total=False): + wbits: int + strategy: int + level: int + + +class ZLibBackendWrapper: + def __init__(self, _zlib_backend: ZLibBackendProtocol): + self._zlib_backend: ZLibBackendProtocol = _zlib_backend + + @property + def name(self) -> str: + return getattr(self._zlib_backend, "__name__", "undefined") + + @property + def MAX_WBITS(self) -> int: + return self._zlib_backend.MAX_WBITS + + @property + def Z_FULL_FLUSH(self) -> int: + return self._zlib_backend.Z_FULL_FLUSH + + @property + def Z_SYNC_FLUSH(self) -> int: + return self._zlib_backend.Z_SYNC_FLUSH + + @property + def Z_BEST_SPEED(self) -> int: + return self._zlib_backend.Z_BEST_SPEED + + @property + def Z_FINISH(self) -> int: + return self._zlib_backend.Z_FINISH + + def compressobj(self, *args: Any, **kwargs: Any) -> ZLibCompressObjProtocol: + return self._zlib_backend.compressobj(*args, **kwargs) + + def decompressobj(self, *args: Any, **kwargs: Any) -> ZLibDecompressObjProtocol: + return self._zlib_backend.decompressobj(*args, **kwargs) + + def compress(self, data: Buffer, *args: Any, **kwargs: Any) -> bytes: + return self._zlib_backend.compress(data, *args, **kwargs) + + def decompress(self, data: Buffer, *args: Any, **kwargs: Any) -> bytes: + return self._zlib_backend.decompress(data, *args, **kwargs) + + # Everything not explicitly listed in the Protocol we just pass through + def __getattr__(self, attrname: str) -> Any: + return getattr(self._zlib_backend, attrname) + + +ZLibBackend: ZLibBackendWrapper = ZLibBackendWrapper(zlib) + + +def set_zlib_backend(new_zlib_backend: ZLibBackendProtocol) -> None: + ZLibBackend._zlib_backend = new_zlib_backend + + +def encoding_to_mode( + encoding: str | None = None, + suppress_deflate_header: bool = False, +) -> int: + if encoding == "gzip": + return 16 + ZLibBackend.MAX_WBITS + + return -ZLibBackend.MAX_WBITS if suppress_deflate_header else ZLibBackend.MAX_WBITS + + +class DecompressionBaseHandler(ABC): + def __init__( + self, + executor: Executor | None = None, + max_sync_chunk_size: int | None = MAX_SYNC_CHUNK_SIZE, + ): + """Base class for decompression handlers.""" + self._executor = executor + self._max_sync_chunk_size = max_sync_chunk_size + + @abstractmethod + def decompress_sync( + self, data: Buffer, max_length: int = ZLIB_MAX_LENGTH_UNLIMITED + ) -> bytes: + """Decompress the given data.""" + + async def decompress( + self, data: Buffer, max_length: int = ZLIB_MAX_LENGTH_UNLIMITED + ) -> bytes: + """Decompress the given data.""" + if ( + self._max_sync_chunk_size is not None + and len(data) > self._max_sync_chunk_size + ): + return await asyncio.get_event_loop().run_in_executor( + self._executor, self.decompress_sync, data, max_length + ) + return self.decompress_sync(data, max_length) + + @property + @abstractmethod + def data_available(self) -> bool: + """Return True if more output is available by passing b"".""" + + +class ZLibCompressor: + def __init__( + self, + encoding: str | None = None, + suppress_deflate_header: bool = False, + level: int | None = None, + wbits: int | None = None, + strategy: int | None = None, + executor: Executor | None = None, + max_sync_chunk_size: int | None = MAX_SYNC_CHUNK_SIZE, + ): + self._executor = executor + self._max_sync_chunk_size = max_sync_chunk_size + self._mode = ( + encoding_to_mode(encoding, suppress_deflate_header) + if wbits is None + else wbits + ) + self._zlib_backend: Final = ZLibBackendWrapper(ZLibBackend._zlib_backend) + + kwargs: CompressObjArgs = {} + kwargs["wbits"] = self._mode + if strategy is not None: + kwargs["strategy"] = strategy + if level is not None: + kwargs["level"] = level + self._compressor = self._zlib_backend.compressobj(**kwargs) + + def compress_sync(self, data: Buffer) -> bytes: + return self._compressor.compress(data) + + async def compress(self, data: Buffer) -> bytes: + """Compress the data and returned the compressed bytes. + + Note that flush() must be called after the last call to compress() + + If the data size is large than the max_sync_chunk_size, the compression + will be done in the executor. Otherwise, the compression will be done + in the event loop. + + **WARNING: This method is NOT cancellation-safe when used with flush().** + If this operation is cancelled, the compressor state may be corrupted. + The connection MUST be closed after cancellation to avoid data corruption + in subsequent compress operations. + + For cancellation-safe compression (e.g., WebSocket), the caller MUST wrap + compress() + flush() + send operations in a shield and lock to ensure atomicity. + """ + # For large payloads, offload compression to executor to avoid blocking event loop + should_use_executor = ( + self._max_sync_chunk_size is not None + and len(data) > self._max_sync_chunk_size + ) + if should_use_executor: + return await asyncio.get_running_loop().run_in_executor( + self._executor, self._compressor.compress, data + ) + return self.compress_sync(data) + + def flush(self, mode: int | None = None) -> bytes: + """Flush the compressor synchronously. + + **WARNING: This method is NOT cancellation-safe when called after compress().** + The flush() operation accesses shared compressor state. If compress() was + cancelled, calling flush() may result in corrupted data. The connection MUST + be closed after compress() cancellation. + + For cancellation-safe compression (e.g., WebSocket), the caller MUST wrap + compress() + flush() + send operations in a shield and lock to ensure atomicity. + """ + return self._compressor.flush( + mode if mode is not None else self._zlib_backend.Z_FINISH + ) + + +class ZLibDecompressor(DecompressionBaseHandler): + def __init__( + self, + encoding: str | None = None, + suppress_deflate_header: bool = False, + executor: Executor | None = None, + max_sync_chunk_size: int | None = MAX_SYNC_CHUNK_SIZE, + ): + super().__init__(executor=executor, max_sync_chunk_size=max_sync_chunk_size) + self._mode = encoding_to_mode(encoding, suppress_deflate_header) + self._zlib_backend: Final = ZLibBackendWrapper(ZLibBackend._zlib_backend) + self._decompressor = self._zlib_backend.decompressobj(wbits=self._mode) + self._last_empty = False + self._pending_unused_data: bytes | None = None + + def decompress_sync( + self, data: Buffer, max_length: int = ZLIB_MAX_LENGTH_UNLIMITED + ) -> bytes: + if self._pending_unused_data is not None: + data = self._pending_unused_data + bytes(data) + self._pending_unused_data = None + result = self._decompressor.decompress( + self._decompressor.unconsumed_tail + data, max_length + ) + # Only way to know that isal has no further data is checking we get no output + self._last_empty = result == b"" + + # Handle concatenated gzip/deflate streams (multi-member). + # After a member ends, unused_data holds the start of the next member. + # Create a fresh decompressor for each subsequent member. + while self._decompressor.eof and self._decompressor.unused_data: + unused = self._decompressor.unused_data + self._decompressor = self._zlib_backend.decompressobj(wbits=self._mode) + if max_length != ZLIB_MAX_LENGTH_UNLIMITED: + max_length -= len(result) + if max_length <= 0: + self._pending_unused_data = unused + break + chunk = self._decompressor.decompress(unused, max_length) + self._last_empty = chunk == b"" + result += chunk + + # Member ended exactly at chunk boundary — no unused_data, but the + # next feed_data() call would fail on the spent decompressor. + # Only reset for gzip; deflate's feed_eof() relies on eof=True to + # confirm the stream is complete. + if self._decompressor.eof and self._mode > self._zlib_backend.MAX_WBITS: + self._decompressor = self._zlib_backend.decompressobj(wbits=self._mode) + + return result + + def flush(self, length: int = 0) -> bytes: + return ( + self._decompressor.flush(length) + if length > 0 + else self._decompressor.flush() + ) + + @property + def data_available(self) -> bool: + return ( + bool(self._decompressor.unconsumed_tail) + or not self._last_empty + or self._pending_unused_data is not None + ) + + @property + def eof(self) -> bool: + return self._decompressor.eof + + +class BrotliDecompressor(DecompressionBaseHandler): + # Supports both 'brotlipy' and 'Brotli' packages + # since they share an import name. The top branches + # are for 'brotlipy' and bottom branches for 'Brotli' + def __init__( + self, + executor: Executor | None = None, + max_sync_chunk_size: int | None = MAX_SYNC_CHUNK_SIZE, + ) -> None: + """Decompress data using the Brotli library.""" + if not HAS_BROTLI: + raise RuntimeError( + "The brotli decompression is not available. " + "Please install `Brotli` module" + ) + self._obj = brotli.Decompressor() + self._last_empty = False + super().__init__(executor=executor, max_sync_chunk_size=max_sync_chunk_size) + + def decompress_sync( + self, data: Buffer, max_length: int = ZLIB_MAX_LENGTH_UNLIMITED + ) -> bytes: + """Decompress the given data.""" + if hasattr(self._obj, "decompress"): + if max_length == ZLIB_MAX_LENGTH_UNLIMITED: + result = cast(bytes, self._obj.decompress(data)) + else: + result = cast(bytes, self._obj.decompress(data, max_length)) + else: + if max_length == ZLIB_MAX_LENGTH_UNLIMITED: + result = cast(bytes, self._obj.process(data)) + else: + result = cast(bytes, self._obj.process(data, max_length)) + # Only way to know that brotli has no further data is checking we get no output + self._last_empty = result == b"" + return result + + def flush(self) -> bytes: + """Flush the decompressor.""" + if hasattr(self._obj, "flush"): + return cast(bytes, self._obj.flush()) + return b"" + + @property + def data_available(self) -> bool: + return not self._obj.is_finished() and not self._last_empty + + +class ZSTDDecompressor(DecompressionBaseHandler): + def __init__( + self, + executor: Executor | None = None, + max_sync_chunk_size: int | None = MAX_SYNC_CHUNK_SIZE, + ) -> None: + if not HAS_ZSTD: + raise RuntimeError( + "The zstd decompression is not available. " + "Please install `backports.zstd` module" + ) + self._obj = ZstdDecompressor() + self._pending_unused_data: bytes | None = None + super().__init__(executor=executor, max_sync_chunk_size=max_sync_chunk_size) + + def decompress_sync( + self, data: Buffer, max_length: int = ZLIB_MAX_LENGTH_UNLIMITED + ) -> bytes: + # zstd uses -1 for unlimited, while zlib uses 0 for unlimited + # Convert the zlib convention (0=unlimited) to zstd convention (-1=unlimited) + zstd_max_length = ( + ZSTD_MAX_LENGTH_UNLIMITED + if max_length == ZLIB_MAX_LENGTH_UNLIMITED + else max_length + ) + if self._pending_unused_data is not None: + data = self._pending_unused_data + data + self._pending_unused_data = None + result = self._obj.decompress(data, zstd_max_length) + + # Handle multi-frame zstd streams. + # https://datatracker.ietf.org/doc/html/rfc8878#section-3.1.1 + # ZstdDecompressor handles one frame only. When a frame ends, + # eof becomes True and any trailing data goes to unused_data. + # We create a fresh decompressor to continue with the next frame. + while self._obj.eof and self._obj.unused_data: + unused_data = self._obj.unused_data + self._obj = ZstdDecompressor() + if zstd_max_length != ZSTD_MAX_LENGTH_UNLIMITED: + zstd_max_length -= len(result) + if zstd_max_length <= 0: + self._pending_unused_data = unused_data + break + result += self._obj.decompress(unused_data, zstd_max_length) + + # Frame ended exactly at chunk boundary — no unused_data, but the + # next feed_data() call would fail on the spent decompressor. + # Prepare a fresh one for the next chunk. + if self._obj.eof: + self._obj = ZstdDecompressor() + + return result + + def flush(self) -> bytes: + return b"" + + @property + def data_available(self) -> bool: + return ( + not self._obj.needs_input and not self._obj.eof + ) or self._pending_unused_data is not None diff --git a/py_modules/aiohttp/connector.py b/py_modules/aiohttp/connector.py new file mode 100644 index 0000000..b7aa7b7 --- /dev/null +++ b/py_modules/aiohttp/connector.py @@ -0,0 +1,1851 @@ +import asyncio +import functools +import random +import socket +import sys +import traceback +import warnings +from collections import OrderedDict, defaultdict, deque +from collections.abc import Awaitable, Callable, Iterator, Sequence +from contextlib import suppress +from http import HTTPStatus +from itertools import chain, cycle, islice +from time import monotonic +from types import TracebackType +from typing import TYPE_CHECKING, Any, Literal, Optional, Union, cast + +import aiohappyeyeballs +from aiohappyeyeballs import AddrInfoType, SocketFactoryType + +from . import hdrs, helpers +from .abc import AbstractResolver, ResolveResult +from .client_exceptions import ( + ClientConnectionError, + ClientConnectorCertificateError, + ClientConnectorDNSError, + ClientConnectorError, + ClientConnectorSSLError, + ClientHttpProxyError, + ClientProxyConnectionError, + InvalidUrlClientError, + ServerFingerprintMismatch, + UnixClientConnectorError, + cert_errors, + ssl_errors, +) +from .client_proto import ResponseHandler +from .client_reqrep import ClientRequest, Fingerprint, _merge_ssl_params +from .helpers import ( + _SENTINEL, + ceil_timeout, + is_canonical_ipv4_address, + is_ip_address, + noop, + sentinel, + set_exception, + set_result, +) +from .log import client_logger +from .resolver import DefaultResolver + +if sys.version_info >= (3, 12): + from collections.abc import Buffer +else: + Buffer = Union[bytes, bytearray, "memoryview[int]", "memoryview[bytes]"] + +if TYPE_CHECKING: + import ssl + + SSLContext = ssl.SSLContext +else: + try: + import ssl + + SSLContext = ssl.SSLContext + except ImportError: # pragma: no cover + ssl = None # type: ignore[assignment] + SSLContext = object # type: ignore[misc,assignment] + +EMPTY_SCHEMA_SET = frozenset({""}) +HTTP_SCHEMA_SET = frozenset({"http", "https"}) +WS_SCHEMA_SET = frozenset({"ws", "wss"}) + +HTTP_AND_EMPTY_SCHEMA_SET = HTTP_SCHEMA_SET | EMPTY_SCHEMA_SET +HIGH_LEVEL_SCHEMA_SET = HTTP_AND_EMPTY_SCHEMA_SET | WS_SCHEMA_SET + +NEEDS_CLEANUP_CLOSED = (3, 13, 0) <= sys.version_info < ( + 3, + 13, + 1, +) or sys.version_info < (3, 12, 8) +# Cleanup closed is no longer needed after https://github.com/python/cpython/pull/118960 +# which first appeared in Python 3.12.8 and 3.13.1 + + +__all__ = ( + "BaseConnector", + "TCPConnector", + "UnixConnector", + "NamedPipeConnector", + "AddrInfoType", + "SocketFactoryType", +) + + +if TYPE_CHECKING: + from .client import ClientTimeout + from .client_reqrep import ConnectionKey + from .tracing import Trace + + +class _DeprecationWaiter: + __slots__ = ("_awaitable", "_awaited") + + def __init__(self, awaitable: Awaitable[Any]) -> None: + self._awaitable = awaitable + self._awaited = False + + def __await__(self) -> Any: + self._awaited = True + return self._awaitable.__await__() + + def __del__(self) -> None: + if not self._awaited: + warnings.warn( + "Connector.close() is a coroutine, " + "please use await connector.close()", + DeprecationWarning, + ) + + +async def _wait_for_close(waiters: list[Awaitable[object]]) -> None: + """Wait for all waiters to finish closing.""" + results = await asyncio.gather(*waiters, return_exceptions=True) + for res in results: + if isinstance(res, Exception): + client_logger.debug("Error while closing connector: %r", res) + + +class Connection: + + _source_traceback = None + + def __init__( + self, + connector: "BaseConnector", + key: "ConnectionKey", + protocol: ResponseHandler, + loop: asyncio.AbstractEventLoop, + ) -> None: + self._key = key + self._connector = connector + self._loop = loop + self._protocol: ResponseHandler | None = protocol + self._callbacks: list[Callable[[], None]] = [] + + if loop.get_debug(): + self._source_traceback = traceback.extract_stack(sys._getframe(1)) + + def __repr__(self) -> str: + return f"Connection<{self._key}>" + + def __del__(self, _warnings: Any = warnings) -> None: + if self._protocol is not None: + kwargs = {"source": self} + _warnings.warn(f"Unclosed connection {self!r}", ResourceWarning, **kwargs) + if self._loop.is_closed(): + return + + self._connector._release(self._key, self._protocol, should_close=True) + + context = {"client_connection": self, "message": "Unclosed connection"} + if self._source_traceback is not None: + context["source_traceback"] = self._source_traceback + self._loop.call_exception_handler(context) + + def __bool__(self) -> Literal[True]: + """Force subclasses to not be falsy, to make checks simpler.""" + return True + + @property + def loop(self) -> asyncio.AbstractEventLoop: + warnings.warn( + "connector.loop property is deprecated", DeprecationWarning, stacklevel=2 + ) + return self._loop + + @property + def transport(self) -> asyncio.Transport | None: + if self._protocol is None: + return None + return self._protocol.transport + + @property + def protocol(self) -> ResponseHandler | None: + return self._protocol + + def add_callback(self, callback: Callable[[], None]) -> None: + if callback is not None: + self._callbacks.append(callback) + + def _notify_release(self) -> None: + callbacks, self._callbacks = self._callbacks[:], [] + + for cb in callbacks: + with suppress(Exception): + cb() + + def close(self) -> None: + self._notify_release() + + if self._protocol is not None: + self._connector._release(self._key, self._protocol, should_close=True) + self._protocol = None + + def release(self) -> None: + self._notify_release() + + if self._protocol is not None: + self._connector._release(self._key, self._protocol) + self._protocol = None + + @property + def closed(self) -> bool: + return self._protocol is None or not self._protocol.is_connected() + + +class _ConnectTunnelConnection(Connection): + """Special connection wrapper for CONNECT tunnels that must never be pooled. + + This connection wraps the proxy connection that will be upgraded with TLS. + It must never be released to the pool because: + 1. Its 'closed' future will never complete, causing session.close() to hang + 2. It represents an intermediate state, not a reusable connection + 3. The real connection (with TLS) will be created separately + """ + + def release(self) -> None: + """Do nothing - don't pool or close the connection. + + These connections are an intermediate state during the CONNECT tunnel + setup and will be cleaned up naturally after the TLS upgrade. If they + were to be pooled, they would never be properly closed, causing + session.close() to wait forever for their 'closed' future. + """ + + +class _TransportPlaceholder: + """placeholder for BaseConnector.connect function""" + + __slots__ = ("closed", "transport") + + def __init__(self, closed_future: asyncio.Future[Exception | None]) -> None: + """Initialize a placeholder for a transport.""" + self.closed = closed_future + self.transport = None + + def close(self) -> None: + """Close the placeholder.""" + + def abort(self) -> None: + """Abort the placeholder (does nothing).""" + + +class BaseConnector: + """Base connector class. + + keepalive_timeout - (optional) Keep-alive timeout. + force_close - Set to True to force close and do reconnect + after each request (and between redirects). + limit - The total number of simultaneous connections. + limit_per_host - Number of simultaneous connections to one host. + enable_cleanup_closed - Enables clean-up closed ssl transports. + Disabled by default. + timeout_ceil_threshold - Trigger ceiling of timeout values when + it's above timeout_ceil_threshold. + loop - Optional event loop. + """ + + _closed = True # prevent AttributeError in __del__ if ctor was failed + _source_traceback = None + + # abort transport after 2 seconds (cleanup broken connections) + _cleanup_closed_period = 2.0 + + allowed_protocol_schema_set = HIGH_LEVEL_SCHEMA_SET + + def __init__( + self, + *, + keepalive_timeout: object | None | float = sentinel, + force_close: bool = False, + limit: int = 100, + limit_per_host: int = 0, + enable_cleanup_closed: bool = False, + loop: asyncio.AbstractEventLoop | None = None, + timeout_ceil_threshold: float = 5, + ) -> None: + + if force_close: + if keepalive_timeout is not None and keepalive_timeout is not sentinel: + raise ValueError( + "keepalive_timeout cannot be set if force_close is True" + ) + else: + if keepalive_timeout is sentinel: + keepalive_timeout = 15.0 + + loop = loop or asyncio.get_running_loop() + self._timeout_ceil_threshold = timeout_ceil_threshold + + self._closed = False + if loop.get_debug(): + self._source_traceback = traceback.extract_stack(sys._getframe(1)) + + # Connection pool of reusable connections. + # We use a deque to store connections because it has O(1) popleft() + # and O(1) append() operations to implement a FIFO queue. + self._conns: defaultdict[ + ConnectionKey, deque[tuple[ResponseHandler, float]] + ] = defaultdict(deque) + self._limit = limit + self._limit_per_host = limit_per_host + self._acquired: set[ResponseHandler] = set() + self._acquired_per_host: defaultdict[ConnectionKey, set[ResponseHandler]] = ( + defaultdict(set) + ) + self._keepalive_timeout = cast(float, keepalive_timeout) + self._force_close = force_close + + # {host_key: FIFO list of waiters} + # The FIFO is implemented with an OrderedDict with None keys because + # python does not have an ordered set. + self._waiters: defaultdict[ + ConnectionKey, OrderedDict[asyncio.Future[None], None] + ] = defaultdict(OrderedDict) + + self._loop = loop + self._factory = functools.partial(ResponseHandler, loop=loop) + + # start keep-alive connection cleanup task + self._cleanup_handle: asyncio.TimerHandle | None = None + + # start cleanup closed transports task + self._cleanup_closed_handle: asyncio.TimerHandle | None = None + + if enable_cleanup_closed and not NEEDS_CLEANUP_CLOSED: + warnings.warn( + "enable_cleanup_closed ignored because " + "https://github.com/python/cpython/pull/118960 is fixed " + f"in Python version {sys.version_info}", + DeprecationWarning, + stacklevel=2, + ) + enable_cleanup_closed = False + + self._cleanup_closed_disabled = not enable_cleanup_closed + self._cleanup_closed_transports: list[asyncio.Transport | None] = [] + self._placeholder_future: asyncio.Future[Exception | None] = ( + loop.create_future() + ) + self._placeholder_future.set_result(None) + self._cleanup_closed() + + def __del__(self, _warnings: Any = warnings) -> None: + if self._closed: + return + if not self._conns: + return + + conns = [repr(c) for c in self._conns.values()] + + self._close() + + kwargs = {"source": self} + _warnings.warn(f"Unclosed connector {self!r}", ResourceWarning, **kwargs) + context = { + "connector": self, + "connections": conns, + "message": "Unclosed connector", + } + if self._source_traceback is not None: + context["source_traceback"] = self._source_traceback + self._loop.call_exception_handler(context) + + def __enter__(self) -> "BaseConnector": + warnings.warn( + '"with Connector():" is deprecated, ' + 'use "async with Connector():" instead', + DeprecationWarning, + ) + return self + + def __exit__(self, *exc: Any) -> None: + self._close() + + async def __aenter__(self) -> "BaseConnector": + return self + + async def __aexit__( + self, + exc_type: type[BaseException] | None = None, + exc_value: BaseException | None = None, + exc_traceback: TracebackType | None = None, + ) -> None: + await self.close() + + @property + def force_close(self) -> bool: + """Ultimately close connection on releasing if True.""" + return self._force_close + + @property + def limit(self) -> int: + """The total number for simultaneous connections. + + If limit is 0 the connector has no limit. + The default limit size is 100. + """ + return self._limit + + @property + def limit_per_host(self) -> int: + """The limit for simultaneous connections to the same endpoint. + + Endpoints are the same if they are have equal + (host, port, is_ssl) triple. + """ + return self._limit_per_host + + def _cleanup(self) -> None: + """Cleanup unused transports.""" + if self._cleanup_handle: + self._cleanup_handle.cancel() + # _cleanup_handle should be unset, otherwise _release() will not + # recreate it ever! + self._cleanup_handle = None + + now = monotonic() + timeout = self._keepalive_timeout + + if self._conns: + connections = defaultdict(deque) + deadline = now - timeout + for key, conns in self._conns.items(): + alive: deque[tuple[ResponseHandler, float]] = deque() + for proto, use_time in conns: + if proto.is_connected() and use_time - deadline >= 0: + alive.append((proto, use_time)) + continue + transport = proto.transport + proto.close() + if not self._cleanup_closed_disabled and key.is_ssl: + self._cleanup_closed_transports.append(transport) + + if alive: + connections[key] = alive + + self._conns = connections + + if self._conns: + self._cleanup_handle = helpers.weakref_handle( + self, + "_cleanup", + timeout, + self._loop, + timeout_ceil_threshold=self._timeout_ceil_threshold, + ) + + def _cleanup_closed(self) -> None: + """Double confirmation for transport close. + + Some broken ssl servers may leave socket open without proper close. + """ + if self._cleanup_closed_handle: + self._cleanup_closed_handle.cancel() + + for transport in self._cleanup_closed_transports: + if transport is not None: + transport.abort() + + self._cleanup_closed_transports = [] + + if not self._cleanup_closed_disabled: + self._cleanup_closed_handle = helpers.weakref_handle( + self, + "_cleanup_closed", + self._cleanup_closed_period, + self._loop, + timeout_ceil_threshold=self._timeout_ceil_threshold, + ) + + def close(self, *, abort_ssl: bool = False) -> Awaitable[None]: + """Close all opened transports. + + :param abort_ssl: If True, SSL connections will be aborted immediately + without performing the shutdown handshake. This provides + faster cleanup at the cost of less graceful disconnection. + """ + if not (waiters := self._close(abort_ssl=abort_ssl)): + # If there are no connections to close, we can return a noop + # awaitable to avoid scheduling a task on the event loop. + return _DeprecationWaiter(noop()) + coro = _wait_for_close(waiters) + if sys.version_info >= (3, 12): + # Optimization for Python 3.12, try to close connections + # immediately to avoid having to schedule the task on the event loop. + task = asyncio.Task(coro, loop=self._loop, eager_start=True) + else: + task = self._loop.create_task(coro) + return _DeprecationWaiter(task) + + def _close(self, *, abort_ssl: bool = False) -> list[Awaitable[object]]: + waiters: list[Awaitable[object]] = [] + + if self._closed: + return waiters + + self._closed = True + + try: + if self._loop.is_closed(): + return waiters + + # cancel cleanup task + if self._cleanup_handle: + self._cleanup_handle.cancel() + + # cancel cleanup close task + if self._cleanup_closed_handle: + self._cleanup_closed_handle.cancel() + + for data in self._conns.values(): + for proto, _ in data: + if ( + abort_ssl + and proto.transport + and proto.transport.get_extra_info("sslcontext") is not None + ): + proto.abort() + else: + proto.close() + if closed := proto.closed: + waiters.append(closed) + + for proto in self._acquired: + if ( + abort_ssl + and proto.transport + and proto.transport.get_extra_info("sslcontext") is not None + ): + proto.abort() + else: + proto.close() + if closed := proto.closed: + waiters.append(closed) + + for transport in self._cleanup_closed_transports: + if transport is not None: + transport.abort() + + return waiters + + finally: + self._conns.clear() + self._acquired.clear() + for keyed_waiters in self._waiters.values(): + for keyed_waiter in keyed_waiters: + keyed_waiter.cancel() + self._waiters.clear() + self._cleanup_handle = None + self._cleanup_closed_transports.clear() + self._cleanup_closed_handle = None + + @property + def closed(self) -> bool: + """Is connector closed. + + A readonly property. + """ + return self._closed + + def _available_connections(self, key: "ConnectionKey") -> int: + """ + Return number of available connections. + + The limit, limit_per_host and the connection key are taken into account. + + If it returns less than 1 means that there are no connections + available. + """ + # check total available connections + # If there are no limits, this will always return 1 + total_remain = 1 + + if self._limit and (total_remain := self._limit - len(self._acquired)) <= 0: + return total_remain + + # check limit per host + if host_remain := self._limit_per_host: + if acquired := self._acquired_per_host.get(key): + host_remain -= len(acquired) + if total_remain > host_remain: + return host_remain + + return total_remain + + def _update_proxy_auth_header_and_build_proxy_req( + self, req: ClientRequest + ) -> ClientRequest: + """Set Proxy-Authorization header for non-SSL proxy requests and builds the proxy request for SSL proxy requests.""" + url = req.proxy + assert url is not None + headers: dict[str, str] = {} + if req.proxy_headers is not None: + headers = req.proxy_headers # type: ignore[assignment] + headers[hdrs.HOST] = req.headers[hdrs.HOST] + proxy_req = ClientRequest( + hdrs.METH_GET, + url, + headers=headers, + auth=req.proxy_auth, + loop=self._loop, + ssl=req.ssl, + ) + auth = proxy_req.headers.pop(hdrs.AUTHORIZATION, None) + if auth is not None: + if not req.is_ssl(): + req.headers[hdrs.PROXY_AUTHORIZATION] = auth + else: + proxy_req.headers[hdrs.PROXY_AUTHORIZATION] = auth + return proxy_req + + async def connect( + self, req: ClientRequest, traces: list["Trace"], timeout: "ClientTimeout" + ) -> Connection: + """Get from pool or create new connection.""" + key = req.connection_key + if (conn := await self._get(key, traces)) is not None: + # If we do not have to wait and we can get a connection from the pool + # we can avoid the timeout ceil logic and directly return the connection + if req.proxy: + self._update_proxy_auth_header_and_build_proxy_req(req) + return conn + + async with ceil_timeout(timeout.connect, timeout.ceil_threshold): + if self._available_connections(key) <= 0: + await self._wait_for_available_connection(key, traces) + if (conn := await self._get(key, traces)) is not None: + if req.proxy: + self._update_proxy_auth_header_and_build_proxy_req(req) + return conn + + placeholder = cast( + ResponseHandler, _TransportPlaceholder(self._placeholder_future) + ) + self._acquired.add(placeholder) + if self._limit_per_host: + self._acquired_per_host[key].add(placeholder) + + try: + # Traces are done inside the try block to ensure that the + # that the placeholder is still cleaned up if an exception + # is raised. + if traces: + for trace in traces: + await trace.send_connection_create_start() + proto = await self._create_connection(req, traces, timeout) + if traces: + for trace in traces: + await trace.send_connection_create_end() + except BaseException: + self._release_acquired(key, placeholder) + raise + else: + if self._closed: + proto.close() + raise ClientConnectionError("Connector is closed.") + + # The connection was successfully created, drop the placeholder + # and add the real connection to the acquired set. There should + # be no awaits after the proto is added to the acquired set + # to ensure that the connection is not left in the acquired set + # on cancellation. + self._acquired.remove(placeholder) + self._acquired.add(proto) + if self._limit_per_host: + acquired_per_host = self._acquired_per_host[key] + acquired_per_host.remove(placeholder) + acquired_per_host.add(proto) + return Connection(self, key, proto, self._loop) + + async def _wait_for_available_connection( + self, key: "ConnectionKey", traces: list["Trace"] + ) -> None: + """Wait for an available connection slot.""" + # We loop here because there is a race between + # the connection limit check and the connection + # being acquired. If the connection is acquired + # between the check and the await statement, we + # need to loop again to check if the connection + # slot is still available. + attempts = 0 + while True: + fut: asyncio.Future[None] = self._loop.create_future() + keyed_waiters = self._waiters[key] + keyed_waiters[fut] = None + if attempts: + # If we have waited before, we need to move the waiter + # to the front of the queue as otherwise we might get + # starved and hit the timeout. + keyed_waiters.move_to_end(fut, last=False) + + try: + # Traces happen in the try block to ensure that the + # the waiter is still cleaned up if an exception is raised. + if traces: + for trace in traces: + await trace.send_connection_queued_start() + await fut + if traces: + for trace in traces: + await trace.send_connection_queued_end() + finally: + # pop the waiter from the queue if its still + # there and not already removed by _release_waiter + keyed_waiters.pop(fut, None) + if not self._waiters.get(key, True): + del self._waiters[key] + + if self._available_connections(key) > 0: + break + attempts += 1 + + async def _get( + self, key: "ConnectionKey", traces: list["Trace"] + ) -> Connection | None: + """Get next reusable connection for the key or None. + + The connection will be marked as acquired. + """ + if (conns := self._conns.get(key)) is None: + return None + + t1 = monotonic() + while conns: + proto, t0 = conns.popleft() + # We will we reuse the connection if its connected and + # the keepalive timeout has not been exceeded + if proto.is_connected() and t1 - t0 <= self._keepalive_timeout: + if not conns: + # The very last connection was reclaimed: drop the key + del self._conns[key] + self._acquired.add(proto) + if self._limit_per_host: + self._acquired_per_host[key].add(proto) + if traces: + for trace in traces: + try: + await trace.send_connection_reuseconn() + except BaseException: + self._release_acquired(key, proto) + raise + return Connection(self, key, proto, self._loop) + + # Connection cannot be reused, close it + transport = proto.transport + proto.close() + # only for SSL transports + if not self._cleanup_closed_disabled and key.is_ssl: + self._cleanup_closed_transports.append(transport) + + # No more connections: drop the key + del self._conns[key] + return None + + def _release_waiter(self) -> None: + """ + Iterates over all waiters until one to be released is found. + + The one to be released is not finished and + belongs to a host that has available connections. + """ + if not self._waiters: + return + + # Having the dict keys ordered this avoids to iterate + # at the same order at each call. + queues = list(self._waiters) + random.shuffle(queues) + + for key in queues: + if self._available_connections(key) < 1: + continue + + waiters = self._waiters[key] + while waiters: + waiter, _ = waiters.popitem(last=False) + if not waiter.done(): + waiter.set_result(None) + return + + def _release_acquired(self, key: "ConnectionKey", proto: ResponseHandler) -> None: + """Release acquired connection.""" + if self._closed: + # acquired connection is already released on connector closing + return + + self._acquired.discard(proto) + if self._limit_per_host and (conns := self._acquired_per_host.get(key)): + conns.discard(proto) + if not conns: + del self._acquired_per_host[key] + self._release_waiter() + + def _release( + self, + key: "ConnectionKey", + protocol: ResponseHandler, + *, + should_close: bool = False, + ) -> None: + if self._closed: + # acquired connection is already released on connector closing + return + + self._release_acquired(key, protocol) + + if self._force_close or should_close or protocol.should_close: + transport = protocol.transport + protocol.close() + + if key.is_ssl and not self._cleanup_closed_disabled: + self._cleanup_closed_transports.append(transport) + return + + self._conns[key].append((protocol, monotonic())) + + if self._cleanup_handle is None: + self._cleanup_handle = helpers.weakref_handle( + self, + "_cleanup", + self._keepalive_timeout, + self._loop, + timeout_ceil_threshold=self._timeout_ceil_threshold, + ) + + async def _create_connection( + self, req: ClientRequest, traces: list["Trace"], timeout: "ClientTimeout" + ) -> ResponseHandler: + raise NotImplementedError() + + +class _DNSCacheTable: + def __init__(self, ttl: float | None = None, max_size: int = 1000) -> None: + self._addrs_rr: OrderedDict[ + tuple[str, int], tuple[Iterator[ResolveResult], int] + ] = OrderedDict() + self._timestamps: dict[tuple[str, int], float] = {} + self._ttl = ttl + self._max_size = max_size + + def __contains__(self, host: object) -> bool: + return host in self._addrs_rr + + def add(self, key: tuple[str, int], addrs: list[ResolveResult]) -> None: + if key in self._addrs_rr: + self._addrs_rr.move_to_end(key) + + self._addrs_rr[key] = (cycle(addrs), len(addrs)) + + if self._ttl is not None: + self._timestamps[key] = monotonic() + + if len(self._addrs_rr) > self._max_size: + oldest_key, _ = self._addrs_rr.popitem(last=False) + self._timestamps.pop(oldest_key, None) + + def remove(self, key: tuple[str, int]) -> None: + self._addrs_rr.pop(key, None) + self._timestamps.pop(key, None) + + def clear(self) -> None: + self._addrs_rr.clear() + self._timestamps.clear() + + def next_addrs(self, key: tuple[str, int]) -> list[ResolveResult]: + loop, length = self._addrs_rr[key] + addrs = list(islice(loop, length)) + # Consume one more element to shift internal state of `cycle` + next(loop) + self._addrs_rr.move_to_end(key) + return addrs + + def expired(self, key: tuple[str, int]) -> bool: + if self._ttl is None: + return False + + return self._timestamps[key] + self._ttl < monotonic() + + +def _make_ssl_context(verified: bool) -> SSLContext: + """Create SSL context. + + This method is not async-friendly and should be called from a thread + because it will load certificates from disk and do other blocking I/O. + """ + if ssl is None: + # No ssl support + return None + if verified: + sslcontext = ssl.create_default_context() + else: + sslcontext = ssl.SSLContext(ssl.PROTOCOL_TLS_CLIENT) + sslcontext.options |= ssl.OP_NO_SSLv2 + sslcontext.options |= ssl.OP_NO_SSLv3 + sslcontext.check_hostname = False + sslcontext.verify_mode = ssl.CERT_NONE + sslcontext.options |= ssl.OP_NO_COMPRESSION + sslcontext.set_default_verify_paths() + sslcontext.set_alpn_protocols(("http/1.1",)) + return sslcontext + + +# The default SSLContext objects are created at import time +# since they do blocking I/O to load certificates from disk, +# and imports should always be done before the event loop starts +# or in a thread. +_SSL_CONTEXT_VERIFIED = _make_ssl_context(True) +_SSL_CONTEXT_UNVERIFIED = _make_ssl_context(False) + + +class TCPConnector(BaseConnector): + """TCP connector. + + verify_ssl - Set to True to check ssl certifications. + fingerprint - Pass the binary sha256 + digest of the expected certificate in DER format to verify + that the certificate the server presents matches. See also + https://en.wikipedia.org/wiki/HTTP_Public_Key_Pinning + resolver - Enable DNS lookups and use this + resolver + use_dns_cache - Use memory cache for DNS lookups. + ttl_dns_cache - Max seconds having cached a DNS entry, None forever. + family - socket address family + local_addr - local tuple of (host, port) to bind socket to + + keepalive_timeout - (optional) Keep-alive timeout. + force_close - Set to True to force close and do reconnect + after each request (and between redirects). + limit - The total number of simultaneous connections. + limit_per_host - Number of simultaneous connections to one host. + enable_cleanup_closed - Enables clean-up closed ssl transports. + Disabled by default. + happy_eyeballs_delay - This is the “Connection Attempt Delay” + as defined in RFC 8305. To disable + the happy eyeballs algorithm, set to None. + interleave - “First Address Family Count” as defined in RFC 8305 + loop - Optional event loop. + socket_factory - A SocketFactoryType function that, if supplied, + will be used to create sockets given an + AddrInfoType. + ssl_shutdown_timeout - DEPRECATED. Will be removed in aiohttp 4.0. + Grace period for SSL shutdown handshake on TLS + connections. Default is 0 seconds (immediate abort). + This parameter allowed for a clean SSL shutdown by + notifying the remote peer of connection closure, + while avoiding excessive delays during connector cleanup. + Note: Only takes effect on Python 3.11+. + """ + + allowed_protocol_schema_set = HIGH_LEVEL_SCHEMA_SET | frozenset({"tcp"}) + + def __init__( + self, + *, + verify_ssl: bool = True, + fingerprint: bytes | None = None, + use_dns_cache: bool = True, + ttl_dns_cache: int | None = 10, + dns_cache_max_size: int = 1000, + family: socket.AddressFamily = socket.AddressFamily.AF_UNSPEC, + ssl_context: SSLContext | None = None, + ssl: bool | Fingerprint | SSLContext = True, + local_addr: tuple[str, int] | None = None, + resolver: AbstractResolver | None = None, + keepalive_timeout: None | float | object = sentinel, + force_close: bool = False, + limit: int = 100, + limit_per_host: int = 0, + enable_cleanup_closed: bool = False, + loop: asyncio.AbstractEventLoop | None = None, + timeout_ceil_threshold: float = 5, + happy_eyeballs_delay: float | None = 0.25, + interleave: int | None = None, + socket_factory: SocketFactoryType | None = None, + ssl_shutdown_timeout: _SENTINEL | None | float = sentinel, + ): + super().__init__( + keepalive_timeout=keepalive_timeout, + force_close=force_close, + limit=limit, + limit_per_host=limit_per_host, + enable_cleanup_closed=enable_cleanup_closed, + loop=loop, + timeout_ceil_threshold=timeout_ceil_threshold, + ) + + self._ssl = _merge_ssl_params(ssl, verify_ssl, ssl_context, fingerprint) + + self._resolver: AbstractResolver + if resolver is None: + self._resolver = DefaultResolver(loop=self._loop) + self._resolver_owner = True + else: + self._resolver = resolver + self._resolver_owner = False + + self._use_dns_cache = use_dns_cache + self._cached_hosts = _DNSCacheTable( + ttl=ttl_dns_cache, max_size=dns_cache_max_size + ) + self._throttle_dns_futures: dict[tuple[str, int], set[asyncio.Future[None]]] = ( + {} + ) + self._family = family + self._local_addr_infos = aiohappyeyeballs.addr_to_addr_infos(local_addr) + self._happy_eyeballs_delay = happy_eyeballs_delay + self._interleave = interleave + self._resolve_host_tasks: set[asyncio.Task[list[ResolveResult]]] = set() + self._socket_factory = socket_factory + self._ssl_shutdown_timeout: float | None + # Handle ssl_shutdown_timeout with warning for Python < 3.11 + if ssl_shutdown_timeout is sentinel: + self._ssl_shutdown_timeout = 0 + else: + # Deprecation warning for ssl_shutdown_timeout parameter + warnings.warn( + "The ssl_shutdown_timeout parameter is deprecated and will be removed in aiohttp 4.0", + DeprecationWarning, + stacklevel=2, + ) + if ( + sys.version_info < (3, 11) + and ssl_shutdown_timeout is not None + and ssl_shutdown_timeout != 0 + ): + warnings.warn( + f"ssl_shutdown_timeout={ssl_shutdown_timeout} is ignored on Python < 3.11; " + "only ssl_shutdown_timeout=0 is supported. The timeout will be ignored.", + RuntimeWarning, + stacklevel=2, + ) + self._ssl_shutdown_timeout = ssl_shutdown_timeout + + def _close(self, *, abort_ssl: bool = False) -> list[Awaitable[object]]: + """Close all ongoing DNS calls.""" + for fut in chain.from_iterable(self._throttle_dns_futures.values()): + fut.cancel() + + waiters = super()._close(abort_ssl=abort_ssl) + + for t in self._resolve_host_tasks: + t.cancel() + waiters.append(t) + + return waiters + + async def close(self, *, abort_ssl: bool = False) -> None: + """ + Close all opened transports. + + :param abort_ssl: If True, SSL connections will be aborted immediately + without performing the shutdown handshake. If False (default), + the behavior is determined by ssl_shutdown_timeout: + - If ssl_shutdown_timeout=0: connections are aborted + - If ssl_shutdown_timeout>0: graceful shutdown is performed + """ + # Use abort_ssl param if explicitly set, otherwise use ssl_shutdown_timeout default + await super().close(abort_ssl=abort_ssl or self._ssl_shutdown_timeout == 0) + if self._resolver_owner: + await self._resolver.close() + + @property + def family(self) -> int: + """Socket family like AF_INET.""" + return self._family + + @property + def use_dns_cache(self) -> bool: + """True if local DNS caching is enabled.""" + return self._use_dns_cache + + def clear_dns_cache(self, host: str | None = None, port: int | None = None) -> None: + """Remove specified host/port or clear all dns local cache.""" + if host is not None and port is not None: + self._cached_hosts.remove((host, port)) + elif host is not None or port is not None: + raise ValueError("either both host and port or none of them are allowed") + else: + self._cached_hosts.clear() + + async def _resolve_host( + self, host: str, port: int, traces: Sequence["Trace"] | None = None + ) -> list[ResolveResult]: + """Resolve host and return list of addresses.""" + if is_ip_address(host): + # Reject legacy numeric IPv4 forms (e.g. 2130706433, 127.1) that + # socket would map onto an address, slipping past a connector-level + # policy that only sees the raw host. + if ":" not in host and not is_canonical_ipv4_address(host): + raise InvalidUrlClientError(host, "is not a canonical IPv4 address") + return [ + { + "hostname": host, + "host": host, + "port": port, + "family": self._family, + "proto": 0, + "flags": 0, + } + ] + + if not self._use_dns_cache: + + if traces: + for trace in traces: + await trace.send_dns_resolvehost_start(host) + + if self._closed: + raise ClientConnectionError("Connector is closed") + + res = await self._resolver.resolve(host, port, family=self._family) + + if traces: + for trace in traces: + await trace.send_dns_resolvehost_end(host) + + return res + + key = (host, port) + if key in self._cached_hosts and not self._cached_hosts.expired(key): + # get result early, before any await (#4014) + result = self._cached_hosts.next_addrs(key) + + if traces: + for trace in traces: + await trace.send_dns_cache_hit(host) + return result + + futures: set[asyncio.Future[None]] + # + # If multiple connectors are resolving the same host, we wait + # for the first one to resolve and then use the result for all of them. + # We use a throttle to ensure that we only resolve the host once + # and then use the result for all the waiters. + # + if key in self._throttle_dns_futures: + # get futures early, before any await (#4014) + futures = self._throttle_dns_futures[key] + future: asyncio.Future[None] = self._loop.create_future() + futures.add(future) + if traces: + for trace in traces: + await trace.send_dns_cache_hit(host) + try: + await future + finally: + futures.discard(future) + return self._cached_hosts.next_addrs(key) + + # update dict early, before any await (#4014) + self._throttle_dns_futures[key] = futures = set() + # In this case we need to create a task to ensure that we can shield + # the task from cancellation as cancelling this lookup should not cancel + # the underlying lookup or else the cancel event will get broadcast to + # all the waiters across all connections. + # + coro = self._resolve_host_with_throttle(key, host, port, futures, traces) + loop = asyncio.get_running_loop() + if sys.version_info >= (3, 12): + # Optimization for Python 3.12, try to send immediately + resolved_host_task = asyncio.Task(coro, loop=loop, eager_start=True) + else: + resolved_host_task = loop.create_task(coro) + + if not resolved_host_task.done(): + self._resolve_host_tasks.add(resolved_host_task) + resolved_host_task.add_done_callback(self._resolve_host_tasks.discard) + + try: + return await asyncio.shield(resolved_host_task) + except asyncio.CancelledError: + + def drop_exception(fut: "asyncio.Future[list[ResolveResult]]") -> None: + with suppress(Exception, asyncio.CancelledError): + fut.result() + + resolved_host_task.add_done_callback(drop_exception) + raise + + async def _resolve_host_with_throttle( + self, + key: tuple[str, int], + host: str, + port: int, + futures: set["asyncio.Future[None]"], + traces: Sequence["Trace"] | None, + ) -> list[ResolveResult]: + """Resolve host and set result for all waiters. + + This method must be run in a task and shielded from cancellation + to avoid cancelling the underlying lookup. + """ + try: + if traces: + for trace in traces: + await trace.send_dns_cache_miss(host) + + for trace in traces: + await trace.send_dns_resolvehost_start(host) + + addrs = await self._resolver.resolve(host, port, family=self._family) + if traces: + for trace in traces: + await trace.send_dns_resolvehost_end(host) + + self._cached_hosts.add(key, addrs) + for fut in futures: + set_result(fut, None) + except BaseException as e: + # any DNS exception is set for the waiters to raise the same exception. + # This coro is always run in task that is shielded from cancellation so + # we should never be propagating cancellation here. + for fut in futures: + set_exception(fut, e) + raise + finally: + self._throttle_dns_futures.pop(key) + + return self._cached_hosts.next_addrs(key) + + async def _create_connection( + self, req: ClientRequest, traces: list["Trace"], timeout: "ClientTimeout" + ) -> ResponseHandler: + """Create connection. + + Has same keyword arguments as BaseEventLoop.create_connection. + """ + if req.proxy: + _, proto = await self._create_proxy_connection(req, traces, timeout) + else: + _, proto = await self._create_direct_connection(req, traces, timeout) + + return proto + + def _get_ssl_context(self, req: ClientRequest) -> SSLContext | None: + """Logic to get the correct SSL context + + 0. if req.ssl is false, return None + + 1. if ssl_context is specified in req, use it + 2. if _ssl_context is specified in self, use it + 3. otherwise: + 1. if verify_ssl is not specified in req, use self.ssl_context + (will generate a default context according to self.verify_ssl) + 2. if verify_ssl is True in req, generate a default SSL context + 3. if verify_ssl is False in req, generate a SSL context that + won't verify + """ + if not req.is_ssl(): + return None + + if ssl is None: # pragma: no cover + raise RuntimeError("SSL is not supported.") + sslcontext = req.ssl + if isinstance(sslcontext, ssl.SSLContext): + return sslcontext + if sslcontext is not True: + # not verified or fingerprinted + return _SSL_CONTEXT_UNVERIFIED + sslcontext = self._ssl + if isinstance(sslcontext, ssl.SSLContext): + return sslcontext + if sslcontext is not True: + # not verified or fingerprinted + return _SSL_CONTEXT_UNVERIFIED + return _SSL_CONTEXT_VERIFIED + + def _get_fingerprint(self, req: ClientRequest) -> Optional["Fingerprint"]: + ret = req.ssl + if isinstance(ret, Fingerprint): + return ret + ret = self._ssl + if isinstance(ret, Fingerprint): + return ret + return None + + async def _wrap_create_connection( + self, + *args: Any, + addr_infos: list[AddrInfoType], + req: ClientRequest, + timeout: "ClientTimeout", + client_error: type[Exception] = ClientConnectorError, + **kwargs: Any, + ) -> tuple[asyncio.Transport, ResponseHandler]: + try: + async with ceil_timeout( + timeout.sock_connect, ceil_threshold=timeout.ceil_threshold + ): + sock = await aiohappyeyeballs.start_connection( + addr_infos=addr_infos, + local_addr_infos=self._local_addr_infos, + happy_eyeballs_delay=self._happy_eyeballs_delay, + interleave=self._interleave, + loop=self._loop, + socket_factory=self._socket_factory, + ) + # Add ssl_shutdown_timeout for Python 3.11+ when SSL is used + if ( + kwargs.get("ssl") + and self._ssl_shutdown_timeout + and sys.version_info >= (3, 11) + ): + kwargs["ssl_shutdown_timeout"] = self._ssl_shutdown_timeout + return await self._loop.create_connection(*args, **kwargs, sock=sock) + except cert_errors as exc: + raise ClientConnectorCertificateError(req.connection_key, exc) from exc + except ssl_errors as exc: + raise ClientConnectorSSLError(req.connection_key, exc) from exc + except OSError as exc: + if exc.errno is None and isinstance(exc, asyncio.TimeoutError): + raise + raise client_error(req.connection_key, exc) from exc + + async def _wrap_existing_connection( + self, + *args: Any, + req: ClientRequest, + timeout: "ClientTimeout", + client_error: type[Exception] = ClientConnectorError, + **kwargs: Any, + ) -> tuple[asyncio.Transport, ResponseHandler]: + try: + async with ceil_timeout( + timeout.sock_connect, ceil_threshold=timeout.ceil_threshold + ): + return await self._loop.create_connection(*args, **kwargs) + except cert_errors as exc: + raise ClientConnectorCertificateError(req.connection_key, exc) from exc + except ssl_errors as exc: + raise ClientConnectorSSLError(req.connection_key, exc) from exc + except OSError as exc: + if exc.errno is None and isinstance(exc, asyncio.TimeoutError): + raise + raise client_error(req.connection_key, exc) from exc + + def _fail_on_no_start_tls(self, req: "ClientRequest") -> None: + """Raise a :py:exc:`RuntimeError` on missing ``start_tls()``. + + It is necessary for TLS-in-TLS so that it is possible to + send HTTPS queries through HTTPS proxies. + + This doesn't affect regular HTTP requests, though. + """ + if not req.is_ssl(): + return + + proxy_url = req.proxy + assert proxy_url is not None + if proxy_url.scheme != "https": + return + + self._check_loop_for_start_tls() + + def _check_loop_for_start_tls(self) -> None: + try: + self._loop.start_tls + except AttributeError as attr_exc: + raise RuntimeError( + "An HTTPS request is being sent through an HTTPS proxy. " + "This needs support for TLS in TLS but it is not implemented " + "in your runtime for the stdlib asyncio.\n\n" + "Please upgrade to Python 3.11 or higher. For more details, " + "please see:\n" + "* https://bugs.python.org/issue37179\n" + "* https://github.com/python/cpython/pull/28073\n" + "* https://docs.aiohttp.org/en/stable/" + "client_advanced.html#proxy-support\n" + "* https://github.com/aio-libs/aiohttp/discussions/6044\n", + ) from attr_exc + + def _loop_supports_start_tls(self) -> bool: + try: + self._check_loop_for_start_tls() + except RuntimeError: + return False + else: + return True + + def _warn_about_tls_in_tls( + self, + underlying_transport: asyncio.Transport, + req: ClientRequest, + ) -> None: + """Issue a warning if the requested URL has HTTPS scheme.""" + if req.request_info.url.scheme != "https": + return + + # TLS-in-TLS only applies when the proxy itself is HTTPS. + # When the proxy is HTTP, start_tls upgrades a plain TCP connection, + # which is standard TLS and works on all event loops and Python versions. + if req.proxy is None or req.proxy.scheme != "https": + return + + # Check if uvloop is being used, which supports TLS in TLS, + # otherwise assume that asyncio's native transport is being used. + if type(underlying_transport).__module__.startswith("uvloop"): + return + + # Support in asyncio was added in Python 3.11 (bpo-44011) + asyncio_supports_tls_in_tls = sys.version_info >= (3, 11) or getattr( + underlying_transport, + "_start_tls_compatible", + False, + ) + + if asyncio_supports_tls_in_tls: + return + + warnings.warn( + "An HTTPS request is being sent through an HTTPS proxy. " + "This support for TLS in TLS is known to be disabled " + "in the stdlib asyncio (Python <3.11). This is why you'll probably see " + "an error in the log below.\n\n" + "It is possible to enable it via monkeypatching. " + "For more details, see:\n" + "* https://bugs.python.org/issue37179\n" + "* https://github.com/python/cpython/pull/28073\n\n" + "You can temporarily patch this as follows:\n" + "* https://docs.aiohttp.org/en/stable/client_advanced.html#proxy-support\n" + "* https://github.com/aio-libs/aiohttp/discussions/6044\n", + RuntimeWarning, + source=self, + # Why `4`? At least 3 of the calls in the stack originate + # from the methods in this class. + stacklevel=3, + ) + + async def _start_tls_connection( + self, + underlying_transport: asyncio.Transport, + req: ClientRequest, + timeout: "ClientTimeout", + client_error: type[Exception] = ClientConnectorError, + ) -> tuple[asyncio.BaseTransport, ResponseHandler]: + """Wrap the raw TCP transport with TLS.""" + tls_proto = self._factory() # Create a brand new proto for TLS + sslcontext = self._get_ssl_context(req) + if TYPE_CHECKING: + # _start_tls_connection is unreachable in the current code path + # if sslcontext is None. + assert sslcontext is not None + + try: + async with ceil_timeout( + timeout.sock_connect, ceil_threshold=timeout.ceil_threshold + ): + try: + # ssl_shutdown_timeout is only available in Python 3.11+ + if sys.version_info >= (3, 11) and self._ssl_shutdown_timeout: + tls_transport = await self._loop.start_tls( + underlying_transport, + tls_proto, + sslcontext, + server_hostname=req.server_hostname or req.host, + ssl_handshake_timeout=timeout.total or None, + ssl_shutdown_timeout=self._ssl_shutdown_timeout, + ) + else: + tls_transport = await self._loop.start_tls( + underlying_transport, + tls_proto, + sslcontext, + server_hostname=req.server_hostname or req.host, + ssl_handshake_timeout=timeout.total or None, + ) + except BaseException: + # We need to close the underlying transport since + # `start_tls()` probably failed before it had a + # chance to do this: + if self._ssl_shutdown_timeout == 0: + underlying_transport.abort() + else: + underlying_transport.close() + raise + if isinstance(tls_transport, asyncio.Transport): + fingerprint = self._get_fingerprint(req) + if fingerprint: + try: + fingerprint.check(tls_transport) + except ServerFingerprintMismatch: + tls_transport.close() + if not self._cleanup_closed_disabled: + self._cleanup_closed_transports.append(tls_transport) + raise + except cert_errors as exc: + raise ClientConnectorCertificateError(req.connection_key, exc) from exc + except ssl_errors as exc: + raise ClientConnectorSSLError(req.connection_key, exc) from exc + except OSError as exc: + if exc.errno is None and isinstance(exc, asyncio.TimeoutError): + raise + raise client_error(req.connection_key, exc) from exc + except TypeError as type_err: + # Example cause looks like this: + # TypeError: transport is not supported by start_tls() + + raise ClientConnectionError( + "Cannot initialize a TLS-in-TLS connection to host " + f"{req.host!s}:{req.port:d} through an underlying connection " + f"to an HTTPS proxy {req.proxy!s} ssl:{req.ssl or 'default'} " + f"[{type_err!s}]" + ) from type_err + else: + if tls_transport is None: + msg = "Failed to start TLS (possibly caused by closing transport)" + raise client_error(req.connection_key, OSError(msg)) + tls_proto.connection_made( + tls_transport + ) # Kick the state machine of the new TLS protocol + + return tls_transport, tls_proto + + def _convert_hosts_to_addr_infos( + self, hosts: list[ResolveResult] + ) -> list[AddrInfoType]: + """Converts the list of hosts to a list of addr_infos. + + The list of hosts is the result of a DNS lookup. The list of + addr_infos is the result of a call to `socket.getaddrinfo()`. + """ + addr_infos: list[AddrInfoType] = [] + for hinfo in hosts: + host = hinfo["host"] + is_ipv6 = ":" in host + family = socket.AF_INET6 if is_ipv6 else socket.AF_INET + if self._family and self._family != family: + continue + addr = (host, hinfo["port"], 0, 0) if is_ipv6 else (host, hinfo["port"]) + addr_infos.append( + (family, socket.SOCK_STREAM, socket.IPPROTO_TCP, "", addr) + ) + return addr_infos + + async def _create_direct_connection( + self, + req: ClientRequest, + traces: list["Trace"], + timeout: "ClientTimeout", + *, + client_error: type[Exception] = ClientConnectorError, + ) -> tuple[asyncio.Transport, ResponseHandler]: + sslcontext = self._get_ssl_context(req) + fingerprint = self._get_fingerprint(req) + + host = req.url.raw_host + assert host is not None + # Replace multiple trailing dots with a single one. + # A trailing dot is only present for fully-qualified domain names. + # See https://github.com/aio-libs/aiohttp/pull/7364. + if host.endswith(".."): + host = host.rstrip(".") + "." + port = req.port + assert port is not None + try: + # Cancelling this lookup should not cancel the underlying lookup + # or else the cancel event will get broadcast to all the waiters + # across all connections. + hosts = await self._resolve_host(host, port, traces=traces) + except OSError as exc: + if exc.errno is None and isinstance(exc, asyncio.TimeoutError): + raise + # in case of proxy it is not ClientProxyConnectionError + # it is problem of resolving proxy ip itself + raise ClientConnectorDNSError(req.connection_key, exc) from exc + + last_exc: Exception | None = None + addr_infos = self._convert_hosts_to_addr_infos(hosts) + while addr_infos: + # Strip trailing dots, certificates contain FQDN without dots. + # See https://github.com/aio-libs/aiohttp/issues/3636 + server_hostname = ( + (req.server_hostname or host).rstrip(".") if sslcontext else None + ) + + try: + transp, proto = await self._wrap_create_connection( + self._factory, + timeout=timeout, + ssl=sslcontext, + addr_infos=addr_infos, + server_hostname=server_hostname, + req=req, + client_error=client_error, + ) + except (ClientConnectorError, asyncio.TimeoutError) as exc: + last_exc = exc + aiohappyeyeballs.pop_addr_infos_interleave(addr_infos, self._interleave) + continue + + if req.is_ssl() and fingerprint: + try: + fingerprint.check(transp) + except ServerFingerprintMismatch as exc: + transp.close() + if not self._cleanup_closed_disabled: + self._cleanup_closed_transports.append(transp) + last_exc = exc + # Remove the bad peer from the list of addr_infos + sock: socket.socket = transp.get_extra_info("socket") + bad_peer = sock.getpeername() + aiohappyeyeballs.remove_addr_infos(addr_infos, bad_peer) + continue + + return transp, proto + else: + assert last_exc is not None + raise last_exc + + async def _create_proxy_connection( + self, req: ClientRequest, traces: list["Trace"], timeout: "ClientTimeout" + ) -> tuple[asyncio.BaseTransport, ResponseHandler]: + self._fail_on_no_start_tls(req) + runtime_has_start_tls = self._loop_supports_start_tls() + proxy_req = self._update_proxy_auth_header_and_build_proxy_req(req) + + # create connection to proxy server + transport, proto = await self._create_direct_connection( + proxy_req, [], timeout, client_error=ClientProxyConnectionError + ) + + if req.is_ssl(): + if runtime_has_start_tls: + self._warn_about_tls_in_tls(transport, req) + + # For HTTPS requests over HTTP proxy + # we must notify proxy to tunnel connection + # so we send CONNECT command: + # CONNECT www.python.org:443 HTTP/1.1 + # Host: www.python.org + # + # next we must do TLS handshake and so on + # to do this we must wrap raw socket into secure one + # asyncio handles this perfectly + proxy_req.method = hdrs.METH_CONNECT + proxy_req.url = req.url + key = req.connection_key._replace( + proxy=None, proxy_auth=None, proxy_headers_hash=None + ) + conn = _ConnectTunnelConnection(self, key, proto, self._loop) + proxy_resp = await proxy_req.send(conn) + try: + protocol = conn._protocol + assert protocol is not None + + # read_until_eof=True will ensure the connection isn't closed + # once the response is received and processed allowing + # START_TLS to work on the connection below. + protocol.set_response_params( + read_until_eof=runtime_has_start_tls, + timeout_ceil_threshold=self._timeout_ceil_threshold, + ) + resp = await proxy_resp.start(conn) + except BaseException: + proxy_resp.close() + conn.close() + raise + else: + conn._protocol = None + try: + if resp.status != 200: + message = resp.reason + if message is None: + message = HTTPStatus(resp.status).phrase + raise ClientHttpProxyError( + proxy_resp.request_info, + resp.history, + status=resp.status, + message=message, + headers=resp.headers, + ) + if not runtime_has_start_tls: + rawsock = transport.get_extra_info("socket", default=None) + if rawsock is None: + raise RuntimeError( + "Transport does not expose socket instance" + ) + # Duplicate the socket, so now we can close proxy transport + rawsock = rawsock.dup() + except BaseException: + # It shouldn't be closed in `finally` because it's fed to + # `loop.start_tls()` and the docs say not to touch it after + # passing there. + transport.close() + raise + finally: + if not runtime_has_start_tls: + transport.close() + + if not runtime_has_start_tls: + # HTTP proxy with support for upgrade to HTTPS + sslcontext = self._get_ssl_context(req) + return await self._wrap_existing_connection( + self._factory, + timeout=timeout, + ssl=sslcontext, + sock=rawsock, + server_hostname=req.host, + req=req, + ) + + return await self._start_tls_connection( + # Access the old transport for the last time before it's + # closed and forgotten forever: + transport, + req=req, + timeout=timeout, + ) + finally: + proxy_resp.close() + + return transport, proto + + +class UnixConnector(BaseConnector): + """Unix socket connector. + + path - Unix socket path. + keepalive_timeout - (optional) Keep-alive timeout. + force_close - Set to True to force close and do reconnect + after each request (and between redirects). + limit - The total number of simultaneous connections. + limit_per_host - Number of simultaneous connections to one host. + loop - Optional event loop. + """ + + allowed_protocol_schema_set = HIGH_LEVEL_SCHEMA_SET | frozenset({"unix"}) + + def __init__( + self, + path: str, + force_close: bool = False, + keepalive_timeout: object | float | None = sentinel, + limit: int = 100, + limit_per_host: int = 0, + loop: asyncio.AbstractEventLoop | None = None, + ) -> None: + super().__init__( + force_close=force_close, + keepalive_timeout=keepalive_timeout, + limit=limit, + limit_per_host=limit_per_host, + loop=loop, + ) + self._path = path + + @property + def path(self) -> str: + """Path to unix socket.""" + return self._path + + async def _create_connection( + self, req: ClientRequest, traces: list["Trace"], timeout: "ClientTimeout" + ) -> ResponseHandler: + try: + async with ceil_timeout( + timeout.sock_connect, ceil_threshold=timeout.ceil_threshold + ): + _, proto = await self._loop.create_unix_connection( + self._factory, self._path + ) + except OSError as exc: + if exc.errno is None and isinstance(exc, asyncio.TimeoutError): + raise + raise UnixClientConnectorError(self.path, req.connection_key, exc) from exc + + return proto + + +class NamedPipeConnector(BaseConnector): + """Named pipe connector. + + Only supported by the proactor event loop. + See also: https://docs.python.org/3/library/asyncio-eventloop.html + + path - Windows named pipe path. + keepalive_timeout - (optional) Keep-alive timeout. + force_close - Set to True to force close and do reconnect + after each request (and between redirects). + limit - The total number of simultaneous connections. + limit_per_host - Number of simultaneous connections to one host. + loop - Optional event loop. + """ + + allowed_protocol_schema_set = HIGH_LEVEL_SCHEMA_SET | frozenset({"npipe"}) + + def __init__( + self, + path: str, + force_close: bool = False, + keepalive_timeout: object | float | None = sentinel, + limit: int = 100, + limit_per_host: int = 0, + loop: asyncio.AbstractEventLoop | None = None, + ) -> None: + super().__init__( + force_close=force_close, + keepalive_timeout=keepalive_timeout, + limit=limit, + limit_per_host=limit_per_host, + loop=loop, + ) + if not isinstance( + self._loop, + asyncio.ProactorEventLoop, # type: ignore[attr-defined] + ): + raise RuntimeError( + "Named Pipes only available in proactor loop under windows" + ) + self._path = path + + @property + def path(self) -> str: + """Path to the named pipe.""" + return self._path + + async def _create_connection( + self, req: ClientRequest, traces: list["Trace"], timeout: "ClientTimeout" + ) -> ResponseHandler: + try: + async with ceil_timeout( + timeout.sock_connect, ceil_threshold=timeout.ceil_threshold + ): + _, proto = await self._loop.create_pipe_connection( # type: ignore[attr-defined] + self._factory, self._path + ) + # the drain is required so that the connection_made is called + # and transport is set otherwise it is not set before the + # `assert conn.transport is not None` + # in client.py's _request method + await asyncio.sleep(0) + # other option is to manually set transport like + # `proto.transport = trans` + except OSError as exc: + if exc.errno is None and isinstance(exc, asyncio.TimeoutError): + raise + raise ClientConnectorError(req.connection_key, exc) from exc + + return cast(ResponseHandler, proto) diff --git a/py_modules/aiohttp/cookiejar.py b/py_modules/aiohttp/cookiejar.py new file mode 100644 index 0000000..2b97224 --- /dev/null +++ b/py_modules/aiohttp/cookiejar.py @@ -0,0 +1,678 @@ +import asyncio +import calendar +import contextlib +import datetime +import heapq +import itertools +import json +import os +import pathlib +import pickle +import re +import time +import warnings +from collections import defaultdict +from collections.abc import Iterable, Iterator, Mapping +from http.cookies import BaseCookie, Morsel, SimpleCookie +from types import MappingProxyType +from typing import Union + +from yarl import URL + +from ._cookie_helpers import preserve_morsel_with_coded_value +from .abc import AbstractCookieJar, ClearCookiePredicate +from .helpers import is_ip_address +from .typedefs import LooseCookies, PathLike, StrOrURL + +__all__ = ("CookieJar", "DummyCookieJar") + + +CookieItem = Union[str, "Morsel[str]"] + +# We cache these string methods here as their use is in performance critical code. +_FORMAT_PATH = "{}/{}".format +_FORMAT_DOMAIN_REVERSED = "{1}.{0}".format + +# The minimum number of scheduled cookie expirations before we start cleaning up +# the expiration heap. This is a performance optimization to avoid cleaning up the +# heap too often when there are only a few scheduled expirations. +_MIN_SCHEDULED_COOKIE_EXPIRATION = 100 +_SIMPLE_COOKIE = SimpleCookie() + +# Not persisted; the absolute deadline is saved instead. +_RELATIVE_EXPIRY_ATTRS = frozenset(("max-age", "expires")) + + +class _RestrictedCookieUnpickler(pickle._Unpickler): + """A restricted unpickler that only allows cookie-related types. + + This prevents arbitrary code execution when loading pickled cookie data + from untrusted sources. Only types that are expected in a serialized + CookieJar are permitted. + + Subclasses :class:`pickle._Unpickler` (the pure-Python implementation) + rather than :class:`pickle.Unpickler` because the accelerated unpickler + on some implementations (notably PyPy) does not dispatch through + :meth:`find_class` overrides. + + See: https://docs.python.org/3/library/pickle.html#restricting-globals + """ + + _ALLOWED_CLASSES: frozenset[tuple[str, str]] = frozenset( + { + # Core cookie types + ("http.cookies", "SimpleCookie"), + ("http.cookies", "Morsel"), + # Container types used by CookieJar._cookies + ("collections", "defaultdict"), + # builtins that pickle uses for reconstruction + ("builtins", "tuple"), + ("builtins", "set"), + ("builtins", "frozenset"), + ("builtins", "dict"), + } + ) + + def find_class(self, module: str, name: str) -> type: + if (module, name) not in self._ALLOWED_CLASSES: + raise pickle.UnpicklingError( + f"Forbidden class: {module}.{name}. " + "CookieJar.load() only allows cookie-related types for security. " + "See https://docs.python.org/3/library/pickle.html#restricting-globals" + ) + return super().find_class(module, name) # type: ignore[no-any-return] + + +class CookieJar(AbstractCookieJar): + """Implements cookie storage adhering to RFC 6265.""" + + DATE_TOKENS_RE = re.compile( + r"[\x09\x20-\x2F\x3B-\x40\x5B-\x60\x7B-\x7E]*" + r"(?P[\x00-\x08\x0A-\x1F\d:a-zA-Z\x7F-\xFF]+)" + ) + + DATE_HMS_TIME_RE = re.compile(r"(\d{1,2}):(\d{1,2}):(\d{1,2})") + + DATE_DAY_OF_MONTH_RE = re.compile(r"(\d{1,2})") + + DATE_MONTH_RE = re.compile( + "(jan)|(feb)|(mar)|(apr)|(may)|(jun)|(jul)|(aug)|(sep)|(oct)|(nov)|(dec)", + re.I, + ) + + DATE_YEAR_RE = re.compile(r"(\d{2,4})") + + # calendar.timegm() fails for timestamps after datetime.datetime.max + # Minus one as a loss of precision occurs when timestamp() is called. + MAX_TIME = ( + int(datetime.datetime.max.replace(tzinfo=datetime.timezone.utc).timestamp()) - 1 + ) + try: + calendar.timegm(time.gmtime(MAX_TIME)) + except OSError: + # Hit the maximum representable time on Windows + # https://learn.microsoft.com/en-us/cpp/c-runtime-library/reference/localtime-localtime32-localtime64 + MAX_TIME = calendar.timegm((3000, 12, 31, 23, 59, 59, -1, -1, -1)) + except OverflowError: + # #4515: datetime.max may not be representable on 32-bit platforms + MAX_TIME = 2**31 - 1 + # Avoid minuses in the future, 3x faster + SUB_MAX_TIME = MAX_TIME - 1 + + def __init__( + self, + *, + unsafe: bool = False, + quote_cookie: bool = True, + treat_as_secure_origin: StrOrURL | list[StrOrURL] | None = None, + loop: asyncio.AbstractEventLoop | None = None, + ) -> None: + super().__init__(loop=loop) + self._cookies: defaultdict[tuple[str, str], SimpleCookie] = defaultdict( + SimpleCookie + ) + self._morsel_cache: defaultdict[tuple[str, str], dict[str, Morsel[str]]] = ( + defaultdict(dict) + ) + self._host_only_cookies: set[tuple[str, str]] = set() + self._unsafe = unsafe + self._quote_cookie = quote_cookie + if treat_as_secure_origin is None: + treat_as_secure_origin = [] + elif isinstance(treat_as_secure_origin, URL): + treat_as_secure_origin = [treat_as_secure_origin.origin()] + elif isinstance(treat_as_secure_origin, str): + treat_as_secure_origin = [URL(treat_as_secure_origin).origin()] + else: + treat_as_secure_origin = [ + URL(url).origin() if isinstance(url, str) else url.origin() + for url in treat_as_secure_origin + ] + self._treat_as_secure_origin = treat_as_secure_origin + self._expire_heap: list[tuple[float, tuple[str, str, str]]] = [] + self._expirations: dict[tuple[str, str, str], float] = {} + + @property + def unsafe(self) -> bool: + return self._unsafe + + @property + def quote_cookie(self) -> bool: + return self._quote_cookie + + @property + def cookies(self) -> MappingProxyType[tuple[str, str], SimpleCookie]: + """Return the cookies stored in this jar.""" + return MappingProxyType(self._cookies) + + @property + def host_only_cookies(self) -> frozenset[tuple[str, str]]: + """Return the host-only cookies stored in this jar.""" + return frozenset(self._host_only_cookies) + + def save(self, file_path: PathLike) -> None: + """Save cookies to a file using JSON format. + + :param file_path: Path to file where cookies will be serialized, + :class:`str` or :class:`pathlib.Path` instance. + """ + file_path = pathlib.Path(file_path) + data: dict[str, dict[str, dict[str, str | bool | float]]] = {} + for (domain, path), cookie in self._cookies.items(): + key = f"{domain}|{path}" + data[key] = {} + for name, morsel in cookie.items(): + morsel_data: dict[str, str | bool | float] = { + "key": morsel.key, + "value": morsel.value, + "coded_value": morsel.coded_value, + } + # Skip relative expiry; the absolute deadline is saved below. + for attr in morsel._reserved: # type: ignore[attr-defined] + if attr in _RELATIVE_EXPIRY_ATTRS: + continue + attr_val = morsel[attr] + if attr_val: + morsel_data[attr] = attr_val + # Persist or it reloads as a domain cookie and leaks to subdomains. + if (domain, name) in self._host_only_cookies: + morsel_data["host_only"] = True + if (exp := self._expirations.get((domain, path, name))) is not None: + morsel_data["expires_timestamp"] = exp + data[key][name] = morsel_data + + # Cookie persistence may include authentication/session tokens. + # Use 0o600 at creation time to avoid umask-dependent overexposure + # and enforce least-privilege access to sensitive credential data. + with open( + file_path, + mode="w", + encoding="utf-8", + opener=lambda path, flags: os.open(path, flags, 0o600), + ) as f: + json.dump(data, f, indent=2) + + def load(self, file_path: PathLike) -> None: + """Load cookies from a file. + + Tries to load JSON format first. Falls back to loading legacy + pickle format (using a restricted unpickler) for backward + compatibility with existing cookie files. + + Replaces the current jar contents; loaded cookies pass through the + same acceptance rules as :meth:`update_cookies`. + + :param file_path: Path to file from where cookies will be + imported, :class:`str` or :class:`pathlib.Path` instance. + """ + file_path = pathlib.Path(file_path) + # Try JSON format first + try: + with file_path.open(mode="r", encoding="utf-8") as f: + data = json.load(f) + self._load_json_data(data) + except (json.JSONDecodeError, UnicodeDecodeError, ValueError): + # Fall back to legacy pickle format with restricted unpickler + with file_path.open(mode="rb") as f: + self._cookies = _RestrictedCookieUnpickler(f).load() + + def _load_json_data( + self, data: dict[str, dict[str, dict[str, str | bool | float]]] + ) -> None: + """Replace contents, routing cookies through update_cookies().""" + self.clear() + for compound_key, cookie_data in data.items(): + domain, path = compound_key.split("|", 1) + for name, morsel_data in cookie_data.items(): + morsel: Morsel[str] = Morsel() + # Use __setstate__ to bypass validation, same pattern + # used in _build_morsel and _cookie_helpers. + morsel.__setstate__( # type: ignore[attr-defined] + { + "key": morsel_data["key"], + "value": morsel_data["value"], + "coded_value": morsel_data["coded_value"], + } + ) + # Restore morsel attributes + for attr in morsel._reserved: # type: ignore[attr-defined] + if attr in morsel_data and attr not in ( + "key", + "value", + "coded_value", + ): + morsel[attr] = morsel_data[attr] + # Drop the domain so update_cookies() re-marks it host-only. + if morsel_data.get("host_only"): + morsel["domain"] = "" + response_url = ( + URL.build(scheme="https", host=domain) if domain else URL() + ) + self.update_cookies({name: morsel}, response_url) + # Restore the absolute deadline; update_cookies() schedules none. + if (exp := morsel_data.get("expires_timestamp")) is not None: + self._expire_cookie(float(exp), domain, path, name) + self._do_expiration() + + def clear(self, predicate: ClearCookiePredicate | None = None) -> None: + if predicate is None: + self._expire_heap.clear() + self._cookies.clear() + self._morsel_cache.clear() + self._host_only_cookies.clear() + self._expirations.clear() + return + + now = time.time() + to_del = [ + key + for (domain, path), cookie in self._cookies.items() + for name, morsel in cookie.items() + if ( + (key := (domain, path, name)) in self._expirations + and self._expirations[key] <= now + ) + or predicate(morsel) + ] + if to_del: + self._delete_cookies(to_del) + + def clear_domain(self, domain: str) -> None: + self.clear(lambda x: self._is_domain_match(domain, x["domain"])) + + def __iter__(self) -> "Iterator[Morsel[str]]": + self._do_expiration() + for val in self._cookies.values(): + yield from val.values() + + def __len__(self) -> int: + """Return number of cookies. + + This function does not iterate self to avoid unnecessary expiration + checks. + """ + return sum(len(cookie.values()) for cookie in self._cookies.values()) + + def _do_expiration(self) -> None: + """Remove expired cookies.""" + if not (expire_heap_len := len(self._expire_heap)): + return + + # If the expiration heap grows larger than the number expirations + # times two, we clean it up to avoid keeping expired entries in + # the heap and consuming memory. We guard this with a minimum + # threshold to avoid cleaning up the heap too often when there are + # only a few scheduled expirations. + if ( + expire_heap_len > _MIN_SCHEDULED_COOKIE_EXPIRATION + and expire_heap_len > len(self._expirations) * 2 + ): + # Remove any expired entries from the expiration heap + # that do not match the expiration time in the expirations + # as it means the cookie has been re-added to the heap + # with a different expiration time. + self._expire_heap = [ + entry + for entry in self._expire_heap + if self._expirations.get(entry[1]) == entry[0] + ] + heapq.heapify(self._expire_heap) + + now = time.time() + to_del: list[tuple[str, str, str]] = [] + # Find any expired cookies and add them to the to-delete list + while self._expire_heap: + when, cookie_key = self._expire_heap[0] + if when > now: + break + heapq.heappop(self._expire_heap) + # Check if the cookie hasn't been re-added to the heap + # with a different expiration time as it will be removed + # later when it reaches the top of the heap and its + # expiration time is met. + if self._expirations.get(cookie_key) == when: + to_del.append(cookie_key) + + if to_del: + self._delete_cookies(to_del) + + def _delete_cookies(self, to_del: list[tuple[str, str, str]]) -> None: + for domain, path, name in to_del: + self._host_only_cookies.discard((domain, name)) + self._cookies[(domain, path)].pop(name, None) + self._morsel_cache[(domain, path)].pop(name, None) + self._expirations.pop((domain, path, name), None) + + def _expire_cookie(self, when: float, domain: str, path: str, name: str) -> None: + cookie_key = (domain, path, name) + if self._expirations.get(cookie_key) == when: + # Avoid adding duplicates to the heap + return + heapq.heappush(self._expire_heap, (when, cookie_key)) + self._expirations[cookie_key] = when + + def update_cookies(self, cookies: LooseCookies, response_url: URL = URL()) -> None: + """Update cookies.""" + hostname = response_url.raw_host + + if not self._unsafe and is_ip_address(hostname): + # Don't accept cookies from IPs + return + + if isinstance(cookies, Mapping): + cookies = cookies.items() + + for name, cookie in cookies: + if not isinstance(cookie, Morsel): + tmp = SimpleCookie() + tmp[name] = cookie # type: ignore[assignment] + cookie = tmp[name] + + domain = cookie["domain"] + + # ignore domains with trailing dots + if domain and domain[-1] == ".": + domain = "" + del cookie["domain"] + + if not domain and hostname is not None: + # Set the cookie's domain to the response hostname + # and set its host-only-flag + self._host_only_cookies.add((hostname, name)) + domain = cookie["domain"] = hostname + + if domain and domain[0] == ".": + # Remove leading dot + domain = domain[1:] + cookie["domain"] = domain + + if hostname and not self._is_domain_match(domain, hostname): + # Setting cookies for different domains is not allowed + continue + + path = cookie["path"] + if not path or path[0] != "/": + # Set the cookie's path to the response path + path = response_url.path + if not path.startswith("/"): + path = "/" + else: + # Cut everything from the last slash to the end + path = "/" + path[1 : path.rfind("/")] + cookie["path"] = path + path = path.rstrip("/") + + if max_age := cookie["max-age"]: + try: + delta_seconds = int(max_age) + max_age_expiration = min(time.time() + delta_seconds, self.MAX_TIME) + self._expire_cookie(max_age_expiration, domain, path, name) + except ValueError: + cookie["max-age"] = "" + + elif expires := cookie["expires"]: + if expire_time := self._parse_date(expires): + self._expire_cookie(expire_time, domain, path, name) + else: + cookie["expires"] = "" + + key = (domain, path) + if self._cookies[key].get(name) != cookie: + # Don't blow away the cache if the same + # cookie gets set again + self._cookies[key][name] = cookie + self._morsel_cache[key].pop(name, None) + + self._do_expiration() + + def filter_cookies(self, request_url: URL = URL()) -> "BaseCookie[str]": + """Returns this jar's cookies filtered by their attributes.""" + # We always use BaseCookie now since all + # cookies set on on filtered are fully constructed + # Morsels, not just names and values. + filtered: BaseCookie[str] = BaseCookie() + if not self._cookies: + # Skip do_expiration() if there are no cookies. + return filtered + self._do_expiration() + if not self._cookies: + # Skip rest of function if no non-expired cookies. + return filtered + if type(request_url) is not URL: + warnings.warn( + "filter_cookies expects yarl.URL instances only," + f"and will stop working in 4.x, got {type(request_url)}", + DeprecationWarning, + stacklevel=2, + ) + request_url = URL(request_url) + hostname = request_url.raw_host or "" + + is_not_secure = request_url.scheme not in ("https", "wss") + if is_not_secure and self._treat_as_secure_origin: + request_origin = URL() + with contextlib.suppress(ValueError): + request_origin = request_url.origin() + is_not_secure = request_origin not in self._treat_as_secure_origin + + # Send shared cookie + key = ("", "") + for c in self._cookies[key].values(): + # Check cache first + if c.key in self._morsel_cache[key]: + filtered[c.key] = self._morsel_cache[key][c.key] + continue + + # Build and cache the morsel + mrsl_val = self._build_morsel(c) + self._morsel_cache[key][c.key] = mrsl_val + filtered[c.key] = mrsl_val + + if is_ip_address(hostname): + if not self._unsafe: + return filtered + domains: Iterable[str] = (hostname,) + else: + # Get all the subdomains that might match a cookie (e.g. "foo.bar.com", "bar.com", "com") + domains = itertools.accumulate( + reversed(hostname.split(".")), _FORMAT_DOMAIN_REVERSED + ) + + # Get all the path prefixes that might match a cookie (e.g. "", "/foo", "/foo/bar") + paths = itertools.accumulate(request_url.path.split("/"), _FORMAT_PATH) + # Create every combination of (domain, path) pairs. + pairs = itertools.product(domains, paths) + + path_len = len(request_url.path) + # Point 2: https://www.rfc-editor.org/rfc/rfc6265.html#section-5.4 + for p in pairs: + if p not in self._cookies: + continue + for name, cookie in self._cookies[p].items(): + domain = cookie["domain"] + + if (domain, name) in self._host_only_cookies and domain != hostname: + continue + + # Skip edge case when the cookie has a trailing slash but request doesn't. + if len(cookie["path"]) > path_len: + continue + + if is_not_secure and cookie["secure"]: + continue + + # We already built the Morsel so reuse it here + if name in self._morsel_cache[p]: + filtered[name] = self._morsel_cache[p][name] + continue + + # Build and cache the morsel + mrsl_val = self._build_morsel(cookie) + self._morsel_cache[p][name] = mrsl_val + filtered[name] = mrsl_val + + return filtered + + def _build_morsel(self, cookie: Morsel[str]) -> Morsel[str]: + """Build a morsel for sending, respecting quote_cookie setting.""" + if self._quote_cookie and cookie.coded_value and cookie.coded_value[0] == '"': + return preserve_morsel_with_coded_value(cookie) + morsel: Morsel[str] = Morsel() + if self._quote_cookie: + value, coded_value = _SIMPLE_COOKIE.value_encode(cookie.value) + else: + coded_value = value = cookie.value + # We use __setstate__ instead of the public set() API because it allows us to + # bypass validation and set already validated state. This is more stable than + # setting protected attributes directly and unlikely to change since it would + # break pickling. + morsel.__setstate__({"key": cookie.key, "value": value, "coded_value": coded_value}) # type: ignore[attr-defined] + return morsel + + @staticmethod + def _is_domain_match(domain: str, hostname: str) -> bool: + """Implements domain matching adhering to RFC 6265.""" + if hostname == domain: + return True + + if not hostname.endswith(domain): + return False + + non_matching = hostname[: -len(domain)] + + if not non_matching.endswith("."): + return False + + return not is_ip_address(hostname) + + @classmethod + def _parse_date(cls, date_str: str) -> int | None: + """Implements date string parsing adhering to RFC 6265.""" + if not date_str: + return None + + found_time = False + found_day = False + found_month = False + found_year = False + + hour = minute = second = 0 + day = 0 + month = 0 + year = 0 + + for token_match in cls.DATE_TOKENS_RE.finditer(date_str): + + token = token_match.group("token") + + if not found_time: + time_match = cls.DATE_HMS_TIME_RE.match(token) + if time_match: + found_time = True + hour, minute, second = (int(s) for s in time_match.groups()) + continue + + if not found_day: + day_match = cls.DATE_DAY_OF_MONTH_RE.match(token) + if day_match: + found_day = True + day = int(day_match.group()) + continue + + if not found_month: + month_match = cls.DATE_MONTH_RE.match(token) + if month_match: + found_month = True + assert month_match.lastindex is not None + month = month_match.lastindex + continue + + if not found_year: + year_match = cls.DATE_YEAR_RE.match(token) + if year_match: + found_year = True + year = int(year_match.group()) + + if 70 <= year <= 99: + year += 1900 + elif 0 <= year <= 69: + year += 2000 + + if False in (found_day, found_month, found_year, found_time): + return None + + if not 1 <= day <= 31: + return None + + if year < 1601 or hour > 23 or minute > 59 or second > 59: + return None + + return calendar.timegm((year, month, day, hour, minute, second, -1, -1, -1)) + + +class DummyCookieJar(AbstractCookieJar): + """Implements a dummy cookie storage. + + It can be used with the ClientSession when no cookie processing is needed. + + """ + + def __init__(self, *, loop: asyncio.AbstractEventLoop | None = None) -> None: + super().__init__(loop=loop) + + def __iter__(self) -> "Iterator[Morsel[str]]": + while False: + yield None + + def __len__(self) -> int: + return 0 + + @property + def unsafe(self) -> bool: + return False + + @property + def quote_cookie(self) -> bool: + return True + + @property + def cookies(self) -> MappingProxyType[tuple[str, str], SimpleCookie]: + """Return an empty mapping.""" + return MappingProxyType({}) + + @property + def host_only_cookies(self) -> frozenset[tuple[str, str]]: + """Return an empty frozenset.""" + return frozenset() + + def clear(self, predicate: ClearCookiePredicate | None = None) -> None: + pass + + def clear_domain(self, domain: str) -> None: + pass + + def update_cookies(self, cookies: LooseCookies, response_url: URL = URL()) -> None: + pass + + def filter_cookies(self, request_url: URL) -> "BaseCookie[str]": + return SimpleCookie() diff --git a/py_modules/aiohttp/formdata.py b/py_modules/aiohttp/formdata.py new file mode 100644 index 0000000..f602cdf --- /dev/null +++ b/py_modules/aiohttp/formdata.py @@ -0,0 +1,184 @@ +import io +import warnings +from collections.abc import Iterable +from typing import Any +from urllib.parse import urlencode + +from multidict import MultiDict, MultiDictProxy + +from . import hdrs, multipart, payload +from .helpers import guess_filename +from .http_writer import _safe_header +from .payload import Payload + +__all__ = ("FormData",) + + +class FormData: + """Helper class for form body generation. + + Supports multipart/form-data and application/x-www-form-urlencoded. + """ + + def __init__( + self, + fields: Iterable[Any] = (), + quote_fields: bool = True, + charset: str | None = None, + *, + default_to_multipart: bool = False, + ) -> None: + self._writer = multipart.MultipartWriter("form-data") + self._fields: list[Any] = [] + self._is_multipart = default_to_multipart + self._quote_fields = quote_fields + self._charset = charset + + if isinstance(fields, dict): + fields = list(fields.items()) + elif not isinstance(fields, (list, tuple)): + fields = (fields,) + self.add_fields(*fields) + + @property + def is_multipart(self) -> bool: + return self._is_multipart + + def add_field( + self, + name: str, + value: Any, + *, + content_type: str | None = None, + filename: str | None = None, + content_transfer_encoding: str | None = None, + ) -> None: + + if isinstance(value, io.IOBase): + self._is_multipart = True + elif isinstance(value, (bytes, bytearray, memoryview)): + msg = ( + "In v4, passing bytes will no longer create a file field. " + "Please explicitly use the filename parameter or pass a BytesIO object." + ) + if filename is None and content_transfer_encoding is None: + warnings.warn(msg, DeprecationWarning) + filename = name + + _safe_header(name) + type_options: MultiDict[str] = MultiDict({"name": name}) + if filename is not None and not isinstance(filename, str): + raise TypeError("filename must be an instance of str. Got: %s" % filename) + if filename is None and isinstance(value, io.IOBase): + filename = guess_filename(value, name) + if filename is not None: + _safe_header(filename) + type_options["filename"] = filename + self._is_multipart = True + + headers = {} + if content_type is not None: + if not isinstance(content_type, str): + raise TypeError( + "content_type must be an instance of str. Got: %s" % content_type + ) + _safe_header(content_type) + headers[hdrs.CONTENT_TYPE] = content_type + self._is_multipart = True + if content_transfer_encoding is not None: + if not isinstance(content_transfer_encoding, str): + raise TypeError( + "content_transfer_encoding must be an instance" + " of str. Got: %s" % content_transfer_encoding + ) + msg = ( + "content_transfer_encoding is deprecated. " + "To maintain compatibility with v4 please pass a BytesPayload." + ) + warnings.warn(msg, DeprecationWarning) + self._is_multipart = True + + self._fields.append((type_options, headers, value)) + + def add_fields(self, *fields: Any) -> None: + to_add = list(fields) + + while to_add: + rec = to_add.pop(0) + + if isinstance(rec, io.IOBase): + k = guess_filename(rec, "unknown") + self.add_field(k, rec) # type: ignore[arg-type] + + elif isinstance(rec, (MultiDictProxy, MultiDict)): + to_add.extend(rec.items()) + + elif isinstance(rec, (list, tuple)) and len(rec) == 2: + k, fp = rec + self.add_field(k, fp) + + else: + raise TypeError( + "Only io.IOBase, multidict and (name, file) " + "pairs allowed, use .add_field() for passing " + f"more complex parameters, got {rec!r}" + ) + + def _gen_form_urlencoded(self) -> payload.BytesPayload: + # form data (x-www-form-urlencoded) + data = [] + for type_options, _, value in self._fields: + data.append((type_options["name"], value)) + + charset = self._charset if self._charset is not None else "utf-8" + + if charset == "utf-8": + content_type = "application/x-www-form-urlencoded" + else: + content_type = "application/x-www-form-urlencoded; charset=%s" % charset + + return payload.BytesPayload( + urlencode(data, doseq=True, encoding=charset).encode(), + content_type=content_type, + ) + + def _gen_form_data(self) -> multipart.MultipartWriter: + """Encode a list of fields using the multipart/form-data MIME format""" + for dispparams, headers, value in self._fields: + try: + if hdrs.CONTENT_TYPE in headers: + part = payload.get_payload( + value, + content_type=headers[hdrs.CONTENT_TYPE], + headers=headers, + encoding=self._charset, + ) + else: + part = payload.get_payload( + value, headers=headers, encoding=self._charset + ) + except Exception as exc: + raise TypeError( + "Can not serialize value type: %r\n " + "headers: %r\n value: %r" % (type(value), headers, value) + ) from exc + + if dispparams: + part.set_content_disposition( + "form-data", quote_fields=self._quote_fields, **dispparams + ) + # FIXME cgi.FieldStorage doesn't likes body parts with + # Content-Length which were sent via chunked transfer encoding + assert part.headers is not None + part.headers.popall(hdrs.CONTENT_LENGTH, None) + + self._writer.append_payload(part) + + self._fields.clear() + return self._writer + + def __call__(self) -> Payload: + if self._is_multipart: + return self._gen_form_data() + else: + return self._gen_form_urlencoded() diff --git a/py_modules/aiohttp/hdrs.py b/py_modules/aiohttp/hdrs.py new file mode 100644 index 0000000..1082da3 --- /dev/null +++ b/py_modules/aiohttp/hdrs.py @@ -0,0 +1,115 @@ +"""HTTP Headers constants.""" + +# After changing the file content call ./tools/gen.py +# to regenerate the headers parser +import itertools +from typing import Final + +from multidict import istr + +METH_ANY: Final[str] = "*" +METH_CONNECT: Final[str] = "CONNECT" +METH_HEAD: Final[str] = "HEAD" +METH_GET: Final[str] = "GET" +METH_DELETE: Final[str] = "DELETE" +METH_OPTIONS: Final[str] = "OPTIONS" +METH_PATCH: Final[str] = "PATCH" +METH_POST: Final[str] = "POST" +METH_PUT: Final[str] = "PUT" +METH_TRACE: Final[str] = "TRACE" + +METH_ALL: Final[set[str]] = { + METH_CONNECT, + METH_HEAD, + METH_GET, + METH_DELETE, + METH_OPTIONS, + METH_PATCH, + METH_POST, + METH_PUT, + METH_TRACE, +} + +ACCEPT: Final[istr] = istr("Accept") +ACCEPT_CHARSET: Final[istr] = istr("Accept-Charset") +ACCEPT_ENCODING: Final[istr] = istr("Accept-Encoding") +ACCEPT_LANGUAGE: Final[istr] = istr("Accept-Language") +ACCEPT_RANGES: Final[istr] = istr("Accept-Ranges") +ACCESS_CONTROL_MAX_AGE: Final[istr] = istr("Access-Control-Max-Age") +ACCESS_CONTROL_ALLOW_CREDENTIALS: Final[istr] = istr("Access-Control-Allow-Credentials") +ACCESS_CONTROL_ALLOW_HEADERS: Final[istr] = istr("Access-Control-Allow-Headers") +ACCESS_CONTROL_ALLOW_METHODS: Final[istr] = istr("Access-Control-Allow-Methods") +ACCESS_CONTROL_ALLOW_ORIGIN: Final[istr] = istr("Access-Control-Allow-Origin") +ACCESS_CONTROL_EXPOSE_HEADERS: Final[istr] = istr("Access-Control-Expose-Headers") +ACCESS_CONTROL_REQUEST_HEADERS: Final[istr] = istr("Access-Control-Request-Headers") +ACCESS_CONTROL_REQUEST_METHOD: Final[istr] = istr("Access-Control-Request-Method") +AGE: Final[istr] = istr("Age") +ALLOW: Final[istr] = istr("Allow") +AUTHORIZATION: Final[istr] = istr("Authorization") +CACHE_CONTROL: Final[istr] = istr("Cache-Control") +CONNECTION: Final[istr] = istr("Connection") +CONTENT_DISPOSITION: Final[istr] = istr("Content-Disposition") +CONTENT_ENCODING: Final[istr] = istr("Content-Encoding") +CONTENT_LANGUAGE: Final[istr] = istr("Content-Language") +CONTENT_LENGTH: Final[istr] = istr("Content-Length") +CONTENT_LOCATION: Final[istr] = istr("Content-Location") +CONTENT_MD5: Final[istr] = istr("Content-MD5") +CONTENT_RANGE: Final[istr] = istr("Content-Range") +CONTENT_TRANSFER_ENCODING: Final[istr] = istr("Content-Transfer-Encoding") +CONTENT_TYPE: Final[istr] = istr("Content-Type") +COOKIE: Final[istr] = istr("Cookie") +DATE: Final[istr] = istr("Date") +DESTINATION: Final[istr] = istr("Destination") +DIGEST: Final[istr] = istr("Digest") +ETAG: Final[istr] = istr("Etag") +EXPECT: Final[istr] = istr("Expect") +EXPIRES: Final[istr] = istr("Expires") +FORWARDED: Final[istr] = istr("Forwarded") +FROM: Final[istr] = istr("From") +HOST: Final[istr] = istr("Host") +IF_MATCH: Final[istr] = istr("If-Match") +IF_MODIFIED_SINCE: Final[istr] = istr("If-Modified-Since") +IF_NONE_MATCH: Final[istr] = istr("If-None-Match") +IF_RANGE: Final[istr] = istr("If-Range") +IF_UNMODIFIED_SINCE: Final[istr] = istr("If-Unmodified-Since") +KEEP_ALIVE: Final[istr] = istr("Keep-Alive") +LAST_EVENT_ID: Final[istr] = istr("Last-Event-ID") +LAST_MODIFIED: Final[istr] = istr("Last-Modified") +LINK: Final[istr] = istr("Link") +LOCATION: Final[istr] = istr("Location") +MAX_FORWARDS: Final[istr] = istr("Max-Forwards") +ORIGIN: Final[istr] = istr("Origin") +PRAGMA: Final[istr] = istr("Pragma") +PROXY_AUTHENTICATE: Final[istr] = istr("Proxy-Authenticate") +PROXY_AUTHORIZATION: Final[istr] = istr("Proxy-Authorization") +RANGE: Final[istr] = istr("Range") +REFERER: Final[istr] = istr("Referer") +RETRY_AFTER: Final[istr] = istr("Retry-After") +SEC_WEBSOCKET_ACCEPT: Final[istr] = istr("Sec-WebSocket-Accept") +SEC_WEBSOCKET_VERSION: Final[istr] = istr("Sec-WebSocket-Version") +SEC_WEBSOCKET_PROTOCOL: Final[istr] = istr("Sec-WebSocket-Protocol") +SEC_WEBSOCKET_EXTENSIONS: Final[istr] = istr("Sec-WebSocket-Extensions") +SEC_WEBSOCKET_KEY: Final[istr] = istr("Sec-WebSocket-Key") +SEC_WEBSOCKET_KEY1: Final[istr] = istr("Sec-WebSocket-Key1") +SERVER: Final[istr] = istr("Server") +SET_COOKIE: Final[istr] = istr("Set-Cookie") +TE: Final[istr] = istr("TE") +TRAILER: Final[istr] = istr("Trailer") +TRANSFER_ENCODING: Final[istr] = istr("Transfer-Encoding") +UPGRADE: Final[istr] = istr("Upgrade") +URI: Final[istr] = istr("URI") +USER_AGENT: Final[istr] = istr("User-Agent") +VARY: Final[istr] = istr("Vary") +VIA: Final[istr] = istr("Via") +WANT_DIGEST: Final[istr] = istr("Want-Digest") +WARNING: Final[istr] = istr("Warning") +WWW_AUTHENTICATE: Final[istr] = istr("WWW-Authenticate") +X_FORWARDED_FOR: Final[istr] = istr("X-Forwarded-For") +X_FORWARDED_HOST: Final[istr] = istr("X-Forwarded-Host") +X_FORWARDED_PROTO: Final[istr] = istr("X-Forwarded-Proto") + +# Case permutations of the Host header — for callers that match against +# raw header tokens before istr/CIMultiDict folding. +HOST_ALL: Final = frozenset( + map("".join, itertools.product(*zip(HOST.upper(), HOST.lower()))) +) diff --git a/py_modules/aiohttp/helpers.py b/py_modules/aiohttp/helpers.py new file mode 100644 index 0000000..bf905ce --- /dev/null +++ b/py_modules/aiohttp/helpers.py @@ -0,0 +1,1046 @@ +"""Various helper functions""" + +import asyncio +import base64 +import binascii +import contextlib +import datetime +import enum +import functools +import inspect +import netrc +import os +import platform +import re +import sys +import time +import warnings +import weakref +from collections import namedtuple +from collections.abc import Callable, Generator, Iterable, Iterator, Mapping +from contextlib import suppress +from email.message import EmailMessage +from email.parser import HeaderParser +from email.policy import HTTP +from email.utils import parsedate +from math import ceil +from pathlib import Path +from types import MappingProxyType, TracebackType +from typing import ( + Any, + ContextManager, + Generic, + Optional, + Protocol, + TypeVar, + get_args, + overload, +) +from urllib.parse import quote +from urllib.request import getproxies, proxy_bypass + +import attr +from multidict import MultiDict, MultiDictProxy, MultiMapping +from propcache.api import under_cached_property as reify +from yarl import URL + +from . import hdrs +from .log import client_logger + +if sys.version_info >= (3, 11): + import asyncio as async_timeout +else: + import async_timeout + +__all__ = ("BasicAuth", "ChainMapProxy", "ETag", "reify") + +IS_MACOS = platform.system() == "Darwin" +IS_WINDOWS = platform.system() == "Windows" + +PY_311 = sys.version_info >= (3, 11) + +# This is the default size/limit for several operations. +# Matches the max size we receive from sockets: +# https://github.com/python/cpython/blob/1857a40807daeae3a1bf5efb682de9c9ae6df845/Lib/asyncio/selector_events.py#L766 +DEFAULT_CHUNK_SIZE = 2**18 # 256 KiB + +_T = TypeVar("_T") +_S = TypeVar("_S") + +_SENTINEL = enum.Enum("_SENTINEL", "sentinel") +sentinel = _SENTINEL.sentinel + +NO_EXTENSIONS = bool(os.environ.get("AIOHTTP_NO_EXTENSIONS")) + +# https://datatracker.ietf.org/doc/html/rfc9112#section-6.3-2.1 +EMPTY_BODY_STATUS_CODES = frozenset((204, 304, *range(100, 200))) +# https://datatracker.ietf.org/doc/html/rfc9112#section-6.3-2.1 +# https://datatracker.ietf.org/doc/html/rfc9112#section-6.3-2.2 +EMPTY_BODY_METHODS = frozenset({hdrs.METH_HEAD}) + +DEBUG = sys.flags.dev_mode or ( + not sys.flags.ignore_environment and bool(os.environ.get("PYTHONASYNCIODEBUG")) +) + + +CHAR = {chr(i) for i in range(0, 128)} +CTL = {chr(i) for i in range(0, 32)} | { + chr(127), +} +SEPARATORS = { + "(", + ")", + "<", + ">", + "@", + ",", + ";", + ":", + "\\", + '"', + "/", + "[", + "]", + "?", + "=", + "{", + "}", + " ", + chr(9), +} +TOKEN = CHAR ^ CTL ^ SEPARATORS + + +class noop: + def __await__(self) -> Generator[None, None, None]: + yield + + +def encode_basic_auth(login: str, password: str = "", encoding: str = "utf-8") -> str: + """Encode HTTP Basic Authentication credentials as an Authorization header value. + + Returns a string of the form ``"Basic "`` suitable for use as the + value of the ``Authorization`` (or ``Proxy-Authorization``) header. + """ + if ":" in login: + raise ValueError('A ":" is not allowed in login (RFC 7617#section-2)') + creds = f"{login}:{password}".encode(encoding) + return "Basic " + base64.b64encode(creds).decode(encoding) + + +class BasicAuth(namedtuple("BasicAuth", ["login", "password", "encoding"])): + """Http basic authentication helper.""" + + def __new__( + cls, login: str, password: str = "", encoding: str = "latin1" + ) -> "BasicAuth": + if login is None: + raise ValueError("None is not allowed as login value") + + if password is None: + raise ValueError("None is not allowed as password value") + + if ":" in login: + raise ValueError('A ":" is not allowed in login (RFC 1945#section-11.1)') + + warnings.warn( + "BasicAuth is deprecated and will be removed in aiohttp 4.0; " + "use aiohttp.encode_basic_auth() with " + "headers={'Authorization': ...} instead", + DeprecationWarning, + stacklevel=2, + ) + return super().__new__(cls, login, password, encoding) + + @classmethod + def decode(cls, auth_header: str, encoding: str = "latin1") -> "BasicAuth": + """Create a BasicAuth object from an Authorization HTTP header.""" + try: + auth_type, encoded_credentials = auth_header.split(" ", 1) + except ValueError: + raise ValueError("Could not parse authorization header.") + + if auth_type.lower() != "basic": + raise ValueError("Unknown authorization method %s" % auth_type) + + try: + decoded = base64.b64decode( + encoded_credentials.encode("ascii"), validate=True + ).decode(encoding) + except binascii.Error: + raise ValueError("Invalid base64 encoding.") + + try: + # RFC 2617 HTTP Authentication + # https://www.ietf.org/rfc/rfc2617.txt + # the colon must be present, but the username and password may be + # otherwise blank. + username, password = decoded.split(":", 1) + except ValueError: + raise ValueError("Invalid credentials.") + + return _basic_auth_no_warn(username, password, encoding) + + @classmethod + def from_url(cls, url: URL, *, encoding: str = "latin1") -> Optional["BasicAuth"]: + """Create BasicAuth from url.""" + if not isinstance(url, URL): + raise TypeError("url should be yarl.URL instance") + # Check raw_user and raw_password first as yarl is likely + # to already have these values parsed from the netloc in the cache. + if url.raw_user is None and url.raw_password is None: + return None + return _basic_auth_no_warn(url.user or "", url.password or "", encoding) + + def encode(self) -> str: + """Encode credentials.""" + return encode_basic_auth(self.login, self.password, self.encoding) + + +def _basic_auth_no_warn( + login: str, password: str = "", encoding: str = "latin1" +) -> BasicAuth: + """Construct a BasicAuth without emitting the deprecation warning. + + For internal use only. Bypasses BasicAuth.__new__ so that aiohttp's own + machinery doesn't trigger deprecation warnings in user code. + """ + return tuple.__new__(BasicAuth, (login, password, encoding)) + + +def strip_auth_from_url(url: URL) -> tuple[URL, BasicAuth | None]: + """Remove user and password from URL if present and return BasicAuth object.""" + # Check raw_user and raw_password first as yarl is likely + # to already have these values parsed from the netloc in the cache. + if url.raw_user is None and url.raw_password is None: + return url, None + return url.with_user(None), _basic_auth_no_warn(url.user or "", url.password or "") + + +def netrc_from_env() -> netrc.netrc | None: + """Load netrc from file. + + Attempt to load it from the path specified by the env-var + NETRC or in the default location in the user's home directory. + + Returns None if it couldn't be found or fails to parse. + """ + netrc_env = os.environ.get("NETRC") + + if netrc_env is not None: + netrc_path = Path(netrc_env) + else: + try: + home_dir = Path.home() + except RuntimeError as e: # pragma: no cover + # if pathlib can't resolve home, it may raise a RuntimeError + client_logger.debug( + "Could not resolve home directory when " + "trying to look for .netrc file: %s", + e, + ) + return None + + netrc_path = home_dir / ("_netrc" if IS_WINDOWS else ".netrc") + + try: + return netrc.netrc(str(netrc_path)) + except netrc.NetrcParseError as e: + client_logger.warning("Could not parse .netrc file: %s", e) + except OSError as e: + netrc_exists = False + with contextlib.suppress(OSError): + netrc_exists = netrc_path.is_file() + # we couldn't read the file (doesn't exist, permissions, etc.) + if netrc_env or netrc_exists: + # only warn if the environment wanted us to load it, + # or it appears like the default file does actually exist + client_logger.warning("Could not read .netrc file: %s", e) + + return None + + +@attr.s(auto_attribs=True, frozen=True, slots=True) +class ProxyInfo: + proxy: URL + proxy_auth: BasicAuth | None + + +def basicauth_from_netrc(netrc_obj: netrc.netrc | None, host: str) -> BasicAuth: + """ + Return :py:class:`~aiohttp.BasicAuth` credentials for ``host`` from ``netrc_obj``. + + :raises LookupError: if ``netrc_obj`` is :py:data:`None` or if no + entry is found for the ``host``. + """ + if netrc_obj is None: + raise LookupError("No .netrc file found") + auth_from_netrc = netrc_obj.authenticators(host) + + if auth_from_netrc is None: + raise LookupError(f"No entry for {host!s} found in the `.netrc` file.") + login, account, password = auth_from_netrc + + # TODO(PY311): username = login or account + # Up to python 3.10, account could be None if not specified, + # and login will be empty string if not specified. From 3.11, + # login and account will be empty string if not specified. + username = login if (login or account is None) else account + + # TODO(PY311): Remove this, as password will be empty string + # if not specified + if password is None: + password = "" + + return _basic_auth_no_warn(username, password) + + +def proxies_from_env() -> dict[str, ProxyInfo]: + proxy_urls = { + k: URL(v) + for k, v in getproxies().items() + if k in ("http", "https", "ws", "wss") + } + netrc_obj = netrc_from_env() + stripped = {k: strip_auth_from_url(v) for k, v in proxy_urls.items()} + ret = {} + for proto, val in stripped.items(): + proxy, auth = val + if proxy.scheme in ("https", "wss"): + client_logger.warning( + "%s proxies %s are not supported, ignoring", proxy.scheme.upper(), proxy + ) + continue + if netrc_obj and auth is None: + if proxy.host is not None: + try: + auth = basicauth_from_netrc(netrc_obj, proxy.host) + except LookupError: + auth = None + ret[proto] = ProxyInfo(proxy, auth) + return ret + + +def get_env_proxy_for_url(url: URL) -> tuple[URL, BasicAuth | None]: + """Get a permitted proxy for the given URL from the env.""" + if url.host is not None and proxy_bypass(url.host): + raise LookupError(f"Proxying is disallowed for `{url.host!r}`") + + proxies_in_env = proxies_from_env() + try: + proxy_info = proxies_in_env[url.scheme] + except KeyError: + raise LookupError(f"No proxies found for `{url!s}` in the env") + else: + return proxy_info.proxy, proxy_info.proxy_auth + + +@attr.s(auto_attribs=True, frozen=True, slots=True) +class MimeType: + type: str + subtype: str + suffix: str + parameters: "MultiDictProxy[str]" + + +@functools.lru_cache(maxsize=56) +def parse_mimetype(mimetype: str) -> MimeType: + """Parses a MIME type into its components. + + mimetype is a MIME type string. + + Returns a MimeType object. + + Example: + + >>> parse_mimetype('text/html; charset=utf-8') + MimeType(type='text', subtype='html', suffix='', + parameters={'charset': 'utf-8'}) + + """ + if not mimetype: + return MimeType( + type="", subtype="", suffix="", parameters=MultiDictProxy(MultiDict()) + ) + + parts = mimetype.split(";") + params: MultiDict[str] = MultiDict() + for item in parts[1:]: + if not item.strip(): + continue + key, _, value = item.partition("=") + params.add(key.lower().strip(), value.strip(' "')) + + fulltype = parts[0].strip().lower() + if fulltype == "*": + fulltype = "*/*" + + mtype, _, stype = fulltype.partition("/") + stype, _, suffix = stype.partition("+") + + return MimeType( + type=mtype, subtype=stype, suffix=suffix, parameters=MultiDictProxy(params) + ) + + +class EnsureOctetStream(EmailMessage): + def __init__(self) -> None: + super().__init__() + # https://www.rfc-editor.org/rfc/rfc9110#section-8.3-5 + self.set_default_type("application/octet-stream") + + def get_content_type(self) -> str: + """Re-implementation from Message + + Returns application/octet-stream in place of plain/text when + value is wrong. + + The way this class is used guarantees that content-type will + be present so simplify the checks wrt to the base implementation. + """ + value = self.get("content-type", "").lower() + + # Based on the implementation of _splitparam in the standard library + ctype, _, _ = value.partition(";") + ctype = ctype.strip() + if ctype.count("/") != 1: + return self.get_default_type() + return ctype + + +@functools.lru_cache(maxsize=56) +def parse_content_type(raw: str) -> tuple[str, MappingProxyType[str, str]]: + """Parse Content-Type header. + + Returns a tuple of the parsed content type and a + MappingProxyType of parameters. The default returned value + is `application/octet-stream` + """ + msg = HeaderParser(EnsureOctetStream, policy=HTTP).parsestr(f"Content-Type: {raw}") + content_type = msg.get_content_type() + params = msg.get_params(()) + content_dict = dict(params[1:]) # First element is content type again + return content_type, MappingProxyType(content_dict) + + +def guess_filename(obj: Any, default: str | None = None) -> str | None: + name = getattr(obj, "name", None) + if name and isinstance(name, str) and name[0] != "<" and name[-1] != ">": + return Path(name).name + return default + + +not_qtext_re = re.compile(r"[^\041\043-\133\135-\176]") +QCONTENT = {chr(i) for i in range(0x20, 0x7F)} | {"\t"} + + +def quoted_string(content: str) -> str: + """Return 7-bit content as quoted-string. + + Format content into a quoted-string as defined in RFC5322 for + Internet Message Format. Notice that this is not the 8-bit HTTP + format, but the 7-bit email format. Content must be in usascii or + a ValueError is raised. + """ + if not (QCONTENT > set(content)): + raise ValueError(f"bad content for quoted-string {content!r}") + return not_qtext_re.sub(lambda x: "\\" + x.group(0), content) + + +def content_disposition_header( + disptype: str, quote_fields: bool = True, _charset: str = "utf-8", **params: str +) -> str: + """Sets ``Content-Disposition`` header for MIME. + + This is the MIME payload Content-Disposition header from RFC 2183 + and RFC 7579 section 4.2, not the HTTP Content-Disposition from + RFC 6266. + + disptype is a disposition type: inline, attachment, form-data. + Should be valid extension token (see RFC 2183) + + quote_fields performs value quoting to 7-bit MIME headers + according to RFC 7578. Set to quote_fields to False if recipient + can take 8-bit file names and field values. + + _charset specifies the charset to use when quote_fields is True. + + params is a dict with disposition params. + """ + if not disptype or not (TOKEN > set(disptype)): + raise ValueError(f"bad content disposition type {disptype!r}") + + value = disptype + if params: + lparams = [] + for key, val in params.items(): + if not key or not (TOKEN > set(key)): + raise ValueError(f"bad content disposition parameter {key!r}={val!r}") + if quote_fields: + if key.lower() == "filename": + qval = quote(val, "", encoding=_charset) + lparams.append((key, '"%s"' % qval)) + else: + try: + qval = quoted_string(val) + except ValueError: + qval = "".join( + (_charset, "''", quote(val, "", encoding=_charset)) + ) + lparams.append((key + "*", qval)) + else: + lparams.append((key, '"%s"' % qval)) + else: + qval = val.replace("\\", "\\\\").replace('"', '\\"') + lparams.append((key, '"%s"' % qval)) + sparams = "; ".join("=".join(pair) for pair in lparams) + value = "; ".join((value, sparams)) + return value + + +def is_ip_address(host: str | None) -> bool: + """Check if host looks like an IP Address. + + This check is only meant as a heuristic to ensure that + a host is not a domain name. + """ + if not host: + return False + # For a host to be an ipv4 address, it must be all numeric. + # The host must contain a colon to be an IPv6 address. + return ":" in host or host.replace(".", "").isdigit() + + +def is_canonical_ipv4_address(host: str) -> bool: + """Check if host is a canonical dotted-quad IPv4 address. + + Rejects the legacy numeric forms that ``socket`` still accepts and + maps onto an address, e.g. ``2130706433``, ``017700000001``, ``127.1``. + """ + parts = host.split(".") + if len(parts) != 4: + return False + for part in parts: + # Each octet must be 1-3 ASCII digits; reject unicode digits + # (which ``str.isdigit`` accepts but ``int`` may not), octal + # leading zeros, and values above 255. + if not (1 <= len(part) <= 3) or not part.isascii() or not part.isdigit(): + return False + if part[0] == "0" and len(part) != 1: + return False + if int(part) > 255: + return False + return True + + +_cached_current_datetime: int | None = None +_cached_formatted_datetime = "" + + +def rfc822_formatted_time() -> str: + global _cached_current_datetime + global _cached_formatted_datetime + + now = int(time.time()) + if now != _cached_current_datetime: + # Weekday and month names for HTTP date/time formatting; + # always English! + # Tuples are constants stored in codeobject! + _weekdayname = ("Mon", "Tue", "Wed", "Thu", "Fri", "Sat", "Sun") + _monthname = ( + "", # Dummy so we can use 1-based month numbers + "Jan", + "Feb", + "Mar", + "Apr", + "May", + "Jun", + "Jul", + "Aug", + "Sep", + "Oct", + "Nov", + "Dec", + ) + + year, month, day, hh, mm, ss, wd, *tail = time.gmtime(now) + _cached_formatted_datetime = "%s, %02d %3s %4d %02d:%02d:%02d GMT" % ( + _weekdayname[wd], + day, + _monthname[month], + year, + hh, + mm, + ss, + ) + _cached_current_datetime = now + return _cached_formatted_datetime + + +def _weakref_handle(info: "tuple[weakref.ref[object], str]") -> None: + ref, name = info + ob = ref() + if ob is not None: + with suppress(Exception): + getattr(ob, name)() + + +def weakref_handle( + ob: object, + name: str, + timeout: float, + loop: asyncio.AbstractEventLoop, + timeout_ceil_threshold: float = 5, +) -> asyncio.TimerHandle | None: + if timeout is not None and timeout > 0: + when = loop.time() + timeout + if timeout >= timeout_ceil_threshold: + when = ceil(when) + + return loop.call_at(when, _weakref_handle, (weakref.ref(ob), name)) + return None + + +def call_later( + cb: Callable[[], Any], + timeout: float, + loop: asyncio.AbstractEventLoop, + timeout_ceil_threshold: float = 5, +) -> asyncio.TimerHandle | None: + if timeout is None or timeout <= 0: + return None + now = loop.time() + when = calculate_timeout_when(now, timeout, timeout_ceil_threshold) + return loop.call_at(when, cb) + + +def calculate_timeout_when( + loop_time: float, + timeout: float, + timeout_ceiling_threshold: float, +) -> float: + """Calculate when to execute a timeout.""" + when = loop_time + timeout + if timeout > timeout_ceiling_threshold: + return ceil(when) + return when + + +class TimeoutHandle: + """Timeout handle""" + + __slots__ = ("_timeout", "_loop", "_ceil_threshold", "_callbacks") + + def __init__( + self, + loop: asyncio.AbstractEventLoop, + timeout: float | None, + ceil_threshold: float = 5, + ) -> None: + self._timeout = timeout + self._loop = loop + self._ceil_threshold = ceil_threshold + self._callbacks: list[ + tuple[Callable[..., None], tuple[Any, ...], dict[str, Any]] + ] = [] + + def register( + self, callback: Callable[..., None], *args: Any, **kwargs: Any + ) -> None: + self._callbacks.append((callback, args, kwargs)) + + def close(self) -> None: + self._callbacks.clear() + + def start(self) -> asyncio.TimerHandle | None: + timeout = self._timeout + if timeout is not None and timeout > 0: + when = self._loop.time() + timeout + if timeout >= self._ceil_threshold: + when = ceil(when) + return self._loop.call_at(when, self.__call__) + else: + return None + + def timer(self) -> "BaseTimerContext": + if self._timeout is not None and self._timeout > 0: + timer = TimerContext(self._loop) + self.register(timer.timeout) + return timer + else: + return TimerNoop() + + def __call__(self) -> None: + for cb, args, kwargs in self._callbacks: + with suppress(Exception): + cb(*args, **kwargs) + + self._callbacks.clear() + + +class BaseTimerContext(ContextManager["BaseTimerContext"]): + + __slots__ = () + + def assert_timeout(self) -> None: + """Raise TimeoutError if timeout has been exceeded.""" + + +class TimerNoop(BaseTimerContext): + + __slots__ = () + + def __enter__(self) -> BaseTimerContext: + return self + + def __exit__( + self, + exc_type: type[BaseException] | None, + exc_val: BaseException | None, + exc_tb: TracebackType | None, + ) -> None: + return + + +class TimerContext(BaseTimerContext): + """Low resolution timeout context manager""" + + __slots__ = ("_loop", "_tasks", "_cancelled", "_cancelling") + + def __init__(self, loop: asyncio.AbstractEventLoop) -> None: + self._loop = loop + self._tasks: list[asyncio.Task[Any]] = [] + self._cancelled = False + self._cancelling = 0 + + def assert_timeout(self) -> None: + """Raise TimeoutError if timer has already been cancelled.""" + if self._cancelled: + raise asyncio.TimeoutError from None + + def __enter__(self) -> BaseTimerContext: + task = asyncio.current_task(loop=self._loop) + if task is None: + raise RuntimeError("Timeout context manager should be used inside a task") + + if sys.version_info >= (3, 11): + # Remember if the task was already cancelling + # so when we __exit__ we can decide if we should + # raise asyncio.TimeoutError or let the cancellation propagate + self._cancelling = task.cancelling() + + if self._cancelled: + raise asyncio.TimeoutError from None + + self._tasks.append(task) + return self + + def __exit__( + self, + exc_type: type[BaseException] | None, + exc_val: BaseException | None, + exc_tb: TracebackType | None, + ) -> bool | None: + enter_task: asyncio.Task[Any] | None = None + if self._tasks: + enter_task = self._tasks.pop() + + if exc_type is asyncio.CancelledError and self._cancelled: + assert enter_task is not None + # The timeout was hit, and the task was cancelled + # so we need to uncancel the last task that entered the context manager + # since the cancellation should not leak out of the context manager + if sys.version_info >= (3, 11): + # If the task was already cancelling don't raise + # asyncio.TimeoutError and instead return None + # to allow the cancellation to propagate + if enter_task.uncancel() > self._cancelling: + return None + raise asyncio.TimeoutError from exc_val + return None + + def timeout(self) -> None: + if not self._cancelled: + for task in set(self._tasks): + task.cancel() + + self._cancelled = True + + +def ceil_timeout( + delay: float | None, ceil_threshold: float = 5 +) -> async_timeout.Timeout: + if delay is None or delay <= 0: + return async_timeout.timeout(None) + + loop = asyncio.get_running_loop() + now = loop.time() + when = now + delay + if delay > ceil_threshold: + when = ceil(when) + return async_timeout.timeout_at(when) + + +class HeadersMixin: + """Mixin for handling headers.""" + + ATTRS = frozenset(["_content_type", "_content_dict", "_stored_content_type"]) + + _headers: MultiMapping[str] + _content_type: str | None = None + _content_dict: dict[str, str] | None = None + _stored_content_type: str | None | _SENTINEL = sentinel + + def _parse_content_type(self, raw: str | None) -> None: + self._stored_content_type = raw + if raw is None: + # default value according to RFC 2616 + self._content_type = "application/octet-stream" + self._content_dict = {} + else: + content_type, content_mapping_proxy = parse_content_type(raw) + self._content_type = content_type + # _content_dict needs to be mutable so we can update it + self._content_dict = content_mapping_proxy.copy() + + @property + def content_type(self) -> str: + """The value of content part for Content-Type HTTP header.""" + raw = self._headers.get(hdrs.CONTENT_TYPE) + if self._stored_content_type != raw: + self._parse_content_type(raw) + assert self._content_type is not None + return self._content_type + + @property + def charset(self) -> str | None: + """The value of charset part for Content-Type HTTP header.""" + raw = self._headers.get(hdrs.CONTENT_TYPE) + if self._stored_content_type != raw: + self._parse_content_type(raw) + assert self._content_dict is not None + return self._content_dict.get("charset") + + @property + def content_length(self) -> int | None: + """The value of Content-Length HTTP header.""" + content_length = self._headers.get(hdrs.CONTENT_LENGTH) + return None if content_length is None else int(content_length) + + +def set_result(fut: "asyncio.Future[_T]", result: _T) -> None: + if not fut.done(): + fut.set_result(result) + + +_EXC_SENTINEL = BaseException() + + +class ErrorableProtocol(Protocol): + def set_exception( + self, + exc: BaseException, + exc_cause: BaseException = ..., + ) -> None: ... # pragma: no cover + + +def set_exception( + fut: "asyncio.Future[_T] | ErrorableProtocol", + exc: BaseException, + exc_cause: BaseException = _EXC_SENTINEL, +) -> None: + """Set future exception. + + If the future is marked as complete, this function is a no-op. + + :param exc_cause: An exception that is a direct cause of ``exc``. + Only set if provided. + """ + if asyncio.isfuture(fut) and fut.done(): + return + + exc_is_sentinel = exc_cause is _EXC_SENTINEL + exc_causes_itself = exc is exc_cause + if not exc_is_sentinel and not exc_causes_itself: + exc.__cause__ = exc_cause + + fut.set_exception(exc) + + +@functools.total_ordering +class BaseKey(Generic[_T]): + """Base for concrete context storage key classes. + + Each storage is provided with its own sub-class for the sake of some additional type safety. + """ + + __slots__ = ("_name", "_t", "__orig_class__") + + # This may be set by Python when instantiating with a generic type. We need to + # support this, in order to support types that are not concrete classes, + # like Iterable, which can't be passed as the second parameter to __init__. + __orig_class__: type[object] + + def __init__(self, name: str, t: type[_T] | None = None): + # Prefix with module name to help deduplicate key names. + frame = inspect.currentframe() + while frame: + if frame.f_code.co_name == "": + module: str = frame.f_globals["__name__"] + break + frame = frame.f_back + + self._name = module + "." + name + self._t = t + + def __lt__(self, other: object) -> bool: + if isinstance(other, BaseKey): + return self._name < other._name + return True # Order BaseKey above other types. + + def __repr__(self) -> str: + t = self._t + if t is None: + with suppress(AttributeError): + # Set to type arg. + t = get_args(self.__orig_class__)[0] + + if t is None: + t_repr = "<>" + elif isinstance(t, type): + if t.__module__ == "builtins": + t_repr = t.__qualname__ + else: + t_repr = f"{t.__module__}.{t.__qualname__}" + else: + t_repr = repr(t) + return f"<{self.__class__.__name__}({self._name}, type={t_repr})>" + + +class AppKey(BaseKey[_T]): + """Keys for static typing support in Application.""" + + +class RequestKey(BaseKey[_T]): + """Keys for static typing support in Request.""" + + +class ResponseKey(BaseKey[_T]): + """Keys for static typing support in Response.""" + + +class ChainMapProxy(Mapping[str | AppKey[Any], Any]): + __slots__ = ("_maps",) + + def __init__(self, maps: Iterable[Mapping[str | AppKey[Any], Any]]) -> None: + self._maps = tuple(maps) + + def __init_subclass__(cls) -> None: + raise TypeError( + f"Inheritance class {cls.__name__} from ChainMapProxy is forbidden" + ) + + @overload # type: ignore[override] + def __getitem__(self, key: AppKey[_T]) -> _T: ... + + @overload + def __getitem__(self, key: str) -> Any: ... + + def __getitem__(self, key: str | AppKey[_T]) -> Any: + for mapping in self._maps: + try: + return mapping[key] + except KeyError: + pass + raise KeyError(key) + + @overload # type: ignore[override] + def get(self, key: AppKey[_T], default: _S) -> _T | _S: ... + + @overload + def get(self, key: AppKey[_T], default: None = ...) -> _T | None: ... + + @overload + def get(self, key: str, default: Any = ...) -> Any: ... + + def get(self, key: str | AppKey[_T], default: Any = None) -> Any: + try: + return self[key] + except KeyError: + return default + + def __len__(self) -> int: + # reuses stored hash values if possible + return len(set().union(*self._maps)) + + def __iter__(self) -> Iterator[str | AppKey[Any]]: + d: dict[str | AppKey[Any], Any] = {} + for mapping in reversed(self._maps): + # reuses stored hash values if possible + d.update(mapping) + return iter(d) + + def __contains__(self, key: object) -> bool: + return any(key in m for m in self._maps) + + def __bool__(self) -> bool: + return any(self._maps) + + def __repr__(self) -> str: + content = ", ".join(map(repr, self._maps)) + return f"ChainMapProxy({content})" + + +# https://tools.ietf.org/html/rfc7232#section-2.3 +_ETAGC = r"[!\x23-\x7E\x80-\xff]+" +_ETAGC_RE = re.compile(_ETAGC) +_QUOTED_ETAG = rf'(W/)?"({_ETAGC})"' +QUOTED_ETAG_RE = re.compile(_QUOTED_ETAG) +LIST_QUOTED_ETAG_RE = re.compile(rf"({_QUOTED_ETAG})(?:\s*,\s*|$)|(.)") + +ETAG_ANY = "*" + + +@attr.s(auto_attribs=True, frozen=True, slots=True) +class ETag: + value: str + is_weak: bool = False + + +def validate_etag_value(value: str) -> None: + if value != ETAG_ANY and not _ETAGC_RE.fullmatch(value): + raise ValueError( + f"Value {value!r} is not a valid etag. Maybe it contains '\"'?" + ) + + +def parse_http_date(date_str: str | None) -> datetime.datetime | None: + """Process a date string, return a datetime object""" + if date_str is not None: + timetuple = parsedate(date_str) + if timetuple is not None: + with suppress(ValueError): + return datetime.datetime(*timetuple[:6], tzinfo=datetime.timezone.utc) + return None + + +@functools.lru_cache +def must_be_empty_body(method: str, code: int) -> bool: + """Check if a request must return an empty body.""" + return ( + code in EMPTY_BODY_STATUS_CODES + or method in EMPTY_BODY_METHODS + or (200 <= code < 300 and method == hdrs.METH_CONNECT) + ) + + +def should_remove_content_length(method: str, code: int) -> bool: + """Check if a Content-Length header should be removed. + + This should always be a subset of must_be_empty_body + """ + # https://www.rfc-editor.org/rfc/rfc9110.html#section-8.6-8 + # https://www.rfc-editor.org/rfc/rfc9110.html#section-15.4.5-4 + return code in EMPTY_BODY_STATUS_CODES or ( + 200 <= code < 300 and method == hdrs.METH_CONNECT + ) diff --git a/py_modules/aiohttp/http.py b/py_modules/aiohttp/http.py new file mode 100644 index 0000000..28c8fee --- /dev/null +++ b/py_modules/aiohttp/http.py @@ -0,0 +1,78 @@ +import sys +from collections.abc import Mapping +from http import HTTPStatus + +from . import __version__ +from .http_exceptions import HttpProcessingError as HttpProcessingError +from .http_parser import ( + HeadersParser as HeadersParser, + HttpParser as HttpParser, + HttpRequestParser as HttpRequestParser, + HttpResponseParser as HttpResponseParser, + RawRequestMessage as RawRequestMessage, + RawResponseMessage as RawResponseMessage, +) +from .http_websocket import ( + WS_CLOSED_MESSAGE as WS_CLOSED_MESSAGE, + WS_CLOSING_MESSAGE as WS_CLOSING_MESSAGE, + WS_KEY as WS_KEY, + WebSocketError as WebSocketError, + WebSocketReader as WebSocketReader, + WebSocketWriter as WebSocketWriter, + WSCloseCode as WSCloseCode, + WSMessage as WSMessage, + WSMessageDecodeText as WSMessageDecodeText, + WSMessageNoDecodeText as WSMessageNoDecodeText, + WSMessageTextBytes as WSMessageTextBytes, + WSMsgType as WSMsgType, + ws_ext_gen as ws_ext_gen, + ws_ext_parse as ws_ext_parse, +) +from .http_writer import ( + HttpVersion as HttpVersion, + HttpVersion10 as HttpVersion10, + HttpVersion11 as HttpVersion11, + StreamWriter as StreamWriter, +) + +__all__ = ( + "HttpProcessingError", + "RESPONSES", + "SERVER_SOFTWARE", + # .http_writer + "StreamWriter", + "HttpVersion", + "HttpVersion10", + "HttpVersion11", + # .http_parser + "HeadersParser", + "HttpParser", + "HttpRequestParser", + "HttpResponseParser", + "RawRequestMessage", + "RawResponseMessage", + # .http_websocket + "WS_CLOSED_MESSAGE", + "WS_CLOSING_MESSAGE", + "WS_KEY", + "WebSocketReader", + "WebSocketWriter", + "ws_ext_gen", + "ws_ext_parse", + "WSMessage", + "WSMessageDecodeText", + "WSMessageNoDecodeText", + "WSMessageTextBytes", + "WebSocketError", + "WSMsgType", + "WSCloseCode", +) + + +SERVER_SOFTWARE: str = ( + f"Python/{sys.version_info[0]}.{sys.version_info[1]} aiohttp/{__version__}" +) + +RESPONSES: Mapping[int, tuple[str, str]] = { + v: (v.phrase, v.description) for v in HTTPStatus.__members__.values() +} diff --git a/py_modules/aiohttp/http_exceptions.py b/py_modules/aiohttp/http_exceptions.py new file mode 100644 index 0000000..54cc4af --- /dev/null +++ b/py_modules/aiohttp/http_exceptions.py @@ -0,0 +1,118 @@ +"""Low-level http related exceptions.""" + +from textwrap import indent + +from .typedefs import _CIMultiDict + +__all__ = ("HttpProcessingError",) + + +class HttpProcessingError(Exception): + """HTTP error. + + Shortcut for raising HTTP errors with custom code, message and headers. + + code: HTTP Error code. + message: (optional) Error message. + headers: (optional) Headers to be sent in response, a list of pairs + """ + + code = 0 + message = "" + headers = None + + def __init__( + self, + *, + code: int | None = None, + message: str = "", + headers: _CIMultiDict | None = None, + ) -> None: + if code is not None: + self.code = code + self.headers = headers + self.message = message + + def __str__(self) -> str: + msg = indent(self.message, " ") + return f"{self.code}, message:\n{msg}" + + def __repr__(self) -> str: + return f"<{self.__class__.__name__}: {self.code}, message={self.message!r}>" + + +class BadHttpMessage(HttpProcessingError): + + code = 400 + message = "Bad Request" + + def __init__(self, message: str, *, headers: _CIMultiDict | None = None) -> None: + super().__init__(message=message, headers=headers) + self.args = (message,) + + +class HttpBadRequest(BadHttpMessage): + + code = 400 + message = "Bad Request" + + +class PayloadEncodingError(BadHttpMessage): + """Base class for payload errors""" + + +class ContentEncodingError(PayloadEncodingError): + """Content encoding error.""" + + +class TransferEncodingError(PayloadEncodingError): + """transfer encoding error.""" + + +class ContentLengthError(PayloadEncodingError): + """Not enough data to satisfy content length header.""" + + +class DecompressSizeError(PayloadEncodingError): + """Deprecated. Removed in v4.""" + + +class LineTooLong(BadHttpMessage): + def __init__( + self, + line: str | bytes, + limit: str | int = "Unknown", + actual_size: str = "Unknown", + ) -> None: + super().__init__(f"Got more than {limit} bytes when reading: {line!r}.") + self.args = (line, limit, actual_size) + + +class InvalidHeader(BadHttpMessage): + def __init__(self, hdr: bytes | str) -> None: + hdr_s = hdr.decode(errors="backslashreplace") if isinstance(hdr, bytes) else hdr + super().__init__(f"Invalid HTTP header: {hdr!r}") + self.hdr = hdr_s + self.args = (hdr,) + + +class BadStatusLine(BadHttpMessage): + def __init__(self, line: str = "", error: str | None = None) -> None: + if not isinstance(line, str): + line = repr(line) + super().__init__(error or f"Bad status line {line!r}") + self.args = (line,) + self.line = line + + +class BadHttpMethod(BadStatusLine): + """Invalid HTTP method in status line.""" + + def __init__(self, line: str = "", error: str | None = None) -> None: + if error is None and line.startswith("\x16\x03"): + error = "Received HTTPS traffic on an HTTP port" + super().__init__(line, error or f"Bad HTTP method in status line {line!r}") + + +class InvalidURLError(BadHttpMessage): + pass diff --git a/py_modules/aiohttp/http_parser.py b/py_modules/aiohttp/http_parser.py new file mode 100644 index 0000000..0c9fd08 --- /dev/null +++ b/py_modules/aiohttp/http_parser.py @@ -0,0 +1,1246 @@ +import abc +import asyncio +import re +import string +import sys +from contextlib import suppress +from enum import IntEnum +from re import Pattern +from typing import ( + TYPE_CHECKING, + Any, + ClassVar, + Final, + Generic, + Literal, + NamedTuple, + TypeVar, +) + +from multidict import CIMultiDict, CIMultiDictProxy, istr +from yarl import URL + +from . import hdrs +from .base_protocol import BaseProtocol +from .compression_utils import ( + HAS_BROTLI, + HAS_ZSTD, + BrotliDecompressor, + ZLibDecompressor, + ZSTDDecompressor, +) +from .helpers import ( + _EXC_SENTINEL, + DEBUG, + DEFAULT_CHUNK_SIZE, + EMPTY_BODY_METHODS, + EMPTY_BODY_STATUS_CODES, + NO_EXTENSIONS, + BaseTimerContext, + set_exception, +) +from .http_exceptions import ( + BadHttpMessage, + BadHttpMethod, + BadStatusLine, + ContentEncodingError, + ContentLengthError, + InvalidHeader, + InvalidURLError, + LineTooLong, + TransferEncodingError, +) +from .http_writer import HttpVersion, HttpVersion10, HttpVersion11 +from .streams import EMPTY_PAYLOAD, StreamReader +from .typedefs import RawHeaders + +if TYPE_CHECKING: + from .client_proto import ResponseHandler + +__all__ = ( + "HeadersParser", + "HttpParser", + "HttpRequestParser", + "HttpResponseParser", + "RawRequestMessage", + "RawResponseMessage", +) + +_SEP = Literal[b"\r\n", b"\n"] + +ASCIISET: Final[set[str]] = set(string.printable) + +# See https://www.rfc-editor.org/rfc/rfc9110.html#name-overview +# and https://www.rfc-editor.org/rfc/rfc9110.html#name-tokens +# +# method = token +# tchar = "!" / "#" / "$" / "%" / "&" / "'" / "*" / "+" / "-" / "." / +# "^" / "_" / "`" / "|" / "~" / DIGIT / ALPHA +# token = 1*tchar +_TCHAR_SPECIALS: Final[str] = re.escape("!#$%&'*+-.^_`|~") +TOKENRE: Final[Pattern[str]] = re.compile(f"[0-9A-Za-z{_TCHAR_SPECIALS}]+") +VERSRE: Final[Pattern[str]] = re.compile(r"HTTP/(\d)\.(\d)", re.ASCII) +DIGITS: Final[Pattern[str]] = re.compile(r"\d+", re.ASCII) +HEXDIGITS: Final[Pattern[bytes]] = re.compile(rb"[0-9a-fA-F]+") +# https://www.rfc-editor.org/rfc/rfc9110#section-5.5-5 +_FIELD_VALUE_FORBIDDEN_CTL_RE: Final[Pattern[str]] = re.compile( + r"[\x00-\x08\x0a-\x1f\x7f]" +) + +# RFC 9110 singleton headers — duplicates are rejected in strict mode. +# In lax mode (response parser default), the check is skipped entirely +# since real-world servers (e.g. Google APIs, Werkzeug) commonly send +# duplicate headers like Content-Type or Server. +# Lowercased for case-insensitive matching against wire names. +SINGLETON_HEADERS: Final[frozenset[str]] = frozenset( + { + "content-length", + "content-location", + "content-range", + "content-type", + "etag", + "host", + "max-forwards", + "server", + "transfer-encoding", + "user-agent", + } +) + + +class RawRequestMessage(NamedTuple): + method: str + path: str + version: HttpVersion + headers: "CIMultiDictProxy[str]" + raw_headers: RawHeaders + should_close: bool + compression: str | None + upgrade: bool + chunked: bool + url: URL + + +class RawResponseMessage(NamedTuple): + version: HttpVersion + code: int + reason: str + headers: CIMultiDictProxy[str] + raw_headers: RawHeaders + should_close: bool + compression: str | None + upgrade: bool + chunked: bool + + +_MsgT = TypeVar("_MsgT", RawRequestMessage, RawResponseMessage) + + +class PayloadState(IntEnum): + PAYLOAD_COMPLETE = 0 + PAYLOAD_NEEDS_INPUT = 1 + PAYLOAD_HAS_PENDING_INPUT = 2 + + +class ParseState(IntEnum): + + PARSE_NONE = 0 + PARSE_LENGTH = 1 + PARSE_CHUNKED = 2 + PARSE_UNTIL_EOF = 3 + + +class ChunkState(IntEnum): + PARSE_CHUNKED_SIZE = 0 + PARSE_CHUNKED_CHUNK = 1 + PARSE_CHUNKED_CHUNK_EOF = 2 + PARSE_MAYBE_TRAILERS = 3 + PARSE_TRAILERS = 4 + + +class HeadersParser: + def __init__( + self, + max_line_size: int = 8190, + max_headers: int = 32768, + max_field_size: int = 8190, + lax: bool = False, + ) -> None: + self.max_line_size = max_line_size + self.max_headers = max_headers + self.max_field_size = max_field_size + self._lax = lax + + def parse_headers( + self, lines: list[bytes] + ) -> tuple["CIMultiDictProxy[str]", RawHeaders]: + headers: CIMultiDict[str] = CIMultiDict() + # note: "raw" does not mean inclusion of OWS before/after the field value + raw_headers = [] + + lines_idx = 0 + line = lines[lines_idx] + line_count = len(lines) + + while line: + # Parse initial header name : value pair. + try: + bname, bvalue = line.split(b":", 1) + except ValueError: + raise InvalidHeader(line) from None + + if len(bname) == 0: + raise InvalidHeader(bname) + + # https://www.rfc-editor.org/rfc/rfc9112.html#section-5.1-2 + if {bname[0], bname[-1]} & {32, 9}: # {" ", "\t"} + raise InvalidHeader(line) + + bvalue = bvalue.lstrip(b" \t") + name = bname.decode("utf-8", "surrogateescape") + if not TOKENRE.fullmatch(name): + raise InvalidHeader(bname) + + # next line + lines_idx += 1 + line = lines[lines_idx] + + # consume continuation lines + continuation = self._lax and line and line[0] in (32, 9) # (' ', '\t') + + # Deprecated: https://www.rfc-editor.org/rfc/rfc9112.html#name-obsolete-line-folding + if continuation: + header_length = len(bvalue) + bvalue_lst = [bvalue] + while continuation: + header_length += len(line) + if header_length > self.max_field_size: + header_line = bname + b": " + b"".join(bvalue_lst) + raise LineTooLong( + header_line[:100] + b"...", self.max_field_size + ) + bvalue_lst.append(line) + + # next line + lines_idx += 1 + if lines_idx < line_count: + line = lines[lines_idx] + if line: + continuation = line[0] in (32, 9) # (' ', '\t') + else: + line = b"" + break + bvalue = b"".join(bvalue_lst) + + bvalue = bvalue.strip(b" \t") + value = bvalue.decode("utf-8", "surrogateescape") + + # https://www.rfc-editor.org/rfc/rfc9110.html#section-5.5-5 + if self._lax: + if "\n" in value or "\r" in value or "\x00" in value: + raise InvalidHeader(bvalue) + elif _FIELD_VALUE_FORBIDDEN_CTL_RE.search(value): + raise InvalidHeader(bvalue) + + if not self._lax and name in headers and name.lower() in SINGLETON_HEADERS: + raise BadHttpMessage(f"Duplicate '{name}' header found.") + headers.add(name, value) + raw_headers.append((bname, bvalue)) + + return (CIMultiDictProxy(headers), tuple(raw_headers)) + + +def _is_supported_upgrade(headers: CIMultiDictProxy[str]) -> bool: + """Check if the upgrade header is supported.""" + u = headers.get(hdrs.UPGRADE, "") + # .lower() can transform non-ascii characters. + return u.isascii() and u.lower() in {"tcp", "websocket"} + + +class HttpParser(abc.ABC, Generic[_MsgT]): + lax: ClassVar[bool] = False + + def __init__( + self, + protocol: BaseProtocol | None = None, + loop: asyncio.AbstractEventLoop | None = None, + limit: int = 2**16, + max_line_size: int = 8190, + max_headers: int = 128, + max_field_size: int = 8190, + timer: BaseTimerContext | None = None, + code: int | None = None, + method: str | None = None, + payload_exception: type[BaseException] | None = None, + response_with_body: bool = True, + read_until_eof: bool = False, + auto_decompress: bool = True, + max_msg_queue_size: int = 0, + ) -> None: + self.protocol = protocol + self.loop = loop + self.max_line_size = max_line_size + self.max_headers = max_headers + self.max_field_size = max_field_size + self.max_headers = max_headers + self.timer = timer + self.code = code + self.method = method + self.payload_exception = payload_exception + self.response_with_body = response_with_body + self.read_until_eof = read_until_eof + + self._lines: list[bytes] = [] + self._tail = b"" + self._upgraded = False + self._pending_upgrade = False + self._payload = None + self._payload_parser: HttpPayloadParser | None = None + self._payload_has_more_data = False + self._auto_decompress = auto_decompress + self._limit = limit + self._headers_parser = HeadersParser( + max_line_size, max_headers, max_field_size, self.lax + ) + # Stop emitting messages once this many are queued unconsumed (0 = off). + self._max_msg_queue_size = max_msg_queue_size + self._msg_in_flight = 0 + + @abc.abstractmethod + def parse_message(self, lines: list[bytes]) -> _MsgT: ... + + @abc.abstractmethod + def _is_chunked_te(self, te: str) -> bool: ... + + def pause_reading(self) -> None: + assert self._payload_parser is not None + self._payload_parser.pause_reading() + + def message_consumed(self) -> None: + """Protocol drained a queued message; free a slot for parsing.""" + if self._msg_in_flight > 0: + self._msg_in_flight -= 1 + + def feed_eof(self) -> _MsgT | None: + if self._payload_parser is not None: + self._payload_parser.feed_eof() + if self._payload_parser.done: + self._payload_parser = None + else: + # try to extract partial message + if self._tail: + self._lines.append(self._tail) + + if self._lines: + if self._lines[-1] != "\r\n": + self._lines.append(b"") + with suppress(Exception): + return self.parse_message(self._lines) + return None + + def feed_data( + self, + data: bytes, + SEP: _SEP = b"\r\n", + EMPTY: bytes = b"", + CONTENT_LENGTH: istr = hdrs.CONTENT_LENGTH, + METH_CONNECT: str = hdrs.METH_CONNECT, + SEC_WEBSOCKET_KEY1: istr = hdrs.SEC_WEBSOCKET_KEY1, + ) -> tuple[list[tuple[_MsgT, StreamReader]], bool, bytes]: + + messages = [] + + if self._tail: + data, self._tail = self._tail + data, b"" + + data_len = len(data) + start_pos = 0 + loop = self.loop + max_line_length = self.max_line_size + + should_close = False + while start_pos < data_len or self._payload_has_more_data: + # read HTTP message (request/response line + headers), \r\n\r\n + # and split by lines + if self._payload_parser is None and not self._upgraded: + if ( + self._max_msg_queue_size + and self._msg_in_flight >= self._max_msg_queue_size + ): + # Queue full: buffer the rest and stop. Safe pause point; + # any preceding body is consumed before the next request + # line. Resumes via feed_data(b"") when the queue drains. + self._tail = data[start_pos:] + break + pos = data.find(SEP, start_pos) + # consume \r\n + if pos == start_pos and not self._lines: + start_pos = pos + len(SEP) + continue + + if pos >= start_pos: + if should_close: + raise BadHttpMessage("Data after `Connection: close`") + + # line found + line = data[start_pos:pos] + if SEP == b"\n": # For lax response parsing + line = line.rstrip(b"\r") + if len(line) > max_line_length: + raise LineTooLong(line[:100] + b"...", max_line_length) + + self._lines.append(line) + # After processing the status/request line, everything is a header. + max_line_length = self.max_field_size + + if len(self._lines) > self.max_headers: + raise BadHttpMessage("Too many headers received") + + start_pos = pos + len(SEP) + + # \r\n\r\n found + if self._lines[-1] == EMPTY: + max_trailers = self.max_headers - len(self._lines) + try: + msg: _MsgT = self.parse_message(self._lines) + finally: + self._lines.clear() + + def get_content_length() -> int | None: + # payload length + length_hdr = msg.headers.get(CONTENT_LENGTH) + if length_hdr is None: + return None + + # Shouldn't allow +/- or other number formats. + # https://www.rfc-editor.org/rfc/rfc9110#section-8.6-2 + # msg.headers is already stripped of leading/trailing wsp + if not DIGITS.fullmatch(length_hdr): + raise InvalidHeader(CONTENT_LENGTH) + + return int(length_hdr) + + length = get_content_length() + # do not support old websocket spec + if SEC_WEBSOCKET_KEY1 in msg.headers: + raise InvalidHeader(SEC_WEBSOCKET_KEY1) + + upgraded = msg.upgrade and _is_supported_upgrade(msg.headers) + + method = getattr(msg, "method", self.method) + # code is only present on responses + code = getattr(msg, "code", 0) + + assert self.protocol is not None + # calculate payload + empty_body = code in EMPTY_BODY_STATUS_CODES or bool( + method and method in EMPTY_BODY_METHODS + ) + if not empty_body and ( + (length is not None and length > 0) or msg.chunked + ): + payload = StreamReader( + self.protocol, + timer=self.timer, + loop=loop, + limit=self._limit, + ) + payload_parser = HttpPayloadParser( + payload, + length=length, + chunked=msg.chunked, + method=method, + compression=msg.compression, + code=self.code, + response_with_body=self.response_with_body, + auto_decompress=self._auto_decompress, + lax=self.lax, + headers_parser=self._headers_parser, + max_line_size=self.max_line_size, + max_field_size=self.max_field_size, + max_trailers=max_trailers, + limit=self._limit, + ) + if not payload_parser.done: + self._payload_parser = payload_parser + # https://www.rfc-editor.org/info/rfc9110/#section-7.8-15 + # Defer any requested upgrade until the + # complete request has been read. + self._pending_upgrade = upgraded + elif method == METH_CONNECT: + assert isinstance(msg, RawRequestMessage) + payload = StreamReader( + self.protocol, + timer=self.timer, + loop=loop, + limit=self._limit, + ) + self._upgraded = True + self._payload_parser = HttpPayloadParser( + payload, + method=msg.method, + compression=msg.compression, + auto_decompress=self._auto_decompress, + lax=self.lax, + headers_parser=self._headers_parser, + max_line_size=self.max_line_size, + max_field_size=self.max_field_size, + max_trailers=max_trailers, + limit=self._limit, + ) + elif not empty_body and length is None and self.read_until_eof: + payload = StreamReader( + self.protocol, + timer=self.timer, + loop=loop, + limit=self._limit, + ) + payload_parser = HttpPayloadParser( + payload, + length=length, + chunked=msg.chunked, + method=method, + compression=msg.compression, + code=self.code, + response_with_body=self.response_with_body, + auto_decompress=self._auto_decompress, + lax=self.lax, + headers_parser=self._headers_parser, + max_line_size=self.max_line_size, + max_field_size=self.max_field_size, + max_trailers=max_trailers, + limit=self._limit, + ) + if not payload_parser.done: + self._payload_parser = payload_parser + elif upgraded: + # No body to read, so the connection switches to + # the upgraded protocol immediately. + self._upgraded = True + payload = EMPTY_PAYLOAD + else: + payload = EMPTY_PAYLOAD + + messages.append((msg, payload)) + if self._max_msg_queue_size: + self._msg_in_flight += 1 + should_close = msg.should_close + else: + self._tail = data[start_pos:] + # A bare LF here means CRLF was required: + # reject instead of buffering, else a following request's + # bytes get appended to this line and leak in the error. + if b"\n" in self._tail: + raise BadHttpMessage("Bad line ending, expected CRLF") + if len(self._tail) > self.max_line_size: + raise LineTooLong(self._tail[:100] + b"...", self.max_line_size) + data = EMPTY + break + + # no parser, just store + elif self._payload_parser is None and self._upgraded: + assert not self._lines + break + + # feed payload + else: + assert not self._lines + assert self._payload_parser is not None + try: + payload_state, data = self._payload_parser.feed_data( + data[start_pos:], SEP + ) + except Exception as underlying_exc: + reraised_exc: BaseException = underlying_exc + if self.payload_exception is not None: + reraised_exc = self.payload_exception(str(underlying_exc)) + + set_exception( + self._payload_parser.payload, + reraised_exc, + underlying_exc, + ) + + payload_state = PayloadState.PAYLOAD_COMPLETE + data = b"" + if isinstance( + underlying_exc, (InvalidHeader, TransferEncodingError) + ): + raise + + self._payload_has_more_data = ( + payload_state == PayloadState.PAYLOAD_HAS_PENDING_INPUT + ) + + if payload_state is not PayloadState.PAYLOAD_COMPLETE: + # We've either consumed all available data, or we're pausing + # until the reader buffer is freed up. + break + + start_pos = 0 + data_len = len(data) + self._payload_parser = None + if self._pending_upgrade: + # Body fully read: the deferred upgrade takes effect and + # the rest of the connection is the upgraded protocol. + self._upgraded = True + self._pending_upgrade = False + + if data and start_pos < data_len: + data = data[start_pos:] + else: + data = EMPTY + + return messages, self._upgraded, data + + def parse_headers( + self, lines: list[bytes] + ) -> tuple[ + "CIMultiDictProxy[str]", RawHeaders, bool | None, str | None, bool, bool + ]: + """Parses RFC 5322 headers from a stream. + + Line continuations are supported. Returns list of header name + and value pairs. Header name is in upper case. + """ + headers, raw_headers = self._headers_parser.parse_headers(lines) + close_conn = None + encoding = None + upgrade = False + chunked = False + + # keep-alive and protocol switching + # RFC 9110 section 7.6.1 defines Connection as a comma-separated list. + conn_values = headers.getall(hdrs.CONNECTION, ()) + if conn_values: + conn_tokens = { + token.lower() + for conn_value in conn_values + for token in (part.strip(" \t") for part in conn_value.split(",")) + if token and token.isascii() + } + + if "close" in conn_tokens: + close_conn = True + elif "keep-alive" in conn_tokens: + close_conn = False + + # https://www.rfc-editor.org/rfc/rfc9110.html#name-101-switching-protocols + if "upgrade" in conn_tokens and headers.get(hdrs.UPGRADE): + upgrade = True + + # encoding + enc = headers.get(hdrs.CONTENT_ENCODING, "") + if enc.isascii() and enc.lower() in {"gzip", "deflate", "br", "zstd"}: + encoding = enc + + # chunking + te = headers.get(hdrs.TRANSFER_ENCODING) + if te is not None: + if self._is_chunked_te(te): + chunked = True + + if hdrs.CONTENT_LENGTH in headers: + raise BadHttpMessage( + "Transfer-Encoding can't be present with Content-Length", + ) + + return (headers, raw_headers, close_conn, encoding, upgrade, chunked) + + def set_upgraded(self, val: bool) -> None: + """Set connection upgraded (to websocket) mode. + + :param bool val: new state. + """ + self._upgraded = val + + +class HttpRequestParser(HttpParser[RawRequestMessage]): + """Read request status line. + + Exception .http_exceptions.BadStatusLine + could be raised in case of any errors in status line. + Returns RawRequestMessage. + """ + + def parse_message(self, lines: list[bytes]) -> RawRequestMessage: + # request line + line = lines[0].decode("utf-8", "surrogateescape") + try: + method, path, version = line.split(" ", maxsplit=2) + except ValueError: + raise BadHttpMethod(line) from None + + # method + if not TOKENRE.fullmatch(method): + raise BadHttpMethod(method) + method = method.upper() + + # version + match = VERSRE.fullmatch(version) + if match is None: + raise BadStatusLine(line) + version_o = HttpVersion(int(match.group(1)), int(match.group(2))) + + if method == "CONNECT": + # authority-form, + # https://datatracker.ietf.org/doc/html/rfc7230#section-5.3.3 + url = URL.build(authority=path, encoded=True) + elif path.startswith("/"): + # origin-form, + # https://datatracker.ietf.org/doc/html/rfc7230#section-5.3.1 + path_part, _hash_separator, url_fragment = path.partition("#") + path_part, _question_mark_separator, qs_part = path_part.partition("?") + + # NOTE: `yarl.URL.build()` is used to mimic what the Cython-based + # NOTE: parser does, otherwise it results into the same + # NOTE: HTTP Request-Line input producing different + # NOTE: `yarl.URL()` objects + url = URL.build( + path=path_part, + query_string=qs_part, + fragment=url_fragment, + encoded=True, + ) + elif path == "*" and method == "OPTIONS": + # asterisk-form, + url = URL(path, encoded=True) + else: + # absolute-form for proxy maybe, + # https://datatracker.ietf.org/doc/html/rfc7230#section-5.3.2 + url = URL(path, encoded=True) + if url.scheme == "": + # not absolute-form + raise InvalidURLError( + path.encode(errors="surrogateescape").decode("latin1") + ) + + # read headers + ( + headers, + raw_headers, + close, + compression, + upgrade, + chunked, + ) = self.parse_headers(lines[1:]) + + if version_o == HttpVersion11 and hdrs.HOST not in headers: + raise BadHttpMessage("Missing 'Host' header in request.") + + if close is None: # then the headers weren't set in the request + if version_o <= HttpVersion10: # HTTP 1.0 must asks to not close + close = True + else: # HTTP 1.1 must ask to close. + close = False + + return RawRequestMessage( + method, + path, + version_o, + headers, + raw_headers, + close, + compression, + upgrade, + chunked, + url, + ) + + def _is_chunked_te(self, te: str) -> bool: + te = te.rsplit(",", maxsplit=1)[-1].strip(" \t") + # .lower() transforms some non-ascii chars, so must check first. + if te.isascii() and te.lower() == "chunked": + return True + # https://www.rfc-editor.org/rfc/rfc9112#section-6.3-2.4.3 + raise BadHttpMessage("Request has invalid `Transfer-Encoding`") + + +class HttpResponseParser(HttpParser[RawResponseMessage]): + """Read response status line and headers. + + BadStatusLine could be raised in case of any errors in status line. + Returns RawResponseMessage. + """ + + protocol: "ResponseHandler" + + # Lax mode should only be enabled on response parser. + lax = not DEBUG + + def feed_data( + self, + data: bytes, + SEP: _SEP | None = None, + *args: Any, + **kwargs: Any, + ) -> tuple[list[tuple[RawResponseMessage, StreamReader]], bool, bytes]: + if SEP is None: + SEP = b"\r\n" if DEBUG else b"\n" + return super().feed_data(data, SEP, *args, **kwargs) + + def parse_message(self, lines: list[bytes]) -> RawResponseMessage: + line = lines[0].decode("utf-8", "surrogateescape") + try: + version, status = line.split(maxsplit=1) + except ValueError: + raise BadStatusLine(line) from None + + try: + status, reason = status.split(maxsplit=1) + except ValueError: + status = status.strip() + reason = "" + + # version + match = VERSRE.fullmatch(version) + if match is None: + raise BadStatusLine(line) + version_o = HttpVersion(int(match.group(1)), int(match.group(2))) + + # The status code is a three-digit ASCII number, no padding + if len(status) != 3 or not DIGITS.fullmatch(status): + raise BadStatusLine(line) + status_i = int(status) + + # read headers + ( + headers, + raw_headers, + close, + compression, + upgrade, + chunked, + ) = self.parse_headers(lines[1:]) + + if close is None: + if version_o <= HttpVersion10: + close = True + # https://www.rfc-editor.org/rfc/rfc9112.html#name-message-body-length + elif 100 <= status_i < 200 or status_i in {204, 304}: + close = False + elif hdrs.CONTENT_LENGTH in headers or hdrs.TRANSFER_ENCODING in headers: + close = False + else: + # https://www.rfc-editor.org/rfc/rfc9112.html#section-6.3-2.8 + close = True + + return RawResponseMessage( + version_o, + status_i, + reason.strip(), + headers, + raw_headers, + close, + compression, + upgrade, + chunked, + ) + + def _is_chunked_te(self, te: str) -> bool: + # https://www.rfc-editor.org/rfc/rfc9112#section-6.3-2.4.2 + return te.rsplit(",", maxsplit=1)[-1].strip(" \t").lower() == "chunked" + + +class HttpPayloadParser: + def __init__( + self, + payload: StreamReader, + length: int | None = None, + chunked: bool = False, + compression: str | None = None, + code: int | None = None, + method: str | None = None, + response_with_body: bool = True, + auto_decompress: bool = True, + lax: bool = False, + *, + headers_parser: HeadersParser, + max_line_size: int = 8190, + max_field_size: int = 8190, + max_trailers: int = 128, + limit: int = DEFAULT_CHUNK_SIZE, + ) -> None: + self._length = 0 + self._paused = False + self._type = ParseState.PARSE_UNTIL_EOF + self._chunk = ChunkState.PARSE_CHUNKED_SIZE + self._chunk_size = 0 + self._chunk_tail = b"" + self._auto_decompress = auto_decompress + self._lax = lax + self._headers_parser = headers_parser + self._max_line_size = max_line_size + self._max_field_size = max_field_size + self._max_trailers = max_trailers + self._more_data_available = False + self._trailer_lines: list[bytes] = [] + self.done = False + self._eof_pending = False + + # payload decompression wrapper + if response_with_body and compression and self._auto_decompress: + real_payload: StreamReader | DeflateBuffer = DeflateBuffer( + payload, compression, max_decompress_size=limit + ) + else: + real_payload = payload + + # payload parser + if not response_with_body: + # don't parse payload if it's not expected to be received + self._type = ParseState.PARSE_NONE + real_payload.feed_eof() + self.done = True + elif chunked: + self._type = ParseState.PARSE_CHUNKED + elif length is not None: + self._type = ParseState.PARSE_LENGTH + self._length = length + self._length_expected = length + if self._length == 0: + real_payload.feed_eof() + self.done = True + + self.payload = real_payload + + def pause_reading(self) -> None: + self._paused = True + + def feed_eof(self) -> None: + if self._type == ParseState.PARSE_UNTIL_EOF: + self._eof_pending = True + while self._more_data_available: + if self._paused: + self._paused = False + return # Will resume via feed_data(b"") later + self._more_data_available = self.payload.feed_data(b"", 0) + self.payload.feed_eof() + self.done = True + self._eof_pending = False + elif self._type == ParseState.PARSE_LENGTH: + received = self._length_expected - self._length + raise ContentLengthError( + f"Not enough data to satisfy content length header " + f"(received {received} of {self._length_expected} bytes)." + ) + elif self._type == ParseState.PARSE_CHUNKED: + raise TransferEncodingError( + "Not enough data to satisfy transfer length header." + ) + + def feed_data( + self, chunk: bytes, SEP: _SEP = b"\r\n", CHUNK_EXT: bytes = b";" + ) -> tuple[PayloadState, bytes]: + """Receive a chunk of data to process. + + Return: + PayloadState - The current state of payload processing. + This function may be called with empty bytes after returning + PAYLOAD_HAS_PENDING_INPUT to continue processing after a pause. + bytes - If payload is complete, this is the unconsumed bytes intended for the + next message/payload, b"" otherwise. + """ + # Read specified amount of bytes + if self._type == ParseState.PARSE_LENGTH: + if self._chunk_tail: + chunk = self._chunk_tail + chunk + self._chunk_tail = b"" + + required = self._length + self._length = max(required - len(chunk), 0) + self._more_data_available = self.payload.feed_data( + chunk[:required], required + ) + while self._more_data_available: + if self._paused: + self._paused = False + self._chunk_tail = chunk[required:] + return PayloadState.PAYLOAD_HAS_PENDING_INPUT, b"" + self._more_data_available = self.payload.feed_data(b"", 0) + + if self._length == 0: + self.payload.feed_eof() + return PayloadState.PAYLOAD_COMPLETE, chunk[required:] + # Chunked transfer encoding parser + elif self._type == ParseState.PARSE_CHUNKED: + if self._chunk_tail: + # We should check the length is sane when not processing payload body. + if self._chunk != ChunkState.PARSE_CHUNKED_CHUNK: + max_line_length = self._max_line_size + if self._chunk == ChunkState.PARSE_TRAILERS: + max_line_length = self._max_field_size + if len(self._chunk_tail) > max_line_length: + raise LineTooLong( + self._chunk_tail[:100] + b"...", max_line_length + ) + + chunk = self._chunk_tail + chunk + self._chunk_tail = b"" + + while chunk or self._more_data_available: + # read next chunk size + if self._chunk == ChunkState.PARSE_CHUNKED_SIZE: + pos = chunk.find(SEP) + if pos >= 0: + # Only chunk-size lines reach here; trailers enforce + # _max_field_size separately in PARSE_TRAILERS below. + if pos > self._max_line_size: + raise LineTooLong(chunk[:100] + b"...", self._max_line_size) + i = chunk.find(CHUNK_EXT, 0, pos) + if i >= 0: + size_b = chunk[:i] # strip chunk-extensions + # Verify no LF in the chunk-extension + if b"\n" in (ext := chunk[i:pos]): + exc = TransferEncodingError( + f"Unexpected LF in chunk-extension: {ext!r}" + ) + set_exception(self.payload, exc) + raise exc + else: + size_b = chunk[:pos] + + if self._lax: # Allow whitespace in lax mode. + size_b = size_b.strip() + + if not re.fullmatch(HEXDIGITS, size_b): + exc = TransferEncodingError( + chunk[:pos].decode("ascii", "surrogateescape") + ) + set_exception(self.payload, exc) + raise exc + size = int(bytes(size_b), 16) + + chunk = chunk[pos + len(SEP) :] + if size == 0: # eof marker + self._chunk = ChunkState.PARSE_TRAILERS + if self._lax and chunk.startswith(b"\r"): + chunk = chunk[1:] + else: + self._chunk = ChunkState.PARSE_CHUNKED_CHUNK + self._chunk_size = size + self.payload.begin_http_chunk_receiving() + else: + if b"\n" in chunk: + exc = TransferEncodingError( + "Bad chunk-size line ending, expected CRLF" + ) + set_exception(self.payload, exc) + raise exc + self._chunk_tail = chunk + return PayloadState.PAYLOAD_NEEDS_INPUT, b"" + + # read chunk and feed buffer + if self._chunk == ChunkState.PARSE_CHUNKED_CHUNK: + if self._paused: + self._paused = False + self._chunk_tail = chunk + return PayloadState.PAYLOAD_HAS_PENDING_INPUT, b"" + + required = self._chunk_size + self._chunk_size = max(required - len(chunk), 0) + self._more_data_available = self.payload.feed_data( + chunk[:required], required + ) + chunk = chunk[required:] + + if self._more_data_available: + continue + + if self._chunk_size: + self._paused = False + return PayloadState.PAYLOAD_NEEDS_INPUT, b"" + self._chunk = ChunkState.PARSE_CHUNKED_CHUNK_EOF + self.payload.end_http_chunk_receiving() + + # toss the CRLF at the end of the chunk + if self._chunk == ChunkState.PARSE_CHUNKED_CHUNK_EOF: + if self._lax and chunk.startswith(b"\r"): + chunk = chunk[1:] + if chunk[: len(SEP)] == SEP: + chunk = chunk[len(SEP) :] + self._chunk = ChunkState.PARSE_CHUNKED_SIZE + elif len(chunk) >= len(SEP) or chunk != SEP[: len(chunk)]: + exc = TransferEncodingError( + "Chunk size mismatch: expected CRLF after chunk data" + ) + set_exception(self.payload, exc) + raise exc + else: + self._chunk_tail = chunk + return PayloadState.PAYLOAD_NEEDS_INPUT, b"" + + if self._chunk == ChunkState.PARSE_TRAILERS: + pos = chunk.find(SEP) + if pos < 0: # No line found + if b"\n" in chunk: + exc = TransferEncodingError( + "Bad trailer line ending, expected CRLF" + ) + set_exception(self.payload, exc) + raise exc + self._chunk_tail = chunk + return PayloadState.PAYLOAD_NEEDS_INPUT, b"" + + line = chunk[:pos] + chunk = chunk[pos + len(SEP) :] + if SEP == b"\n": # For lax response parsing + line = line.rstrip(b"\r") + + if len(line) > self._max_field_size: + raise LineTooLong(line[:100] + b"...", self._max_field_size) + + self._trailer_lines.append(line) + + if len(self._trailer_lines) > self._max_trailers: + raise BadHttpMessage("Too many trailers received") + + # \r\n\r\n found, end of stream + if self._trailer_lines[-1] == b"": + # Headers and trailers are defined the same way, + # so we reuse the HeadersParser here. + try: + trailers, raw_trailers = self._headers_parser.parse_headers( + self._trailer_lines + ) + finally: + self._trailer_lines.clear() + self.payload.feed_eof() + return PayloadState.PAYLOAD_COMPLETE, chunk + + # Read all bytes until eof + elif self._type == ParseState.PARSE_UNTIL_EOF: + self._more_data_available = self.payload.feed_data(chunk, len(chunk)) + while self._more_data_available: + if self._paused: + self._paused = False + return PayloadState.PAYLOAD_HAS_PENDING_INPUT, b"" + self._more_data_available = self.payload.feed_data(b"", 0) + + if self._eof_pending: + self.payload.feed_eof() + self.done = True + self._eof_pending = False + return PayloadState.PAYLOAD_COMPLETE, b"" + + return PayloadState.PAYLOAD_NEEDS_INPUT, b"" + + +class DeflateBuffer: + """DeflateStream decompress stream and feed data into specified stream.""" + + decompressor: Any + + def __init__( + self, + out: StreamReader, + encoding: str | None, + max_decompress_size: int = DEFAULT_CHUNK_SIZE, + ) -> None: + self.out = out + self.size = 0 + out.total_compressed_bytes = self.size + self.encoding = encoding + self._started_decoding = False + + self.decompressor: BrotliDecompressor | ZLibDecompressor | ZSTDDecompressor + if encoding == "br": + if not HAS_BROTLI: # pragma: no cover + raise ContentEncodingError( + "Can not decode content-encoding: brotli (br). " + "Please install `Brotli`" + ) + self.decompressor = BrotliDecompressor() + elif encoding == "zstd": + if not HAS_ZSTD: + raise ContentEncodingError( + "Can not decode content-encoding: zstandard (zstd). " + "Please install `backports.zstd`" + ) + self.decompressor = ZSTDDecompressor() + else: + self.decompressor = ZLibDecompressor(encoding=encoding) + + self._max_decompress_size = max_decompress_size + + def set_exception( + self, + exc: BaseException, + exc_cause: BaseException = _EXC_SENTINEL, + ) -> None: + set_exception(self.out, exc, exc_cause) + + def feed_data(self, chunk: bytes, size: int) -> bool: + self.size += size + self.out.total_compressed_bytes = self.size + + # Inspect the first real byte once to choose the decompressor. An empty + # chunk (e.g. a chunk-size line arriving without body bytes) has no + # header to sniff, so skip it and wait for the first data byte. + if not self._started_decoding and chunk: + # RFC1950 + # bits 0..3 = CM = 0b1000 = 8 = "deflate" + # bits 4..7 = CINFO = 1..7 = windows size. + if self.encoding == "deflate" and chunk[0] & 0xF != 8: + # Change the decoder to decompress incorrectly compressed data + # Actually we should issue a warning about non-RFC-compliant data. + self.decompressor = ZLibDecompressor( + encoding=self.encoding, suppress_deflate_header=True + ) + self._started_decoding = True + + low_water = self.out._low_water + max_length = ( + 0 if low_water >= sys.maxsize else max(self._max_decompress_size, low_water) + ) + try: + chunk = self.decompressor.decompress_sync(chunk, max_length=max_length) + except Exception: + raise ContentEncodingError( + "Can not decode content-encoding: %s" % self.encoding + ) + + if chunk: + self.out.feed_data(chunk, len(chunk)) + return self.decompressor.data_available # type: ignore[no-any-return] + + def feed_eof(self) -> None: + chunk = self.decompressor.flush() + # This should never contain data as we defer the call until exhausting + # the decompression. If .flush() is returning data, this may indicate a + # zip bomb vulnerability as it will decompress all remaining data at once. + assert not chunk + + if self.size > 0: + if self.encoding == "deflate" and not self.decompressor.eof: + raise ContentEncodingError("deflate") + + self.out.feed_eof() + + def begin_http_chunk_receiving(self) -> None: + self.out.begin_http_chunk_receiving() + + def end_http_chunk_receiving(self) -> None: + self.out.end_http_chunk_receiving() + + +HttpRequestParserPy = HttpRequestParser +HttpResponseParserPy = HttpResponseParser +RawRequestMessagePy = RawRequestMessage +RawResponseMessagePy = RawResponseMessage + +try: + if not NO_EXTENSIONS: + from ._http_parser import ( # type: ignore[import-not-found,no-redef] + HttpRequestParser, + HttpResponseParser, + RawRequestMessage, + RawResponseMessage, + ) + + HttpRequestParserC = HttpRequestParser + HttpResponseParserC = HttpResponseParser + RawRequestMessageC = RawRequestMessage + RawResponseMessageC = RawResponseMessage +except ImportError: # pragma: no cover + pass diff --git a/py_modules/aiohttp/http_websocket.py b/py_modules/aiohttp/http_websocket.py new file mode 100644 index 0000000..9fb9fba --- /dev/null +++ b/py_modules/aiohttp/http_websocket.py @@ -0,0 +1,41 @@ +"""WebSocket protocol versions 13 and 8.""" + +from ._websocket.helpers import WS_KEY, ws_ext_gen, ws_ext_parse +from ._websocket.models import ( + WS_CLOSED_MESSAGE, + WS_CLOSING_MESSAGE, + WebSocketError, + WSCloseCode, + WSHandshakeError, + WSMessage, + WSMessageDecodeText, + WSMessageNoDecodeText, + WSMessageTextBytes, + WSMsgType, +) +from ._websocket.reader import WebSocketReader +from ._websocket.writer import WebSocketWriter + +# Messages that the WebSocketResponse.receive needs to handle internally +_INTERNAL_RECEIVE_TYPES = frozenset( + (WSMsgType.CLOSE, WSMsgType.CLOSING, WSMsgType.PING, WSMsgType.PONG) +) + + +__all__ = ( + "WS_CLOSED_MESSAGE", + "WS_CLOSING_MESSAGE", + "WS_KEY", + "WebSocketReader", + "WebSocketWriter", + "WSMessage", + "WSMessageDecodeText", + "WSMessageNoDecodeText", + "WSMessageTextBytes", + "WebSocketError", + "WSMsgType", + "WSCloseCode", + "ws_ext_gen", + "ws_ext_parse", + "WSHandshakeError", +) diff --git a/py_modules/aiohttp/http_writer.py b/py_modules/aiohttp/http_writer.py new file mode 100644 index 0000000..a252d92 --- /dev/null +++ b/py_modules/aiohttp/http_writer.py @@ -0,0 +1,381 @@ +"""Http related parsers and protocol.""" + +import asyncio +import re +import sys +from typing import ( # noqa + TYPE_CHECKING, + Any, + Awaitable, + Callable, + Iterable, + List, + NamedTuple, + Optional, +) + +from multidict import CIMultiDict + +from .abc import AbstractStreamWriter +from .base_protocol import BaseProtocol +from .client_exceptions import ClientConnectionResetError +from .compression_utils import ZLibCompressor +from .helpers import NO_EXTENSIONS + +__all__ = ("StreamWriter", "HttpVersion", "HttpVersion10", "HttpVersion11") + +if sys.version_info >= (3, 12): + from collections.abc import Buffer +else: + from typing import Union + + Buffer = Union[bytes, bytearray, "memoryview[int]", "memoryview[bytes]"] + + +MIN_PAYLOAD_FOR_WRITELINES = 2048 +IS_PY313_BEFORE_313_2 = (3, 13, 0) <= sys.version_info < (3, 13, 2) +IS_PY_BEFORE_312_9 = sys.version_info < (3, 12, 9) +SKIP_WRITELINES = IS_PY313_BEFORE_313_2 or IS_PY_BEFORE_312_9 +# writelines is not safe for use +# on Python 3.12+ until 3.12.9 +# on Python 3.13+ until 3.13.2 +# and on older versions it not any faster than write +# CVE-2024-12254: https://github.com/python/cpython/pull/127656 + + +class HttpVersion(NamedTuple): + major: int + minor: int + + +HttpVersion10 = HttpVersion(1, 0) +HttpVersion11 = HttpVersion(1, 1) + + +_T_OnChunkSent = Optional[Callable[[Buffer], Awaitable[None]]] +_T_OnHeadersSent = Optional[Callable[["CIMultiDict[str]"], Awaitable[None]]] + + +class StreamWriter(AbstractStreamWriter): + + length: int | None = None + chunked: bool = False + _eof: bool = False + _compress: ZLibCompressor | None = None + + def __init__( + self, + protocol: BaseProtocol, + loop: asyncio.AbstractEventLoop, + on_chunk_sent: _T_OnChunkSent = None, + on_headers_sent: _T_OnHeadersSent = None, + ) -> None: + self._protocol = protocol + self.loop = loop + self._on_chunk_sent: _T_OnChunkSent = on_chunk_sent + self._on_headers_sent: _T_OnHeadersSent = on_headers_sent + self._headers_buf: bytes | None = None + self._headers_written: bool = False + + @property + def transport(self) -> asyncio.Transport | None: + return self._protocol.transport + + @property + def protocol(self) -> BaseProtocol: + return self._protocol + + def enable_chunking(self) -> None: + self.chunked = True + + def enable_compression( + self, encoding: str = "deflate", strategy: int | None = None + ) -> None: + self._compress = ZLibCompressor(encoding=encoding, strategy=strategy) + + def _write(self, chunk: Buffer) -> None: + size = len(chunk) + self.buffer_size += size + self.output_size += size + transport = self._protocol.transport + if transport is None or transport.is_closing(): + raise ClientConnectionResetError("Cannot write to closing transport") + transport.write(chunk) + + def _writelines(self, chunks: Iterable[Buffer]) -> None: + size = 0 + for chunk in chunks: + size += len(chunk) + self.buffer_size += size + self.output_size += size + transport = self._protocol.transport + if transport is None or transport.is_closing(): + raise ClientConnectionResetError("Cannot write to closing transport") + if SKIP_WRITELINES or size < MIN_PAYLOAD_FOR_WRITELINES: + transport.write(b"".join(chunks)) + else: + transport.writelines(chunks) + + def _write_chunked_payload(self, chunk: Buffer) -> None: + """Write a chunk with proper chunked encoding.""" + chunk_len_pre = f"{len(chunk):x}\r\n".encode("ascii") + self._writelines((chunk_len_pre, chunk, b"\r\n")) + + def _send_headers_with_payload(self, chunk: Buffer, is_eof: bool) -> None: + """Send buffered headers with payload, coalescing into single write.""" + # Mark headers as written + self._headers_written = True + headers_buf = self._headers_buf + self._headers_buf = None + + if TYPE_CHECKING: + # Safe because callers (write() and write_eof()) only invoke this method + # after checking that self._headers_buf is truthy + assert headers_buf is not None + + if not self.chunked: + # Non-chunked: coalesce headers with body + if chunk: + self._writelines((headers_buf, chunk)) + else: + self._write(headers_buf) + return + + # Coalesce headers with chunked data + if chunk: + chunk_len_pre = f"{len(chunk):x}\r\n".encode("ascii") + if is_eof: + self._writelines((headers_buf, chunk_len_pre, chunk, b"\r\n0\r\n\r\n")) + else: + self._writelines((headers_buf, chunk_len_pre, chunk, b"\r\n")) + elif is_eof: + self._writelines((headers_buf, b"0\r\n\r\n")) + else: + self._write(headers_buf) + + async def write( + self, chunk: Buffer, *, drain: bool = True, LIMIT: int = 0x10000 + ) -> None: + """ + Writes chunk of data to a stream. + + write_eof() indicates end of stream. + writer can't be used after write_eof() method being called. + write() return drain future. + """ + if self._on_chunk_sent is not None: + await self._on_chunk_sent(chunk) + + if isinstance(chunk, memoryview): + if chunk.nbytes != len(chunk): + # just reshape it + chunk = chunk.cast("c") + + if self._compress is not None: + chunk = await self._compress.compress(chunk) + if not chunk: + return + + if self.length is not None: + chunk_len = len(chunk) + if self.length >= chunk_len: + self.length = self.length - chunk_len + else: + chunk = chunk[: self.length] + self.length = 0 + if not chunk: + return + + # Handle buffered headers for small payload optimization + if self._headers_buf and not self._headers_written: + self._send_headers_with_payload(chunk, False) + if drain and self.buffer_size > LIMIT: + self.buffer_size = 0 + await self.drain() + return + + if chunk: + if self.chunked: + self._write_chunked_payload(chunk) + else: + self._write(chunk) + + if drain and self.buffer_size > LIMIT: + self.buffer_size = 0 + await self.drain() + + async def write_headers( + self, status_line: str, headers: "CIMultiDict[str]" + ) -> None: + """Write headers to the stream.""" + if self._on_headers_sent is not None: + await self._on_headers_sent(headers) + # status + headers + buf = _serialize_headers(status_line, headers) + self._headers_written = False + self._headers_buf = buf + + def send_headers(self) -> None: + """Force sending buffered headers if not already sent.""" + if not self._headers_buf or self._headers_written: + return + + self._headers_written = True + headers_buf = self._headers_buf + self._headers_buf = None + + if TYPE_CHECKING: + # Safe because we only enter this block when self._headers_buf is truthy + assert headers_buf is not None + + self._write(headers_buf) + + def set_eof(self) -> None: + """Indicate that the message is complete.""" + if self._eof: + return + + # If headers haven't been sent yet, send them now + # This handles the case where there's no body at all + if self._headers_buf and not self._headers_written: + self._headers_written = True + headers_buf = self._headers_buf + self._headers_buf = None + + if TYPE_CHECKING: + # Safe because we only enter this block when self._headers_buf is truthy + assert headers_buf is not None + + # Combine headers and chunked EOF marker in a single write + if self.chunked: + self._writelines((headers_buf, b"0\r\n\r\n")) + else: + self._write(headers_buf) + elif self.chunked and self._headers_written: + # Headers already sent, just send the final chunk marker + self._write(b"0\r\n\r\n") + + self._eof = True + + async def write_eof(self, chunk: bytes = b"") -> None: + if self._eof: + return + + if chunk and self._on_chunk_sent is not None: + await self._on_chunk_sent(chunk) + + # Handle body/compression + if self._compress: + chunks: list[bytes] = [] + chunks_len = 0 + if chunk and (compressed_chunk := await self._compress.compress(chunk)): + chunks_len = len(compressed_chunk) + chunks.append(compressed_chunk) + + flush_chunk = self._compress.flush() + chunks_len += len(flush_chunk) + chunks.append(flush_chunk) + assert chunks_len + + # Send buffered headers with compressed data if not yet sent + if self._headers_buf and not self._headers_written: + self._headers_written = True + headers_buf = self._headers_buf + self._headers_buf = None + + if self.chunked: + # Coalesce headers with compressed chunked data + chunk_len_pre = f"{chunks_len:x}\r\n".encode("ascii") + self._writelines( + (headers_buf, chunk_len_pre, *chunks, b"\r\n0\r\n\r\n") + ) + else: + # Coalesce headers with compressed data + self._writelines((headers_buf, *chunks)) + await self.drain() + self._eof = True + return + + # Headers already sent, just write compressed data + if self.chunked: + chunk_len_pre = f"{chunks_len:x}\r\n".encode("ascii") + self._writelines((chunk_len_pre, *chunks, b"\r\n0\r\n\r\n")) + elif len(chunks) > 1: + self._writelines(chunks) + else: + self._write(chunks[0]) + await self.drain() + self._eof = True + return + + # No compression - send buffered headers if not yet sent + if self._headers_buf and not self._headers_written: + # Use helper to send headers with payload + self._send_headers_with_payload(chunk, True) + await self.drain() + self._eof = True + return + + # Handle remaining body + if self.chunked: + if chunk: + # Write final chunk with EOF marker + self._writelines( + (f"{len(chunk):x}\r\n".encode("ascii"), chunk, b"\r\n0\r\n\r\n") + ) + else: + self._write(b"0\r\n\r\n") + await self.drain() + self._eof = True + return + + if chunk: + self._write(chunk) + await self.drain() + + self._eof = True + + async def drain(self) -> None: + """Flush the write buffer. + + The intended use is to write + + await w.write(data) + await w.drain() + """ + protocol = self._protocol + if protocol.transport is not None and protocol._paused: + await protocol._drain_helper() + + +# https://www.rfc-editor.org/info/rfc9110/#section-5.5-5 +# https://www.rfc-editor.org/info/rfc9112/#section-4-3 +_FORBIDDEN_HEADER_CHARS_RE = re.compile(r"[\x00-\x08\x0a-\x1f\x7f]") + + +def _safe_header(string: str) -> str: + if _FORBIDDEN_HEADER_CHARS_RE.search(string) is not None: + raise ValueError( + "Forbidden control character detected in headers. " + "Potential header injection attack." + ) + return string + + +def _py_serialize_headers(status_line: str, headers: "CIMultiDict[str]") -> bytes: + _safe_header(status_line) + headers_gen = (_safe_header(k) + ": " + _safe_header(v) for k, v in headers.items()) + line = status_line + "\r\n" + "\r\n".join(headers_gen) + "\r\n\r\n" + return line.encode("utf-8") + + +_serialize_headers = _py_serialize_headers + +try: + import aiohttp._http_writer as _http_writer # type: ignore[import-not-found] + + _c_serialize_headers = _http_writer._serialize_headers + if not NO_EXTENSIONS: + _serialize_headers = _c_serialize_headers +except ImportError: + pass diff --git a/py_modules/aiohttp/log.py b/py_modules/aiohttp/log.py new file mode 100644 index 0000000..3cecea2 --- /dev/null +++ b/py_modules/aiohttp/log.py @@ -0,0 +1,8 @@ +import logging + +access_logger = logging.getLogger("aiohttp.access") +client_logger = logging.getLogger("aiohttp.client") +internal_logger = logging.getLogger("aiohttp.internal") +server_logger = logging.getLogger("aiohttp.server") +web_logger = logging.getLogger("aiohttp.web") +ws_logger = logging.getLogger("aiohttp.websocket") diff --git a/py_modules/aiohttp/multipart.py b/py_modules/aiohttp/multipart.py new file mode 100644 index 0000000..b5b87f7 --- /dev/null +++ b/py_modules/aiohttp/multipart.py @@ -0,0 +1,1247 @@ +import base64 +import binascii +import builtins +import json +import re +import sys +import uuid +import warnings +from collections import deque +from collections.abc import AsyncIterator, Iterator, Mapping, Sequence +from types import TracebackType +from typing import TYPE_CHECKING, Any, TypeVar, Union, cast +from urllib.parse import parse_qsl, unquote, urlencode + +from multidict import CIMultiDict, CIMultiDictProxy + +from .abc import AbstractStreamWriter +from .compression_utils import ZLibCompressor, ZLibDecompressor +from .hdrs import ( + CONTENT_DISPOSITION, + CONTENT_ENCODING, + CONTENT_LENGTH, + CONTENT_TRANSFER_ENCODING, + CONTENT_TYPE, +) +from .helpers import CHAR, DEFAULT_CHUNK_SIZE, TOKEN, parse_mimetype, reify +from .http import HeadersParser +from .http_exceptions import BadHttpMessage +from .log import internal_logger +from .payload import ( + JsonPayload, + LookupError, + Order, + Payload, + StringPayload, + get_payload, + payload_type, +) +from .streams import StreamReader + +if sys.version_info >= (3, 11): + from typing import Self +else: + Self = TypeVar("Self", bound="BodyPartReader") + +if sys.version_info >= (3, 12): + from collections.abc import Buffer +else: + Buffer = Union[bytes, bytearray, "memoryview[int]", "memoryview[bytes]"] + +_Buffer = TypeVar("_Buffer", bound=Buffer) + +__all__ = ( + "MultipartReader", + "MultipartWriter", + "BodyPartReader", + "BadContentDispositionHeader", + "BadContentDispositionParam", + "parse_content_disposition", + "content_disposition_filename", +) + + +if TYPE_CHECKING: + from .client_reqrep import ClientResponse + + +class BadContentDispositionHeader(RuntimeWarning): + pass + + +class BadContentDispositionParam(RuntimeWarning): + pass + + +def parse_content_disposition( + header: str | None, +) -> tuple[str | None, dict[str, str]]: + def is_token(string: str) -> bool: + return bool(string) and TOKEN >= set(string) + + def is_quoted(string: str) -> bool: + return len(string) >= 2 and string[0] == string[-1] == '"' + + def is_rfc5987(string: str) -> bool: + return is_token(string) and string.count("'") == 2 + + def is_extended_param(string: str) -> bool: + return string.endswith("*") + + def is_continuous_param(string: str) -> bool: + pos = string.find("*") + 1 + if not pos: + return False + substring = string[pos:-1] if string.endswith("*") else string[pos:] + return substring.isdigit() + + def unescape(text: str, *, chars: str = "".join(map(re.escape, CHAR))) -> str: + return re.sub(f"\\\\([{chars}])", "\\1", text) + + if not header: + return None, {} + + # https://www.rfc-editor.org/info/rfc9110/#section-5.6.6-2 + disptype, *parts = header.split(";") + disptype = disptype.strip() + if not is_token(disptype): + warnings.warn(BadContentDispositionHeader(header)) + return None, {} + + params: dict[str, str] = {} + while parts: + item = parts.pop(0) + + if not item: # To handle trailing semicolons + warnings.warn(BadContentDispositionHeader(header)) + continue + + if "=" not in item: + warnings.warn(BadContentDispositionHeader(header)) + return None, {} + + key, value = item.split("=", 1) + key = key.lower().strip() + value = value.lstrip() + + if key in params: + warnings.warn(BadContentDispositionHeader(header)) + return None, {} + + if not is_token(key): + warnings.warn(BadContentDispositionParam(item)) + continue + + elif is_continuous_param(key): + if is_quoted(value): + value = unescape(value[1:-1]) + elif not is_token(value): + warnings.warn(BadContentDispositionParam(item)) + continue + + elif is_extended_param(key): + if is_rfc5987(value): + encoding, _, value = value.split("'", 2) + encoding = encoding or "utf-8" + else: + warnings.warn(BadContentDispositionParam(item)) + continue + + try: + value = unquote(value, encoding, "strict") + except (builtins.LookupError, UnicodeDecodeError): + # The charset is attacker-controlled here; an unknown name + # raises the builtin LookupError (the bare name is shadowed in + # this module by payload.LookupError). + warnings.warn(BadContentDispositionParam(item)) + continue + + else: + failed = True + rstripped = value.rstrip() + if is_quoted(rstripped): + failed = False + value = unescape(rstripped[1:-1].lstrip("\\/")) + elif is_token(value): + failed = False + elif parts: + # maybe just ; in filename, in any case this is just + # one case fix, for proper fix we need to redesign parser + _value = f"{value};{parts[0]}" + if is_quoted(_value): + parts.pop(0) + value = unescape(_value[1:-1].lstrip("\\/")) + failed = False + + if failed: + warnings.warn(BadContentDispositionHeader(header)) + return None, {} + + params[key] = value + + return disptype.lower(), params + + +def content_disposition_filename( + params: Mapping[str, str], name: str = "filename" +) -> str | None: + name_suf = "%s*" % name + if not params: + return None + elif name_suf in params: + return params[name_suf] + elif name in params: + return params[name] + else: + parts = [] + fnparams = sorted( + (key, value) for key, value in params.items() if key.startswith(name_suf) + ) + for num, (key, value) in enumerate(fnparams): + _, tail = key.split("*", 1) + if tail.endswith("*"): + tail = tail[:-1] + if tail == str(num): + parts.append(value) + else: + break + if not parts: + return None + value = "".join(parts) + if "'" in value: + encoding, _, value = value.split("'", 2) + encoding = encoding or "utf-8" + try: + return unquote(value, encoding, "strict") + except (builtins.LookupError, UnicodeDecodeError): + # Both the charset name and the octets are attacker-controlled + # here; an unknown encoding raises the builtin LookupError + # (shadowed in this module by payload.LookupError) and + # undecodable bytes raise UnicodeDecodeError. + return None + return value + + +class MultipartResponseWrapper: + """Wrapper around the MultipartReader. + + It takes care about + underlying connection and close it when it needs in. + """ + + def __init__( + self, + resp: "ClientResponse", + stream: "MultipartReader", + ) -> None: + self.resp = resp + self.stream = stream + + def __aiter__(self) -> "MultipartResponseWrapper": + return self + + async def __anext__( + self, + ) -> Union["MultipartReader", "BodyPartReader"]: + part = await self.next() + if part is None: + raise StopAsyncIteration + return part + + def at_eof(self) -> bool: + """Returns True when all response data had been read.""" + return self.resp.content.at_eof() + + async def next( + self, + ) -> Union["MultipartReader", "BodyPartReader"] | None: + """Emits next multipart reader object.""" + item = await self.stream.next() + if self.stream.at_eof(): + await self.release() + return item + + async def release(self) -> None: + """Release the connection gracefully. + + All remaining content is read to the void. + """ + await self.resp.release() + + +class BodyPartReader: + """Multipart reader for single body part.""" + + chunk_size = 8192 + + def __init__( + self, + boundary: bytes, + headers: "CIMultiDictProxy[str]", + content: StreamReader, + *, + subtype: str = "mixed", + default_charset: str | None = None, + max_decompress_size: int = DEFAULT_CHUNK_SIZE, + client_max_size: int = sys.maxsize, + max_size_error_cls: type[Exception] = ValueError, + ) -> None: + self.headers = headers + self._boundary = boundary + self._boundary_len = len(boundary) + 2 # Boundary + \r\n + self._content = content + self._default_charset = default_charset + self._at_eof = False + self._is_form_data = subtype == "form-data" + # https://datatracker.ietf.org/doc/html/rfc7578#section-4.8 + length = None if self._is_form_data else self.headers.get(CONTENT_LENGTH, None) + if length is not None and not (length.isascii() and length.isdigit()): + # Reject sign prefixes, underscores, whitespace and non-ASCII + # digits that int() would otherwise accept. + # https://www.rfc-editor.org/rfc/rfc9110#section-8.6 + raise ValueError(f"invalid Content-Length: {length!r}") + self._length = int(length) if length is not None else None + self._read_bytes = 0 + self._unread: deque[bytes] = deque() + self._prev_chunk: bytes | None = None + self._content_eof = 0 + self._cache: dict[str, Any] = {} + self._max_decompress_size = max_decompress_size + self._client_max_size = client_max_size + self._max_size_error_cls = max_size_error_cls + + def __aiter__(self: Self) -> Self: + return self + + async def __anext__(self) -> bytes: + part = await self.next() + if part is None: + raise StopAsyncIteration + return part + + async def next(self) -> bytes | None: + item = await self.read() + if not item: + return None + return item + + async def read(self, *, decode: bool = False) -> bytes: + """Reads body part data. + + decode: Decodes data following by encoding + method from Content-Encoding header. If it missed + data remains untouched + """ + if self._at_eof: + return b"" + data = bytearray() + while not self._at_eof: + data.extend(await self.read_chunk(self.chunk_size)) + if len(data) > self._client_max_size: + raise self._max_size_error_cls(self._client_max_size) + if decode: + decoded_data = bytearray() + async for d in self.decode_iter(data): + decoded_data.extend(d) + if len(decoded_data) > self._client_max_size: + raise self._max_size_error_cls(self._client_max_size) + return decoded_data + return data + + async def read_chunk(self, size: int = chunk_size) -> bytes: + """Reads body part content chunk of the specified size. + + size: chunk size + """ + if self._at_eof: + return b"" + if self._length: + chunk = await self._read_chunk_from_length(size) + else: + chunk = await self._read_chunk_from_stream(size) + + # For the case of base64 data, we must read a fragment of size with a + # remainder of 0 by dividing by 4 for string without symbols \n or \r + encoding = self.headers.get(CONTENT_TRANSFER_ENCODING) + if encoding and encoding.lower() == "base64": + stripped_chunk = b"".join(chunk.split()) + remainder = len(stripped_chunk) % 4 + + while remainder != 0 and not self.at_eof(): + over_chunk_size = 4 - remainder + over_chunk = b"" + + if self._prev_chunk: + over_chunk = self._prev_chunk[:over_chunk_size] + self._prev_chunk = self._prev_chunk[len(over_chunk) :] + + if len(over_chunk) != over_chunk_size: + over_chunk += await self._content.read(4 - len(over_chunk)) + + if not over_chunk: + self._at_eof = True + + stripped_chunk += b"".join(over_chunk.split()) + chunk += over_chunk + remainder = len(stripped_chunk) % 4 + + self._read_bytes += len(chunk) + if self._read_bytes == self._length: + self._at_eof = True + if self._at_eof and await self._content.readline() != b"\r\n": + raise ValueError("Reader did not read all the data or it is malformed") + return chunk + + async def _read_chunk_from_length(self, size: int) -> bytes: + # Reads body part content chunk of the specified size. + # The body part must has Content-Length header with proper value. + assert self._length is not None, "Content-Length required for chunked read" + chunk_size = min(size, self._length - self._read_bytes) + chunk = await self._content.read(chunk_size) + if self._content.at_eof(): + self._at_eof = True + return chunk + + async def _read_chunk_from_stream(self, size: int) -> bytes: + # Reads content chunk of body part with unknown length. + # The Content-Length header for body part is not necessary. + assert ( + size >= self._boundary_len + ), "Chunk size must be greater or equal than boundary length + 2" + first_chunk = self._prev_chunk is None + if first_chunk: + # We need to re-add the CRLF that got removed from headers parsing. + self._prev_chunk = b"\r\n" + await self._content.read(size) + + chunk = b"" + # content.read() may return less than size, so we need to loop to ensure + # we have enough data to detect the boundary. + while len(chunk) < self._boundary_len: + chunk += await self._content.read(size) + self._content_eof += int(self._content.at_eof()) + if self._content_eof > 2: + raise ValueError("Reading after EOF") + if self._content_eof: + break + if len(chunk) > size: + self._content.unread_data(chunk[size:]) + chunk = chunk[:size] + + assert self._prev_chunk is not None + window = self._prev_chunk + chunk + sub = b"\r\n" + self._boundary + if first_chunk: + idx = window.find(sub) + else: + idx = window.find(sub, max(0, len(self._prev_chunk) - len(sub))) + if idx >= 0: + # pushing boundary back to content + with warnings.catch_warnings(): + warnings.filterwarnings("ignore", category=DeprecationWarning) + self._content.unread_data(window[idx:]) + self._prev_chunk = self._prev_chunk[:idx] + chunk = window[len(self._prev_chunk) : idx] + if not chunk: + self._at_eof = True + result = self._prev_chunk[2 if first_chunk else 0 :] # Strip initial CRLF + self._prev_chunk = chunk + return result + + async def readline(self) -> bytes: + """Reads body part by line by line.""" + if self._at_eof: + return b"" + + if self._unread: + line = self._unread.popleft() + else: + line = await self._content.readline() + + if line.startswith(self._boundary): + # the very last boundary may not come with \r\n, + # so set single rules for everyone + sline = line.rstrip(b"\r\n") + boundary = self._boundary + last_boundary = self._boundary + b"--" + # ensure that we read exactly the boundary, not something alike + if sline == boundary or sline == last_boundary: + self._at_eof = True + self._unread.append(line) + return b"" + else: + next_line = await self._content.readline() + if next_line.startswith(self._boundary): + line = line[:-2] # strip CRLF but only once + self._unread.append(next_line) + + return line + + async def release(self) -> None: + """Like read(), but reads all the data to the void.""" + if self._at_eof: + return + while not self._at_eof: + await self.read_chunk(self.chunk_size) + + async def text(self, *, encoding: str | None = None) -> str: + """Like read(), but assumes that body part contains text data.""" + data = await self.read(decode=True) + # see https://www.w3.org/TR/html5/forms.html#multipart/form-data-encoding-algorithm + # and https://dvcs.w3.org/hg/xhr/raw-file/tip/Overview.html#dom-xmlhttprequest-send + encoding = encoding or self.get_charset(default="utf-8") + return data.decode(encoding) + + async def json(self, *, encoding: str | None = None) -> dict[str, Any] | None: + """Like read(), but assumes that body parts contains JSON data.""" + data = await self.read(decode=True) + if not data: + return None + encoding = encoding or self.get_charset(default="utf-8") + return cast(dict[str, Any], json.loads(data.decode(encoding))) + + async def form(self, *, encoding: str | None = None) -> list[tuple[str, str]]: + """Like read(), but assumes that body parts contain form urlencoded data.""" + data = await self.read(decode=True) + if not data: + return [] + if encoding is not None: + real_encoding = encoding + else: + real_encoding = self.get_charset(default="utf-8") + try: + decoded_data = data.rstrip().decode(real_encoding) + except UnicodeDecodeError: + raise ValueError("data cannot be decoded with %s encoding" % real_encoding) + + return parse_qsl( + decoded_data, + keep_blank_values=True, + encoding=real_encoding, + ) + + def at_eof(self) -> bool: + """Returns True if the boundary was reached or False otherwise.""" + return self._at_eof + + def _apply_content_transfer_decoding(self, data: _Buffer) -> _Buffer | bytes: + """Apply Content-Transfer-Encoding decoding if header is present.""" + if CONTENT_TRANSFER_ENCODING in self.headers: + return self._decode_content_transfer(data) + return data + + def _needs_content_decoding(self) -> bool: + """Check if Content-Encoding decoding should be applied.""" + # https://datatracker.ietf.org/doc/html/rfc7578#section-4.8 + return not self._is_form_data and CONTENT_ENCODING in self.headers + + def decode(self, data: _Buffer) -> _Buffer | bytes: + """Decodes data synchronously. + + Decodes data according the specified Content-Encoding + or Content-Transfer-Encoding headers value. + + Note: For large payloads, consider using decode_iter() instead + to avoid blocking the event loop during decompression. + """ + decoded = self._apply_content_transfer_decoding(data) + if self._needs_content_decoding(): + return self._decode_content(decoded) + return decoded + + async def decode_iter(self, data: _Buffer) -> AsyncIterator[_Buffer | bytes]: + """Async generator that yields decoded data chunks. + + Decodes data according the specified Content-Encoding + or Content-Transfer-Encoding headers value. + + This method offloads decompression to an executor for large payloads + to avoid blocking the event loop. + """ + decoded = self._apply_content_transfer_decoding(data) + if self._needs_content_decoding(): + async for d in self._decode_content_async(decoded): + yield d + else: + yield decoded + + def _decode_content(self, data: _Buffer) -> _Buffer | bytes: + encoding = self.headers.get(CONTENT_ENCODING, "").lower() + if encoding == "identity": + return data + if encoding in {"deflate", "gzip"}: + return ZLibDecompressor( + encoding=encoding, + suppress_deflate_header=True, + ).decompress_sync(data, max_length=self._max_decompress_size) + + raise RuntimeError(f"unknown content encoding: {encoding}") + + async def _decode_content_async( + self, data: _Buffer + ) -> AsyncIterator[_Buffer | bytes]: + encoding = self.headers.get(CONTENT_ENCODING, "").lower() + if encoding == "identity": + yield data + elif encoding in {"deflate", "gzip"}: + d = ZLibDecompressor( + encoding=encoding, + suppress_deflate_header=True, + ) + yield await d.decompress(data, max_length=self._max_decompress_size) + while d.data_available: + yield await d.decompress(b"", max_length=self._max_decompress_size) + else: + raise RuntimeError(f"unknown content encoding: {encoding}") + + def _decode_content_transfer(self, data: _Buffer) -> _Buffer | bytes: + encoding = self.headers.get(CONTENT_TRANSFER_ENCODING, "").lower() + + if encoding == "base64": + return base64.b64decode(data) + elif encoding == "quoted-printable": + return binascii.a2b_qp(data) + elif encoding in ("binary", "8bit", "7bit"): + return data + else: + raise RuntimeError(f"unknown content transfer encoding: {encoding}") + + def get_charset(self, default: str) -> str: + """Returns charset parameter from Content-Type header or default.""" + ctype = self.headers.get(CONTENT_TYPE, "") + mimetype = parse_mimetype(ctype) + return mimetype.parameters.get("charset", self._default_charset or default) + + @reify + def name(self) -> str | None: + """Returns name specified in Content-Disposition header. + + If the header is missing or malformed, returns None. + """ + _, params = parse_content_disposition(self.headers.get(CONTENT_DISPOSITION)) + return content_disposition_filename(params, "name") + + @reify + def filename(self) -> str | None: + """Returns filename specified in Content-Disposition header. + + Returns None if the header is missing or malformed. + """ + _, params = parse_content_disposition(self.headers.get(CONTENT_DISPOSITION)) + return content_disposition_filename(params, "filename") + + +@payload_type(BodyPartReader, order=Order.try_first) +class BodyPartReaderPayload(Payload): + _value: BodyPartReader + # _autoclose = False (inherited) - Streaming reader that may have resources + + def __init__(self, value: BodyPartReader, *args: Any, **kwargs: Any) -> None: + super().__init__(value, *args, **kwargs) + + params: dict[str, str] = {} + if value.name is not None: + params["name"] = value.name + if value.filename is not None: + params["filename"] = value.filename + + if params: + self.set_content_disposition("attachment", True, **params) + + def decode(self, encoding: str = "utf-8", errors: str = "strict") -> str: + raise TypeError("Unable to decode.") + + async def as_bytes(self, encoding: str = "utf-8", errors: str = "strict") -> bytes: + """Raises TypeError as body parts should be consumed via write(). + + This is intentional: BodyPartReader payloads are designed for streaming + large data (potentially gigabytes) and must be consumed only once via + the write() method to avoid memory exhaustion. They cannot be buffered + in memory for reuse. + """ + raise TypeError("Unable to read body part as bytes. Use write() to consume.") + + async def write(self, writer: AbstractStreamWriter) -> None: + field = self._value + while chunk := await field.read_chunk(size=DEFAULT_CHUNK_SIZE): + async for d in field.decode_iter(chunk): + await writer.write(d) + + +class MultipartReader: + """Multipart body reader.""" + + #: Response wrapper, used when multipart readers constructs from response. + response_wrapper_cls = MultipartResponseWrapper + #: Multipart reader class, used to handle multipart/* body parts. + #: None points to type(self) + multipart_reader_cls: type["MultipartReader"] | None = None + #: Body part reader class for non multipart/* content types. + part_reader_cls = BodyPartReader + + def __init__( + self, + headers: Mapping[str, str], + content: StreamReader, + *, + client_max_size: int = sys.maxsize, + max_field_size: int = 8190, + max_headers: int = 128, + max_size_error_cls: type[Exception] = ValueError, + ) -> None: + self._mimetype = parse_mimetype(headers[CONTENT_TYPE]) + assert self._mimetype.type == "multipart", "multipart/* content type expected" + if "boundary" not in self._mimetype.parameters: + raise ValueError( + "boundary missed for Content-Type: %s" % headers[CONTENT_TYPE] + ) + + self.headers = headers + self._boundary = ("--" + self._get_boundary()).encode() + self._client_max_size = client_max_size + self._content = content + self._default_charset: str | None = None + self._last_part: MultipartReader | BodyPartReader | None = None + self._max_field_size = max_field_size + self._max_headers = max_headers + self._max_size_error_cls = max_size_error_cls + self._at_eof = False + self._at_bof = True + self._unread: list[bytes] = [] + + def __aiter__(self: Self) -> Self: + return self + + async def __anext__( + self, + ) -> Union["MultipartReader", BodyPartReader] | None: + part = await self.next() + if part is None: + raise StopAsyncIteration + return part + + @classmethod + def from_response( + cls, + response: "ClientResponse", + ) -> MultipartResponseWrapper: + """Constructs reader instance from HTTP response. + + :param response: :class:`~aiohttp.client.ClientResponse` instance + """ + obj = cls.response_wrapper_cls( + response, cls(response.headers, response.content) + ) + return obj + + def at_eof(self) -> bool: + """Returns True if the final boundary was reached, false otherwise.""" + return self._at_eof + + async def next( + self, + ) -> Union["MultipartReader", BodyPartReader] | None: + """Emits the next multipart body part.""" + # So, if we're at BOF, we need to skip till the boundary. + if self._at_eof: + return None + await self._maybe_release_last_part() + if self._at_bof: + await self._read_until_first_boundary() + self._at_bof = False + else: + await self._read_boundary() + if self._at_eof: # we just read the last boundary, nothing to do there + return None + + part = await self.fetch_next_part() + # https://datatracker.ietf.org/doc/html/rfc7578#section-4.6 + if ( + self._last_part is None + and self._mimetype.subtype == "form-data" + and isinstance(part, BodyPartReader) + ): + _, params = parse_content_disposition(part.headers.get(CONTENT_DISPOSITION)) + if params.get("name") == "_charset_": + # Longest encoding in https://encoding.spec.whatwg.org/encodings.json + # is 19 characters, so 32 should be more than enough for any valid encoding. + charset = await part.read_chunk(32) + if len(charset) > 31: + raise RuntimeError("Invalid default charset") + self._default_charset = charset.strip().decode() + part = await self.fetch_next_part() + self._last_part = part + return self._last_part + + async def release(self) -> None: + """Reads all the body parts to the void till the final boundary.""" + while not self._at_eof: + item = await self.next() + if item is None: + break + await item.release() + + async def fetch_next_part( + self, + ) -> Union["MultipartReader", BodyPartReader]: + """Returns the next body part reader.""" + headers = await self._read_headers() + return self._get_part_reader(headers) + + def _get_part_reader( + self, + headers: "CIMultiDictProxy[str]", + ) -> Union["MultipartReader", BodyPartReader]: + """Dispatches the response by the `Content-Type` header. + + Returns a suitable reader instance. + + :param dict headers: Response headers + """ + ctype = headers.get(CONTENT_TYPE, "") + mimetype = parse_mimetype(ctype) + + if mimetype.type == "multipart": + if self.multipart_reader_cls is None: + return type(self)( + headers, + self._content, + client_max_size=self._client_max_size, + max_field_size=self._max_field_size, + max_headers=self._max_headers, + max_size_error_cls=self._max_size_error_cls, + ) + return self.multipart_reader_cls( + headers, + self._content, + client_max_size=self._client_max_size, + max_field_size=self._max_field_size, + max_headers=self._max_headers, + max_size_error_cls=self._max_size_error_cls, + ) + else: + return self.part_reader_cls( + self._boundary, + headers, + self._content, + subtype=self._mimetype.subtype, + default_charset=self._default_charset, + client_max_size=self._client_max_size, + max_size_error_cls=self._max_size_error_cls, + ) + + def _get_boundary(self) -> str: + boundary = self._mimetype.parameters["boundary"] + if len(boundary) > 70: + raise ValueError("boundary %r is too long (70 chars max)" % boundary) + + return boundary + + async def _readline(self) -> bytes: + if self._unread: + return self._unread.pop() + return await self._content.readline() + + async def _read_until_first_boundary(self) -> None: + while True: + chunk = await self._readline() + if chunk == b"": + raise ValueError( + "Could not find starting boundary %r" % (self._boundary) + ) + chunk = chunk.rstrip() + if chunk == self._boundary: + return + elif chunk == self._boundary + b"--": + self._at_eof = True + return + + async def _read_boundary(self) -> None: + chunk = (await self._readline()).rstrip() + if chunk == self._boundary: + pass + elif chunk == self._boundary + b"--": + self._at_eof = True + epilogue = await self._readline() + next_line = await self._readline() + + # the epilogue is expected and then either the end of input or the + # parent multipart boundary, if the parent boundary is found then + # it should be marked as unread and handed to the parent for + # processing + if next_line[:2] == b"--": + self._unread.append(next_line) + # otherwise the request is likely missing an epilogue and both + # lines should be passed to the parent for processing + # (this handles the old behavior gracefully) + else: + self._unread.extend([next_line, epilogue]) + else: + raise ValueError(f"Invalid boundary {chunk!r}, expected {self._boundary!r}") + + async def _read_headers(self) -> "CIMultiDictProxy[str]": + lines = [] + while True: + chunk = await self._content.readline(max_line_length=self._max_field_size) + chunk = chunk.rstrip(b"\r\n") + lines.append(chunk) + if not chunk: + break + if len(lines) > self._max_headers: + raise BadHttpMessage("Too many headers received") + parser = HeadersParser(max_field_size=self._max_field_size) + headers, raw_headers = parser.parse_headers(lines) + return headers + + async def _maybe_release_last_part(self) -> None: + """Ensures that the last read body part is read completely.""" + if self._last_part is not None: + if not self._last_part.at_eof(): + await self._last_part.release() + self._unread.extend(self._last_part._unread) + self._last_part = None + + +_Part = tuple[Payload, str, str] + + +class MultipartWriter(Payload): + """Multipart body writer.""" + + _value: None + # _consumed = False (inherited) - Can be encoded multiple times + _autoclose = True # No file handles, just collects parts in memory + + def __init__(self, subtype: str = "mixed", boundary: str | None = None) -> None: + boundary = boundary if boundary is not None else uuid.uuid4().hex + # The underlying Payload API demands a str (utf-8), not bytes, + # so we need to ensure we don't lose anything during conversion. + # As a result, require the boundary to be ASCII only. + # In both situations. + + try: + self._boundary = boundary.encode("ascii") + except UnicodeEncodeError: + raise ValueError("boundary should contain ASCII only chars") from None + ctype = f"multipart/{subtype}; boundary={self._boundary_value}" + + super().__init__(None, content_type=ctype) + + self._parts: list[_Part] = [] + self._is_form_data = subtype == "form-data" + + def __enter__(self) -> "MultipartWriter": + return self + + def __exit__( + self, + exc_type: type[BaseException] | None, + exc_val: BaseException | None, + exc_tb: TracebackType | None, + ) -> None: + pass + + def __iter__(self) -> Iterator[_Part]: + return iter(self._parts) + + def __len__(self) -> int: + return len(self._parts) + + def __bool__(self) -> bool: + return True + + _valid_tchar_regex = re.compile(rb"\A[!#$%&'*+\-.^_`|~\w]+\Z") + _invalid_qdtext_char_regex = re.compile(rb"[\x00-\x08\x0A-\x1F\x7F]") + + @property + def _boundary_value(self) -> str: + """Wrap boundary parameter value in quotes, if necessary. + + Reads self.boundary and returns a unicode string. + """ + # Refer to RFCs 7231, 7230, 5234. + # + # parameter = token "=" ( token / quoted-string ) + # token = 1*tchar + # quoted-string = DQUOTE *( qdtext / quoted-pair ) DQUOTE + # qdtext = HTAB / SP / %x21 / %x23-5B / %x5D-7E / obs-text + # obs-text = %x80-FF + # quoted-pair = "\" ( HTAB / SP / VCHAR / obs-text ) + # tchar = "!" / "#" / "$" / "%" / "&" / "'" / "*" + # / "+" / "-" / "." / "^" / "_" / "`" / "|" / "~" + # / DIGIT / ALPHA + # ; any VCHAR, except delimiters + # VCHAR = %x21-7E + value = self._boundary + if re.match(self._valid_tchar_regex, value): + return value.decode("ascii") # cannot fail + + if re.search(self._invalid_qdtext_char_regex, value): + raise ValueError("boundary value contains invalid characters") + + # escape %x5C and %x22 + quoted_value_content = value.replace(b"\\", b"\\\\") + quoted_value_content = quoted_value_content.replace(b'"', b'\\"') + + return '"' + quoted_value_content.decode("ascii") + '"' + + @property + def boundary(self) -> str: + return self._boundary.decode("ascii") + + def append(self, obj: Any, headers: Mapping[str, str] | None = None) -> Payload: + if headers is None: + headers = CIMultiDict() + + if isinstance(obj, Payload): + obj.headers.update(headers) + return self.append_payload(obj) + else: + try: + payload = get_payload(obj, headers=headers) + except LookupError: + raise TypeError("Cannot create payload from %r" % obj) + else: + return self.append_payload(payload) + + def append_payload(self, payload: Payload) -> Payload: + """Adds a new body part to multipart writer.""" + encoding: str | None = None + te_encoding: str | None = None + if self._is_form_data: + # https://datatracker.ietf.org/doc/html/rfc7578#section-4.7 + # https://datatracker.ietf.org/doc/html/rfc7578#section-4.8 + assert ( + not {CONTENT_ENCODING, CONTENT_LENGTH, CONTENT_TRANSFER_ENCODING} + & payload.headers.keys() + ) + # Set default Content-Disposition in case user doesn't create one + if CONTENT_DISPOSITION not in payload.headers: + name = f"section-{len(self._parts)}" + payload.set_content_disposition("form-data", name=name) + else: + # compression + encoding = payload.headers.get(CONTENT_ENCODING, "").lower() + if encoding and encoding not in ("deflate", "gzip", "identity"): + raise RuntimeError(f"unknown content encoding: {encoding}") + if encoding == "identity": + encoding = None + + # te encoding + te_encoding = payload.headers.get(CONTENT_TRANSFER_ENCODING, "").lower() + if te_encoding not in ("", "base64", "quoted-printable", "binary"): + raise RuntimeError(f"unknown content transfer encoding: {te_encoding}") + if te_encoding == "binary": + te_encoding = None + + # size + size = payload.size + if size is not None and not (encoding or te_encoding): + payload.headers[CONTENT_LENGTH] = str(size) + + self._parts.append((payload, encoding, te_encoding)) # type: ignore[arg-type] + return payload + + def append_json( + self, obj: Any, headers: Mapping[str, str] | None = None + ) -> Payload: + """Helper to append JSON part.""" + if headers is None: + headers = CIMultiDict() + + return self.append_payload(JsonPayload(obj, headers=headers)) + + def append_form( + self, + obj: Sequence[tuple[str, str]] | Mapping[str, str], + headers: Mapping[str, str] | None = None, + ) -> Payload: + """Helper to append form urlencoded part.""" + assert isinstance(obj, (Sequence, Mapping)) + + if headers is None: + headers = CIMultiDict() + + if isinstance(obj, Mapping): + obj = list(obj.items()) + data = urlencode(obj, doseq=True) + + return self.append_payload( + StringPayload( + data, headers=headers, content_type="application/x-www-form-urlencoded" + ) + ) + + @property + def size(self) -> int | None: + """Size of the payload.""" + total = 0 + for part, encoding, te_encoding in self._parts: + part_size = part.size + if encoding or te_encoding or part_size is None: + return None + + total += int( + 2 + + len(self._boundary) + + 2 + + part_size # b'--'+self._boundary+b'\r\n' + + len(part._binary_headers) + + 2 # b'\r\n' + ) + + total += 2 + len(self._boundary) + 4 # b'--'+self._boundary+b'--\r\n' + return total + + def decode(self, encoding: str = "utf-8", errors: str = "strict") -> str: + """Return string representation of the multipart data. + + WARNING: This method may do blocking I/O if parts contain file payloads. + It should not be called in the event loop. Use as_bytes().decode() instead. + """ + return "".join( + "--" + + self.boundary + + "\r\n" + + part._binary_headers.decode(encoding, errors) + + part.decode() + for part, _e, _te in self._parts + ) + + async def as_bytes(self, encoding: str = "utf-8", errors: str = "strict") -> bytes: + """Return bytes representation of the multipart data. + + This method is async-safe and calls as_bytes on underlying payloads. + """ + parts: list[bytes] = [] + + # Process each part + for part, _e, _te in self._parts: + # Add boundary + parts.append(b"--" + self._boundary + b"\r\n") + + # Add headers + parts.append(part._binary_headers) + + # Add payload content using as_bytes for async safety + part_bytes = await part.as_bytes(encoding, errors) + parts.append(part_bytes) + + # Add trailing CRLF + parts.append(b"\r\n") + + # Add closing boundary + parts.append(b"--" + self._boundary + b"--\r\n") + + return b"".join(parts) + + async def write( + self, writer: AbstractStreamWriter, close_boundary: bool = True + ) -> None: + """Write body.""" + for part, encoding, te_encoding in self._parts: + if self._is_form_data: + # https://datatracker.ietf.org/doc/html/rfc7578#section-4.2 + assert CONTENT_DISPOSITION in part.headers + assert "name=" in part.headers[CONTENT_DISPOSITION] + + await writer.write(b"--" + self._boundary + b"\r\n") + await writer.write(part._binary_headers) + + if encoding or te_encoding: + w = MultipartPayloadWriter(writer) + if encoding: + w.enable_compression(encoding) + if te_encoding: + w.enable_encoding(te_encoding) + await part.write(w) # type: ignore[arg-type] + await w.write_eof() + else: + await part.write(writer) + + await writer.write(b"\r\n") + + if close_boundary: + await writer.write(b"--" + self._boundary + b"--\r\n") + + async def close(self) -> None: + """ + Close all part payloads that need explicit closing. + + IMPORTANT: This method must not await anything that might not finish + immediately, as it may be called during cleanup/cancellation. Schedule + any long-running operations without awaiting them. + """ + if self._consumed: + return + self._consumed = True + + # Close all parts that need explicit closing + # We catch and log exceptions to ensure all parts get a chance to close + # we do not use asyncio.gather() here because we are not allowed + # to suspend given we may be called during cleanup + for idx, (part, _, _) in enumerate(self._parts): + if not part.autoclose and not part.consumed: + try: + await part.close() + except Exception as exc: + internal_logger.error( + "Failed to close multipart part %d: %s", idx, exc, exc_info=True + ) + + +class MultipartPayloadWriter: + def __init__(self, writer: AbstractStreamWriter) -> None: + self._writer = writer + self._encoding: str | None = None + self._compress: ZLibCompressor | None = None + self._encoding_buffer: bytearray | None = None + + def enable_encoding(self, encoding: str) -> None: + if encoding == "base64": + self._encoding = encoding + self._encoding_buffer = bytearray() + elif encoding == "quoted-printable": + self._encoding = "quoted-printable" + + def enable_compression( + self, encoding: str = "deflate", strategy: int | None = None + ) -> None: + self._compress = ZLibCompressor( + encoding=encoding, + suppress_deflate_header=True, + strategy=strategy, + ) + + async def write_eof(self) -> None: + if self._compress is not None: + chunk = self._compress.flush() + if chunk: + self._compress = None + await self.write(chunk) + + if self._encoding == "base64": + if self._encoding_buffer: + await self._writer.write(base64.b64encode(self._encoding_buffer)) + + async def write(self, chunk: bytes) -> None: + if self._compress is not None: + if chunk: + chunk = await self._compress.compress(chunk) + if not chunk: + return + + if self._encoding == "base64": + buf = self._encoding_buffer + assert buf is not None + buf.extend(chunk) + + if buf: + div, mod = divmod(len(buf), 3) + enc_chunk, self._encoding_buffer = (buf[: div * 3], buf[div * 3 :]) + if enc_chunk: + b64chunk = base64.b64encode(enc_chunk) + await self._writer.write(b64chunk) + elif self._encoding == "quoted-printable": + await self._writer.write(binascii.b2a_qp(chunk)) + else: + await self._writer.write(chunk) diff --git a/py_modules/aiohttp/payload.py b/py_modules/aiohttp/payload.py new file mode 100644 index 0000000..b337bcf --- /dev/null +++ b/py_modules/aiohttp/payload.py @@ -0,0 +1,1138 @@ +import asyncio +import enum +import io +import json +import mimetypes +import os +import sys +import warnings +from abc import ABC, abstractmethod +from collections.abc import Iterable +from itertools import chain +from typing import IO, TYPE_CHECKING, Any, Final, TextIO + +from multidict import CIMultiDict + +from . import hdrs +from .abc import AbstractStreamWriter +from .helpers import ( + _SENTINEL, + DEFAULT_CHUNK_SIZE, + content_disposition_header, + guess_filename, + parse_mimetype, + sentinel, +) +from .http_writer import _safe_header +from .streams import StreamReader +from .typedefs import JSONBytesEncoder, JSONEncoder, _CIMultiDict + +__all__ = ( + "PAYLOAD_REGISTRY", + "get_payload", + "payload_type", + "Payload", + "BytesPayload", + "StringPayload", + "IOBasePayload", + "BytesIOPayload", + "BufferedReaderPayload", + "TextIOPayload", + "StringIOPayload", + "JsonPayload", + "JsonBytesPayload", + "AsyncIterablePayload", +) + +TOO_LARGE_BYTES_BODY: Final[int] = 2**20 # 1 MB +_CLOSE_FUTURES: set[asyncio.Future[None]] = set() + + +class LookupError(Exception): + """Raised when no payload factory is found for the given data type.""" + + +class Order(str, enum.Enum): + normal = "normal" + try_first = "try_first" + try_last = "try_last" + + +def get_payload(data: Any, *args: Any, **kwargs: Any) -> "Payload": + return PAYLOAD_REGISTRY.get(data, *args, **kwargs) + + +def register_payload( + factory: type["Payload"], type: Any, *, order: Order = Order.normal +) -> None: + PAYLOAD_REGISTRY.register(factory, type, order=order) + + +class payload_type: + def __init__(self, type: Any, *, order: Order = Order.normal) -> None: + self.type = type + self.order = order + + def __call__(self, factory: type["Payload"]) -> type["Payload"]: + register_payload(factory, self.type, order=self.order) + return factory + + +PayloadType = type["Payload"] +_PayloadRegistryItem = tuple[PayloadType, Any] + + +class PayloadRegistry: + """Payload registry. + + note: we need zope.interface for more efficient adapter search + """ + + __slots__ = ("_first", "_normal", "_last", "_normal_lookup") + + def __init__(self) -> None: + self._first: list[_PayloadRegistryItem] = [] + self._normal: list[_PayloadRegistryItem] = [] + self._last: list[_PayloadRegistryItem] = [] + self._normal_lookup: dict[Any, PayloadType] = {} + + def get( + self, + data: Any, + *args: Any, + _CHAIN: "type[chain[_PayloadRegistryItem]]" = chain, + **kwargs: Any, + ) -> "Payload": + if self._first: + for factory, type_ in self._first: + if isinstance(data, type_): + return factory(data, *args, **kwargs) + # Try the fast lookup first + if lookup_factory := self._normal_lookup.get(type(data)): + return lookup_factory(data, *args, **kwargs) + # Bail early if its already a Payload + if isinstance(data, Payload): + return data + # Fallback to the slower linear search + for factory, type_ in _CHAIN(self._normal, self._last): + if isinstance(data, type_): + return factory(data, *args, **kwargs) + raise LookupError() + + def register( + self, factory: PayloadType, type: Any, *, order: Order = Order.normal + ) -> None: + if order is Order.try_first: + self._first.append((factory, type)) + elif order is Order.normal: + self._normal.append((factory, type)) + if isinstance(type, Iterable): + for t in type: + self._normal_lookup[t] = factory + else: + self._normal_lookup[type] = factory + elif order is Order.try_last: + self._last.append((factory, type)) + else: + raise ValueError(f"Unsupported order {order!r}") + + +class Payload(ABC): + + _default_content_type: str = "application/octet-stream" + _size: int | None = None + _consumed: bool = False # Default: payload has not been consumed yet + _autoclose: bool = False # Default: assume resource needs explicit closing + + def __init__( + self, + value: Any, + headers: ( + _CIMultiDict | dict[str, str] | Iterable[tuple[str, str]] | None + ) = None, + content_type: str | None | _SENTINEL = sentinel, + filename: str | None = None, + encoding: str | None = None, + **kwargs: Any, + ) -> None: + self._encoding = encoding + self._filename = filename + self._headers: _CIMultiDict = CIMultiDict() + self._value = value + if content_type is not sentinel and content_type is not None: + self._headers[hdrs.CONTENT_TYPE] = content_type + elif self._filename is not None: + if sys.version_info >= (3, 13): + guesser = mimetypes.guess_file_type + else: + guesser = mimetypes.guess_type + content_type = guesser(self._filename)[0] + if content_type is None: + content_type = self._default_content_type + self._headers[hdrs.CONTENT_TYPE] = content_type + else: + self._headers[hdrs.CONTENT_TYPE] = self._default_content_type + if headers: + self._headers.update(headers) + + @property + def size(self) -> int | None: + """Size of the payload in bytes. + + Returns the number of bytes that will be transmitted when the payload + is written. For string payloads, this is the size after encoding to bytes, + not the length of the string. + """ + return self._size + + @property + def filename(self) -> str | None: + """Filename of the payload.""" + return self._filename + + @property + def headers(self) -> _CIMultiDict: + """Custom item headers""" + return self._headers + + @property + def _binary_headers(self) -> bytes: + return ( + "".join( + _safe_header(k) + ": " + _safe_header(v) + "\r\n" + for k, v in self.headers.items() + ).encode("utf-8") + + b"\r\n" + ) + + @property + def encoding(self) -> str | None: + """Payload encoding""" + return self._encoding + + @property + def content_type(self) -> str: + """Content type""" + return self._headers[hdrs.CONTENT_TYPE] + + @property + def consumed(self) -> bool: + """Whether the payload has been consumed and cannot be reused.""" + return self._consumed + + @property + def autoclose(self) -> bool: + """ + Whether the payload can close itself automatically. + + Returns True if the payload has no file handles or resources that need + explicit closing. If False, callers must await close() to release resources. + """ + return self._autoclose + + def set_content_disposition( + self, + disptype: str, + quote_fields: bool = True, + _charset: str = "utf-8", + **params: Any, + ) -> None: + """Sets ``Content-Disposition`` header.""" + self._headers[hdrs.CONTENT_DISPOSITION] = content_disposition_header( + disptype, quote_fields=quote_fields, _charset=_charset, **params + ) + + @abstractmethod + def decode(self, encoding: str = "utf-8", errors: str = "strict") -> str: + """ + Return string representation of the value. + + This is named decode() to allow compatibility with bytes objects. + """ + + @abstractmethod + async def write(self, writer: AbstractStreamWriter) -> None: + """ + Write payload to the writer stream. + + Args: + writer: An AbstractStreamWriter instance that handles the actual writing + + This is a legacy method that writes the entire payload without length constraints. + + Important: + For new implementations, use write_with_length() instead of this method. + This method is maintained for backwards compatibility and will eventually + delegate to write_with_length(writer, None) in all implementations. + + All payload subclasses must override this method for backwards compatibility, + but new code should use write_with_length for more flexibility and control. + + """ + + # write_with_length is new in aiohttp 3.12 + # it should be overridden by subclasses + async def write_with_length( + self, writer: AbstractStreamWriter, content_length: int | None + ) -> None: + """ + Write payload with a specific content length constraint. + + Args: + writer: An AbstractStreamWriter instance that handles the actual writing + content_length: Maximum number of bytes to write (None for unlimited) + + This method allows writing payload content with a specific length constraint, + which is particularly useful for HTTP responses with Content-Length header. + + Note: + This is the base implementation that provides backwards compatibility + for subclasses that don't override this method. Specific payload types + should override this method to implement proper length-constrained writing. + + """ + # Backwards compatibility for subclasses that don't override this method + # and for the default implementation + await self.write(writer) + + async def as_bytes(self, encoding: str = "utf-8", errors: str = "strict") -> bytes: + """ + Return bytes representation of the value. + + This is a convenience method that calls decode() and encodes the result + to bytes using the specified encoding. + """ + # Use instance encoding if available, otherwise use parameter + actual_encoding = self._encoding or encoding + return self.decode(actual_encoding, errors).encode(actual_encoding) + + def _close(self) -> None: + """ + Async safe synchronous close operations for backwards compatibility. + + This method exists only for backwards compatibility with code that + needs to clean up payloads synchronously. In the future, we will + drop this method and only support the async close() method. + + WARNING: This method must be safe to call from within the event loop + without blocking. Subclasses should not perform any blocking I/O here. + + WARNING: This method must be called from within an event loop for + certain payload types (e.g., IOBasePayload). Calling it outside an + event loop may raise RuntimeError. + """ + # This is a no-op by default, but subclasses can override it + # for non-blocking cleanup operations. + + async def close(self) -> None: + """ + Close the payload if it holds any resources. + + IMPORTANT: This method must not await anything that might not finish + immediately, as it may be called during cleanup/cancellation. Schedule + any long-running operations without awaiting them. + + In the future, this will be the only close method supported. + """ + self._close() + + +class BytesPayload(Payload): + _value: bytes + # _consumed = False (inherited) - Bytes are immutable and can be reused + _autoclose = True # No file handle, just bytes in memory + + def __init__( + self, value: bytes | bytearray | memoryview, *args: Any, **kwargs: Any + ) -> None: + if "content_type" not in kwargs: + kwargs["content_type"] = "application/octet-stream" + + super().__init__(value, *args, **kwargs) + + if isinstance(value, memoryview): + self._size = value.nbytes + elif isinstance(value, (bytes, bytearray)): + self._size = len(value) + else: + raise TypeError(f"value argument must be byte-ish, not {type(value)!r}") + + if self._size > TOO_LARGE_BYTES_BODY: + kwargs = {"source": self} + warnings.warn( + "Sending a large body directly with raw bytes might" + " lock the event loop. You should probably pass an " + "io.BytesIO object instead", + ResourceWarning, + **kwargs, + ) + + def decode(self, encoding: str = "utf-8", errors: str = "strict") -> str: + return self._value.decode(encoding, errors) + + async def as_bytes(self, encoding: str = "utf-8", errors: str = "strict") -> bytes: + """ + Return bytes representation of the value. + + This method returns the raw bytes content of the payload. + It is equivalent to accessing the _value attribute directly. + """ + return self._value + + async def write(self, writer: AbstractStreamWriter) -> None: + """ + Write the entire bytes payload to the writer stream. + + Args: + writer: An AbstractStreamWriter instance that handles the actual writing + + This method writes the entire bytes content without any length constraint. + + Note: + For new implementations that need length control, use write_with_length(). + This method is maintained for backwards compatibility and is equivalent + to write_with_length(writer, None). + + """ + await writer.write(self._value) + + async def write_with_length( + self, writer: AbstractStreamWriter, content_length: int | None + ) -> None: + """ + Write bytes payload with a specific content length constraint. + + Args: + writer: An AbstractStreamWriter instance that handles the actual writing + content_length: Maximum number of bytes to write (None for unlimited) + + This method writes either the entire byte sequence or a slice of it + up to the specified content_length. For BytesPayload, this operation + is performed efficiently using array slicing. + + """ + if content_length is not None: + await writer.write(self._value[:content_length]) + else: + await writer.write(self._value) + + +class StringPayload(BytesPayload): + def __init__( + self, + value: str, + *args: Any, + encoding: str | None = None, + content_type: str | None = None, + **kwargs: Any, + ) -> None: + + if encoding is None: + if content_type is None: + real_encoding = "utf-8" + content_type = "text/plain; charset=utf-8" + else: + mimetype = parse_mimetype(content_type) + real_encoding = mimetype.parameters.get("charset", "utf-8") + else: + if content_type is None: + content_type = "text/plain; charset=%s" % encoding + real_encoding = encoding + + super().__init__( + value.encode(real_encoding), + encoding=real_encoding, + content_type=content_type, + *args, + **kwargs, + ) + + +class StringIOPayload(StringPayload): + def __init__(self, value: IO[str], *args: Any, **kwargs: Any) -> None: + super().__init__(value.read(), *args, **kwargs) + + +class IOBasePayload(Payload): + _value: io.IOBase + # _consumed = False (inherited) - File can be re-read from the same position + _start_position: int | None = None + # _autoclose = False (inherited) - Has file handle that needs explicit closing + + def __init__( + self, value: IO[Any], disposition: str = "attachment", *args: Any, **kwargs: Any + ) -> None: + if "filename" not in kwargs: + kwargs["filename"] = guess_filename(value) + + super().__init__(value, *args, **kwargs) + + if self._filename is not None and disposition is not None: + if hdrs.CONTENT_DISPOSITION not in self.headers: + self.set_content_disposition(disposition, filename=self._filename) + + def _set_or_restore_start_position(self) -> None: + """Set or restore the start position of the file-like object.""" + if self._start_position is None: + try: + self._start_position = self._value.tell() + except (OSError, AttributeError): + self._consumed = True # Cannot seek, mark as consumed + return + try: + self._value.seek(self._start_position) + except (OSError, AttributeError): + # Failed to seek back - mark as consumed since we've already read + self._consumed = True + + def _read_and_available_len( + self, remaining_content_len: int | None + ) -> tuple[int | None, bytes]: + """ + Read the file-like object and return both its total size and the first chunk. + + Args: + remaining_content_len: Optional limit on how many bytes to read in this operation. + If None, DEFAULT_CHUNK_SIZE will be used as the default chunk size. + + Returns: + A tuple containing: + - The total size of the remaining unread content (None if size cannot be determined) + - The first chunk of bytes read from the file object + + This method is optimized to perform both size calculation and initial read + in a single operation, which is executed in a single executor job to minimize + context switches and file operations when streaming content. + + """ + self._set_or_restore_start_position() + size = self.size # Call size only once since it does I/O + return size, self._value.read( + min( + DEFAULT_CHUNK_SIZE, + size or DEFAULT_CHUNK_SIZE, + remaining_content_len or DEFAULT_CHUNK_SIZE, + ) + ) + + def _read(self, remaining_content_len: int | None) -> bytes: + """ + Read a chunk of data from the file-like object. + + Args: + remaining_content_len: Optional maximum number of bytes to read. + If None, DEFAULT_CHUNK_SIZE will be used as the default chunk size. + + Returns: + A chunk of bytes read from the file object, respecting the + remaining_content_len limit if specified. + + This method is used for subsequent reads during streaming after + the initial _read_and_available_len call has been made. + + """ + return self._value.read(remaining_content_len or DEFAULT_CHUNK_SIZE) # type: ignore[no-any-return] + + @property + def size(self) -> int | None: + """ + Size of the payload in bytes. + + Returns the total size of the payload content from the initial position. + This ensures consistent Content-Length for requests, including 307/308 redirects + where the same payload instance is reused. + + Returns None if the size cannot be determined (e.g., for unseekable streams). + """ + try: + # Store the start position on first access. + # This is critical when the same payload instance is reused (e.g., 307/308 + # redirects). Without storing the initial position, after the payload is + # read once, the file position would be at EOF, which would cause the + # size calculation to return 0 (file_size - EOF position). + # By storing the start position, we ensure the size calculation always + # returns the correct total size for any subsequent use. + if self._start_position is None: + self._start_position = self._value.tell() + + # Return the total size from the start position + # This ensures Content-Length is correct even after reading + return os.fstat(self._value.fileno()).st_size - self._start_position + except (AttributeError, OSError): + return None + + async def write(self, writer: AbstractStreamWriter) -> None: + """ + Write the entire file-like payload to the writer stream. + + Args: + writer: An AbstractStreamWriter instance that handles the actual writing + + This method writes the entire file content without any length constraint. + It delegates to write_with_length() with no length limit for implementation + consistency. + + Note: + For new implementations that need length control, use write_with_length() directly. + This method is maintained for backwards compatibility with existing code. + + """ + await self.write_with_length(writer, None) + + async def write_with_length( + self, writer: AbstractStreamWriter, content_length: int | None + ) -> None: + """ + Write file-like payload with a specific content length constraint. + + Args: + writer: An AbstractStreamWriter instance that handles the actual writing + content_length: Maximum number of bytes to write (None for unlimited) + + This method implements optimized streaming of file content with length constraints: + + 1. File reading is performed in a thread pool to avoid blocking the event loop + 2. Content is read and written in chunks to maintain memory efficiency + 3. Writing stops when either: + - All available file content has been written (when size is known) + - The specified content_length has been reached + 4. File resources are properly closed even if the operation is cancelled + + The implementation carefully handles both known-size and unknown-size payloads, + as well as constrained and unconstrained content lengths. + + """ + loop = asyncio.get_running_loop() + total_written_len = 0 + remaining_content_len = content_length + + # Get initial data and available length + available_len, chunk = await loop.run_in_executor( + None, self._read_and_available_len, remaining_content_len + ) + # Process data chunks until done + while chunk: + chunk_len = len(chunk) + + # Write data with or without length constraint + if remaining_content_len is None: + await writer.write(chunk) + else: + await writer.write(chunk[:remaining_content_len]) + remaining_content_len -= chunk_len + + total_written_len += chunk_len + + # Check if we're done writing + if self._should_stop_writing( + available_len, total_written_len, remaining_content_len + ): + return + + # Read next chunk + chunk = await loop.run_in_executor( + None, + self._read, + ( + min(DEFAULT_CHUNK_SIZE, remaining_content_len) + if remaining_content_len is not None + else DEFAULT_CHUNK_SIZE + ), + ) + + def _should_stop_writing( + self, + available_len: int | None, + total_written_len: int, + remaining_content_len: int | None, + ) -> bool: + """ + Determine if we should stop writing data. + + Args: + available_len: Known size of the payload if available (None if unknown) + total_written_len: Number of bytes already written + remaining_content_len: Remaining bytes to be written for content-length limited responses + + Returns: + True if we should stop writing data, based on either: + - Having written all available data (when size is known) + - Having written all requested content (when content-length is specified) + + """ + return (available_len is not None and total_written_len >= available_len) or ( + remaining_content_len is not None and remaining_content_len <= 0 + ) + + def _close(self) -> None: + """ + Async safe synchronous close operations for backwards compatibility. + + This method exists only for backwards + compatibility. Use the async close() method instead. + + WARNING: This method MUST be called from within an event loop. + Calling it outside an event loop will raise RuntimeError. + """ + # Skip if already consumed + if self._consumed: + return + self._consumed = True # Mark as consumed to prevent further writes + # Schedule file closing without awaiting to prevent cancellation issues + loop = asyncio.get_running_loop() + close_future = loop.run_in_executor(None, self._value.close) + # Hold a strong reference to the future to prevent it from being + # garbage collected before it completes. + _CLOSE_FUTURES.add(close_future) + close_future.add_done_callback(_CLOSE_FUTURES.remove) + + async def close(self) -> None: + """ + Close the payload if it holds any resources. + + IMPORTANT: This method must not await anything that might not finish + immediately, as it may be called during cleanup/cancellation. Schedule + any long-running operations without awaiting them. + """ + self._close() + + def decode(self, encoding: str = "utf-8", errors: str = "strict") -> str: + """ + Return string representation of the value. + + WARNING: This method does blocking I/O and should not be called in the event loop. + """ + return self._read_all().decode(encoding, errors) + + def _read_all(self) -> bytes: + """Read the entire file-like object and return its content as bytes.""" + self._set_or_restore_start_position() + # Use readlines() to ensure we get all content + return b"".join(self._value.readlines()) + + async def as_bytes(self, encoding: str = "utf-8", errors: str = "strict") -> bytes: + """ + Return bytes representation of the value. + + This method reads the entire file content and returns it as bytes. + It is equivalent to reading the file-like object directly. + The file reading is performed in an executor to avoid blocking the event loop. + """ + loop = asyncio.get_running_loop() + return await loop.run_in_executor(None, self._read_all) + + +class TextIOPayload(IOBasePayload): + _value: io.TextIOBase + # _autoclose = False (inherited) - Has text file handle that needs explicit closing + + def __init__( + self, + value: TextIO, + *args: Any, + encoding: str | None = None, + content_type: str | None = None, + **kwargs: Any, + ) -> None: + + if encoding is None: + if content_type is None: + encoding = "utf-8" + content_type = "text/plain; charset=utf-8" + else: + mimetype = parse_mimetype(content_type) + encoding = mimetype.parameters.get("charset", "utf-8") + else: + if content_type is None: + content_type = "text/plain; charset=%s" % encoding + + super().__init__( + value, + content_type=content_type, + encoding=encoding, + *args, + **kwargs, + ) + + def _read_and_available_len( + self, remaining_content_len: int | None + ) -> tuple[int | None, bytes]: + """ + Read the text file-like object and return both its total size and the first chunk. + + Args: + remaining_content_len: Optional limit on how many bytes to read in this operation. + If None, DEFAULT_CHUNK_SIZE will be used as the default chunk size. + + Returns: + A tuple containing: + - The total size of the remaining unread content (None if size cannot be determined) + - The first chunk of bytes read from the file object, encoded using the payload's encoding + + This method is optimized to perform both size calculation and initial read + in a single operation, which is executed in a single executor job to minimize + context switches and file operations when streaming content. + + Note: + TextIOPayload handles encoding of the text content before writing it + to the stream. If no encoding is specified, UTF-8 is used as the default. + + """ + self._set_or_restore_start_position() + size = self.size + chunk = self._value.read( + min( + DEFAULT_CHUNK_SIZE, + size or DEFAULT_CHUNK_SIZE, + remaining_content_len or DEFAULT_CHUNK_SIZE, + ) + ) + return size, chunk.encode(self._encoding) if self._encoding else chunk.encode() + + def _read(self, remaining_content_len: int | None) -> bytes: + """ + Read a chunk of data from the text file-like object. + + Args: + remaining_content_len: Optional maximum number of bytes to read. + If None, DEFAULT_CHUNK_SIZE will be used as the default chunk size. + + Returns: + A chunk of bytes read from the file object and encoded using the payload's + encoding. The data is automatically converted from text to bytes. + + This method is used for subsequent reads during streaming after + the initial _read_and_available_len call has been made. It properly + handles text encoding, converting the text content to bytes using + the specified encoding (or UTF-8 if none was provided). + + """ + chunk = self._value.read(remaining_content_len or DEFAULT_CHUNK_SIZE) + return chunk.encode(self._encoding) if self._encoding else chunk.encode() + + def decode(self, encoding: str = "utf-8", errors: str = "strict") -> str: + """ + Return string representation of the value. + + WARNING: This method does blocking I/O and should not be called in the event loop. + """ + self._set_or_restore_start_position() + return self._value.read() + + async def as_bytes(self, encoding: str = "utf-8", errors: str = "strict") -> bytes: + """ + Return bytes representation of the value. + + This method reads the entire text file content and returns it as bytes. + It encodes the text content using the specified encoding. + The file reading is performed in an executor to avoid blocking the event loop. + """ + loop = asyncio.get_running_loop() + + # Use instance encoding if available, otherwise use parameter + actual_encoding = self._encoding or encoding + + def _read_and_encode() -> bytes: + self._set_or_restore_start_position() + # TextIO read() always returns the full content + return self._value.read().encode(actual_encoding, errors) + + return await loop.run_in_executor(None, _read_and_encode) + + +class BytesIOPayload(IOBasePayload): + _value: io.BytesIO + _size: int # Always initialized in __init__ + _autoclose = True # BytesIO is in-memory, safe to auto-close + + def __init__(self, value: io.BytesIO, *args: Any, **kwargs: Any) -> None: + super().__init__(value, *args, **kwargs) + # Calculate size once during initialization + self._size = len(self._value.getbuffer()) - self._value.tell() + + @property + def size(self) -> int: + """Size of the payload in bytes. + + Returns the number of bytes in the BytesIO buffer that will be transmitted. + This is calculated once during initialization for efficiency. + """ + return self._size + + def decode(self, encoding: str = "utf-8", errors: str = "strict") -> str: + self._set_or_restore_start_position() + return self._value.read().decode(encoding, errors) + + async def write(self, writer: AbstractStreamWriter) -> None: + return await self.write_with_length(writer, None) + + async def write_with_length( + self, writer: AbstractStreamWriter, content_length: int | None + ) -> None: + """ + Write BytesIO payload with a specific content length constraint. + + Args: + writer: An AbstractStreamWriter instance that handles the actual writing + content_length: Maximum number of bytes to write (None for unlimited) + + This implementation is specifically optimized for BytesIO objects: + + 1. Reads content in chunks to maintain memory efficiency + 2. Yields control back to the event loop periodically to prevent blocking + when dealing with large BytesIO objects + 3. Respects content_length constraints when specified + 4. Properly cleans up by closing the BytesIO object when done or on error + + The periodic yielding to the event loop is important for maintaining + responsiveness when processing large in-memory buffers. + + """ + self._set_or_restore_start_position() + loop_count = 0 + remaining_bytes = content_length + while chunk := self._value.read(DEFAULT_CHUNK_SIZE): + if loop_count > 0: + # Avoid blocking the event loop + # if they pass a large BytesIO object + # and we are not in the first iteration + # of the loop + await asyncio.sleep(0) + if remaining_bytes is None: + await writer.write(chunk) + else: + await writer.write(chunk[:remaining_bytes]) + remaining_bytes -= len(chunk) + if remaining_bytes <= 0: + return + loop_count += 1 + + async def as_bytes(self, encoding: str = "utf-8", errors: str = "strict") -> bytes: + """ + Return bytes representation of the value. + + This method reads the entire BytesIO content and returns it as bytes. + It is equivalent to accessing the _value attribute directly. + """ + self._set_or_restore_start_position() + return self._value.read() + + async def close(self) -> None: + """ + Close the BytesIO payload. + + This does nothing since BytesIO is in-memory and does not require explicit closing. + """ + + +class BufferedReaderPayload(IOBasePayload): + _value: io.BufferedIOBase + # _autoclose = False (inherited) - Has buffered file handle that needs explicit closing + + def decode(self, encoding: str = "utf-8", errors: str = "strict") -> str: + self._set_or_restore_start_position() + return self._value.read().decode(encoding, errors) + + +class JsonPayload(BytesPayload): + def __init__( + self, + value: Any, + encoding: str = "utf-8", + content_type: str = "application/json", + dumps: JSONEncoder = json.dumps, + *args: Any, + **kwargs: Any, + ) -> None: + + super().__init__( + dumps(value).encode(encoding), + content_type=content_type, + encoding=encoding, + *args, + **kwargs, + ) + + +class JsonBytesPayload(BytesPayload): + """JSON payload for encoders that return bytes directly. + + Use this when your JSON encoder (like orjson) returns bytes + instead of str, avoiding the encode/decode overhead. + """ + + def __init__( + self, + value: Any, + dumps: JSONBytesEncoder, + content_type: str = "application/json", + *args: Any, + **kwargs: Any, + ) -> None: + super().__init__( + dumps(value), + content_type=content_type, + *args, + **kwargs, + ) + + +if TYPE_CHECKING: + from collections.abc import AsyncIterable, AsyncIterator + + _AsyncIterator = AsyncIterator[bytes] + _AsyncIterable = AsyncIterable[bytes] +else: + from collections.abc import AsyncIterable, AsyncIterator + + _AsyncIterator = AsyncIterator + _AsyncIterable = AsyncIterable + + +class AsyncIterablePayload(Payload): + + _iter: _AsyncIterator | None = None + _value: _AsyncIterable + _cached_chunks: list[bytes] | None = None + # _consumed stays False to allow reuse with cached content + _autoclose = True # Iterator doesn't need explicit closing + + def __init__(self, value: _AsyncIterable, *args: Any, **kwargs: Any) -> None: + if not isinstance(value, AsyncIterable): + raise TypeError( + "value argument must support " + "collections.abc.AsyncIterable interface, " + f"got {type(value)!r}" + ) + + if "content_type" not in kwargs: + kwargs["content_type"] = "application/octet-stream" + + super().__init__(value, *args, **kwargs) + + self._iter = value.__aiter__() + + async def write(self, writer: AbstractStreamWriter) -> None: + """ + Write the entire async iterable payload to the writer stream. + + Args: + writer: An AbstractStreamWriter instance that handles the actual writing + + This method iterates through the async iterable and writes each chunk + to the writer without any length constraint. + + Note: + For new implementations that need length control, use write_with_length() directly. + This method is maintained for backwards compatibility with existing code. + + """ + await self.write_with_length(writer, None) + + async def write_with_length( + self, writer: AbstractStreamWriter, content_length: int | None + ) -> None: + """ + Write async iterable payload with a specific content length constraint. + + Args: + writer: An AbstractStreamWriter instance that handles the actual writing + content_length: Maximum number of bytes to write (None for unlimited) + + This implementation handles streaming of async iterable content with length constraints: + + 1. If cached chunks are available, writes from them + 2. Otherwise iterates through the async iterable one chunk at a time + 3. Respects content_length constraints when specified + 4. Does NOT generate cache - that's done by as_bytes() + + """ + # If we have cached chunks, use them + if self._cached_chunks is not None: + remaining_bytes = content_length + for chunk in self._cached_chunks: + if remaining_bytes is None: + await writer.write(chunk) + elif remaining_bytes > 0: + await writer.write(chunk[:remaining_bytes]) + remaining_bytes -= len(chunk) + else: + break + return + + # If iterator is exhausted and we don't have cached chunks, nothing to write + if self._iter is None: + return + + # Stream from the iterator + remaining_bytes = content_length + + try: + while True: + chunk = await anext(self._iter) + if remaining_bytes is None: + await writer.write(chunk) + # If we have a content length limit + elif remaining_bytes > 0: + await writer.write(chunk[:remaining_bytes]) + remaining_bytes -= len(chunk) + # We still want to exhaust the iterator even + # if we have reached the content length limit + # since the file handle may not get closed by + # the iterator if we don't do this + except StopAsyncIteration: + # Iterator is exhausted + self._iter = None + self._consumed = True # Mark as consumed when streamed without caching + + def decode(self, encoding: str = "utf-8", errors: str = "strict") -> str: + """Decode the payload content as a string if cached chunks are available.""" + if self._cached_chunks is not None: + return b"".join(self._cached_chunks).decode(encoding, errors) + raise TypeError("Unable to decode - content not cached. Call as_bytes() first.") + + async def as_bytes(self, encoding: str = "utf-8", errors: str = "strict") -> bytes: + """ + Return bytes representation of the value. + + This method reads the entire async iterable content and returns it as bytes. + It generates and caches the chunks for future reuse. + """ + # If we have cached chunks, return them joined + if self._cached_chunks is not None: + return b"".join(self._cached_chunks) + + # If iterator is exhausted and no cache, return empty + if self._iter is None: + return b"" + + # Read all chunks and cache them + chunks: list[bytes] = [] + async for chunk in self._iter: + chunks.append(chunk) + + # Iterator is exhausted, cache the chunks + self._iter = None + self._cached_chunks = chunks + # Keep _consumed as False to allow reuse with cached chunks + + return b"".join(chunks) + + +class StreamReaderPayload(AsyncIterablePayload): + def __init__(self, value: StreamReader, *args: Any, **kwargs: Any) -> None: + super().__init__(value.iter_any(), *args, **kwargs) + + +PAYLOAD_REGISTRY = PayloadRegistry() +PAYLOAD_REGISTRY.register(BytesPayload, (bytes, bytearray, memoryview)) +PAYLOAD_REGISTRY.register(StringPayload, str) +PAYLOAD_REGISTRY.register(StringIOPayload, io.StringIO) +PAYLOAD_REGISTRY.register(TextIOPayload, io.TextIOBase) +PAYLOAD_REGISTRY.register(BytesIOPayload, io.BytesIO) +PAYLOAD_REGISTRY.register(BufferedReaderPayload, (io.BufferedReader, io.BufferedRandom)) +PAYLOAD_REGISTRY.register(IOBasePayload, io.IOBase) +PAYLOAD_REGISTRY.register(StreamReaderPayload, StreamReader) +# try_last for giving a chance to more specialized async interables like +# multipart.BodyPartReaderPayload override the default +PAYLOAD_REGISTRY.register(AsyncIterablePayload, AsyncIterable, order=Order.try_last) diff --git a/py_modules/aiohttp/payload_streamer.py b/py_modules/aiohttp/payload_streamer.py new file mode 100644 index 0000000..67afedf --- /dev/null +++ b/py_modules/aiohttp/payload_streamer.py @@ -0,0 +1,79 @@ +""" +Payload implementation for coroutines as data provider. + +As a simple case, you can upload data from file:: + + @aiohttp.streamer + async def file_sender(writer, file_name=None): + with open(file_name, 'rb') as f: + chunk = f.read(2**16) + while chunk: + await writer.write(chunk) + + chunk = f.read(2**16) + +Then you can use `file_sender` like this: + + async with session.post('http://httpbin.org/post', + data=file_sender(file_name='huge_file')) as resp: + print(await resp.text()) + +..note:: Coroutine must accept `writer` as first argument + +""" + +import types +import warnings +from collections.abc import Awaitable, Callable +from typing import Any + +from .abc import AbstractStreamWriter +from .payload import Payload, payload_type + +__all__ = ("streamer",) + + +class _stream_wrapper: + def __init__( + self, + coro: Callable[..., Awaitable[None]], + args: tuple[Any, ...], + kwargs: dict[str, Any], + ) -> None: + self.coro = types.coroutine(coro) + self.args = args + self.kwargs = kwargs + + async def __call__(self, writer: AbstractStreamWriter) -> None: + await self.coro(writer, *self.args, **self.kwargs) + + +class streamer: + def __init__(self, coro: Callable[..., Awaitable[None]]) -> None: + warnings.warn( + "@streamer is deprecated, use async generators instead", + DeprecationWarning, + stacklevel=2, + ) + self.coro = coro + + def __call__(self, *args: Any, **kwargs: Any) -> _stream_wrapper: + return _stream_wrapper(self.coro, args, kwargs) + + +@payload_type(_stream_wrapper) +class StreamWrapperPayload(Payload): + async def write(self, writer: AbstractStreamWriter) -> None: + await self._value(writer) + + def decode(self, encoding: str = "utf-8", errors: str = "strict") -> str: + raise TypeError("Unable to decode.") + + +@payload_type(streamer) +class StreamPayload(StreamWrapperPayload): + def __init__(self, value: Any, *args: Any, **kwargs: Any) -> None: + super().__init__(value(), *args, **kwargs) + + async def write(self, writer: AbstractStreamWriter) -> None: + await self._value(writer) diff --git a/py_modules/aiohttp/py.typed b/py_modules/aiohttp/py.typed new file mode 100644 index 0000000..f5642f7 --- /dev/null +++ b/py_modules/aiohttp/py.typed @@ -0,0 +1 @@ +Marker diff --git a/py_modules/aiohttp/pytest_plugin.py b/py_modules/aiohttp/pytest_plugin.py new file mode 100644 index 0000000..a2c7bdc --- /dev/null +++ b/py_modules/aiohttp/pytest_plugin.py @@ -0,0 +1,439 @@ +import asyncio +import contextlib +import inspect +import warnings +from collections.abc import Awaitable, Callable, Iterator +from typing import Any, Protocol, overload + +import pytest + +from .test_utils import ( + BaseTestServer, + RawTestServer, + TestClient, + TestServer, + loop_context, + setup_test_loop, + teardown_test_loop, + unused_port as _unused_port, +) +from .web import Application, BaseRequest, Request +from .web_protocol import _RequestHandler + +try: + import uvloop +except ImportError: # pragma: no cover + uvloop = None # type: ignore[assignment] + + +class AiohttpClient(Protocol): + @overload + async def __call__( + self, + __param: Application, + *, + server_kwargs: dict[str, Any] | None = None, + **kwargs: Any, + ) -> TestClient[Request, Application]: ... + @overload + async def __call__( + self, + __param: BaseTestServer, + *, + server_kwargs: dict[str, Any] | None = None, + **kwargs: Any, + ) -> TestClient[BaseRequest, None]: ... + + +class AiohttpServer(Protocol): + def __call__( + self, app: Application, *, port: int | None = None, **kwargs: Any + ) -> Awaitable[TestServer]: ... + + +class AiohttpRawServer(Protocol): + def __call__( + self, handler: _RequestHandler, *, port: int | None = None, **kwargs: Any + ) -> Awaitable[RawTestServer]: ... + + +def pytest_addoption(parser): # type: ignore[no-untyped-def] + parser.addoption( + "--aiohttp-fast", + action="store_true", + default=False, + help="run tests faster by disabling extra checks", + ) + parser.addoption( + "--aiohttp-loop", + action="store", + default="pyloop", + help="run tests with specific loop: pyloop, uvloop or all", + ) + parser.addoption( + "--aiohttp-enable-loop-debug", + action="store_true", + default=False, + help="enable event loop debug mode", + ) + + +def pytest_fixture_setup(fixturedef): # type: ignore[no-untyped-def] + """Set up pytest fixture. + + Allow fixtures to be coroutines. Run coroutine fixtures in an event loop. + """ + func = fixturedef.func + + if inspect.isasyncgenfunction(func): + # async generator fixture + is_async_gen = True + elif inspect.iscoroutinefunction(func): + # regular async fixture + is_async_gen = False + else: + # not an async fixture, nothing to do + return + + strip_request = False + if "request" not in fixturedef.argnames: + fixturedef.argnames += ("request",) + strip_request = True + + def wrapper(*args, **kwargs): # type: ignore[no-untyped-def] + request = kwargs["request"] + if strip_request: + del kwargs["request"] + + # if neither the fixture nor the test use the 'loop' fixture, + # 'getfixturevalue' will fail because the test is not parameterized + # (this can be removed someday if 'loop' is no longer parameterized) + if "loop" not in request.fixturenames: + raise Exception( + "Asynchronous fixtures must depend on the 'loop' fixture or " + "be used in tests depending from it." + ) + + _loop = request.getfixturevalue("loop") + + if is_async_gen: + # for async generators, we need to advance the generator once, + # then advance it again in a finalizer + gen = func(*args, **kwargs) + + def finalizer(): # type: ignore[no-untyped-def] + try: + return _loop.run_until_complete(gen.__anext__()) + except StopAsyncIteration: + pass + + request.addfinalizer(finalizer) + return _loop.run_until_complete(gen.__anext__()) + else: + return _loop.run_until_complete(func(*args, **kwargs)) + + fixturedef.func = wrapper + + +@pytest.fixture +def fast(request): # type: ignore[no-untyped-def] + """--fast config option""" + return request.config.getoption("--aiohttp-fast") + + +@pytest.fixture +def loop_debug(request): # type: ignore[no-untyped-def] + """--enable-loop-debug config option""" + return request.config.getoption("--aiohttp-enable-loop-debug") + + +@contextlib.contextmanager +def _runtime_warning_context(): # type: ignore[no-untyped-def] + """Context manager which checks for RuntimeWarnings. + + This exists specifically to + avoid "coroutine 'X' was never awaited" warnings being missed. + + If RuntimeWarnings occur in the context a RuntimeError is raised. + """ + with warnings.catch_warnings(record=True) as _warnings: + yield + rw = [ + f"{w.filename}:{w.lineno}:{w.message}" + for w in _warnings + if w.category == RuntimeWarning + ] + if rw: + raise RuntimeError( + "{} Runtime Warning{},\n{}".format( + len(rw), "" if len(rw) == 1 else "s", "\n".join(rw) + ) + ) + + +@contextlib.contextmanager +def _passthrough_loop_context(loop, fast=False): # type: ignore[no-untyped-def] + """Passthrough loop context. + + Sets up and tears down a loop unless one is passed in via the loop + argument when it's passed straight through. + """ + if loop: + # loop already exists, pass it straight through + yield loop + else: + # this shadows loop_context's standard behavior + loop = setup_test_loop() + yield loop + teardown_test_loop(loop, fast=fast) + + +def pytest_pycollect_makeitem(collector, name, obj): # type: ignore[no-untyped-def] + """Fix pytest collecting for coroutines.""" + if collector.funcnamefilter(name) and inspect.iscoroutinefunction(obj): + return list(collector._genfunctions(name, obj)) + + +def pytest_pyfunc_call(pyfuncitem): # type: ignore[no-untyped-def] + """Run coroutines in an event loop instead of a normal function call.""" + fast = pyfuncitem.config.getoption("--aiohttp-fast") + if inspect.iscoroutinefunction(pyfuncitem.function): + warnings.warn( + "aiohttp.pytest_plugin will be removed in v4. Please install pytest-aiohttp.", + DeprecationWarning, + ) + existing_loop = ( + pyfuncitem.funcargs.get("proactor_loop") + or pyfuncitem.funcargs.get("selector_loop") + or pyfuncitem.funcargs.get("uvloop_loop") + or pyfuncitem.funcargs.get("loop", None) + ) + + with _runtime_warning_context(): + with _passthrough_loop_context(existing_loop, fast=fast) as _loop: + testargs = { + arg: pyfuncitem.funcargs[arg] + for arg in pyfuncitem._fixtureinfo.argnames + } + _loop.run_until_complete(pyfuncitem.obj(**testargs)) + + return True + + +def pytest_generate_tests(metafunc): # type: ignore[no-untyped-def] + if "loop_factory" not in metafunc.fixturenames: + return + + loops = metafunc.config.option.aiohttp_loop + avail_factories: dict[str, Callable[[], asyncio.AbstractEventLoop]] + avail_factories = {"pyloop": asyncio.new_event_loop} + + if uvloop is not None: # pragma: no cover + avail_factories["uvloop"] = uvloop.new_event_loop + + if loops == "all": + loops = "pyloop,uvloop?" + + factories = {} # type: ignore[var-annotated] + for name in loops.split(","): + required = not name.endswith("?") + name = name.strip(" ?") + if name not in avail_factories: # pragma: no cover + if required: + raise ValueError( + "Unknown loop '%s', available loops: %s" + % (name, list(factories.keys())) + ) + else: + continue + factories[name] = avail_factories[name] + metafunc.parametrize( + "loop_factory", list(factories.values()), ids=list(factories.keys()) + ) + + +@pytest.fixture +def loop( + loop_factory: Callable[[], asyncio.AbstractEventLoop], + fast: bool, + loop_debug: bool, +) -> Iterator[asyncio.AbstractEventLoop]: + """Return an instance of the event loop.""" + with loop_context(loop_factory, fast=fast) as _loop: + if loop_debug: + _loop.set_debug(True) # pragma: no cover + asyncio.set_event_loop(_loop) + yield _loop + + +@pytest.fixture +def proactor_loop() -> Iterator[asyncio.AbstractEventLoop]: + factory = asyncio.ProactorEventLoop # type: ignore[attr-defined] + + with loop_context(factory) as _loop: + asyncio.set_event_loop(_loop) + yield _loop + + +@pytest.fixture +def unused_port(aiohttp_unused_port: Callable[[], int]) -> Callable[[], int]: + warnings.warn( + "Deprecated, use aiohttp_unused_port fixture instead", + DeprecationWarning, + stacklevel=2, + ) + return aiohttp_unused_port + + +@pytest.fixture +def aiohttp_unused_port() -> Callable[[], int]: + """Return a port that is unused on the current host.""" + return _unused_port + + +@pytest.fixture +def aiohttp_server(loop: asyncio.AbstractEventLoop) -> Iterator[AiohttpServer]: + """Factory to create a TestServer instance, given an app. + + aiohttp_server(app, **kwargs) + """ + servers = [] + + async def go( + app: Application, + *, + host: str = "127.0.0.1", + port: int | None = None, + **kwargs: Any, + ) -> TestServer: + server = TestServer(app, host=host, port=port) + await server.start_server(loop=loop, **kwargs) + servers.append(server) + return server + + yield go + + async def finalize() -> None: + while servers: + await servers.pop().close() + + loop.run_until_complete(finalize()) + + +@pytest.fixture +def test_server(aiohttp_server): # type: ignore[no-untyped-def] # pragma: no cover + warnings.warn( + "Deprecated, use aiohttp_server fixture instead", + DeprecationWarning, + stacklevel=2, + ) + return aiohttp_server + + +@pytest.fixture +def aiohttp_raw_server(loop: asyncio.AbstractEventLoop) -> Iterator[AiohttpRawServer]: + """Factory to create a RawTestServer instance, given a web handler. + + aiohttp_raw_server(handler, **kwargs) + """ + servers = [] + + async def go( + handler: _RequestHandler, *, port: int | None = None, **kwargs: Any + ) -> RawTestServer: + server = RawTestServer(handler, port=port) + await server.start_server(loop=loop, **kwargs) + servers.append(server) + return server + + yield go + + async def finalize() -> None: + while servers: + await servers.pop().close() + + loop.run_until_complete(finalize()) + + +@pytest.fixture +def raw_test_server( # type: ignore[no-untyped-def] # pragma: no cover + aiohttp_raw_server, +): + warnings.warn( + "Deprecated, use aiohttp_raw_server fixture instead", + DeprecationWarning, + stacklevel=2, + ) + return aiohttp_raw_server + + +@pytest.fixture +def aiohttp_client(loop: asyncio.AbstractEventLoop) -> Iterator[AiohttpClient]: + """Factory to create a TestClient instance. + + aiohttp_client(app, **kwargs) + aiohttp_client(server, **kwargs) + aiohttp_client(raw_server, **kwargs) + """ + clients = [] + + @overload + async def go( + __param: Application, + *, + server_kwargs: dict[str, Any] | None = None, + **kwargs: Any, + ) -> TestClient[Request, Application]: ... + + @overload + async def go( + __param: BaseTestServer, + *, + server_kwargs: dict[str, Any] | None = None, + **kwargs: Any, + ) -> TestClient[BaseRequest, None]: ... + + async def go( + __param: Application | BaseTestServer, + *args: Any, + server_kwargs: dict[str, Any] | None = None, + **kwargs: Any, + ) -> TestClient[Any, Any]: + if isinstance(__param, Callable) and not isinstance( # type: ignore[arg-type] + __param, (Application, BaseTestServer) + ): + __param = __param(loop, *args, **kwargs) + kwargs = {} + else: + assert not args, "args should be empty" + + if isinstance(__param, Application): + server_kwargs = server_kwargs or {} + server = TestServer(__param, loop=loop, **server_kwargs) + client = TestClient(server, loop=loop, **kwargs) + elif isinstance(__param, BaseTestServer): + client = TestClient(__param, loop=loop, **kwargs) + else: + raise ValueError("Unknown argument type: %r" % type(__param)) + + await client.start_server() + clients.append(client) + return client + + yield go + + async def finalize() -> None: + while clients: + await clients.pop().close() + + loop.run_until_complete(finalize()) + + +@pytest.fixture +def test_client(aiohttp_client): # type: ignore[no-untyped-def] # pragma: no cover + warnings.warn( + "Deprecated, use aiohttp_client fixture instead", + DeprecationWarning, + stacklevel=2, + ) + return aiohttp_client diff --git a/py_modules/aiohttp/resolver.py b/py_modules/aiohttp/resolver.py new file mode 100644 index 0000000..28a5cbb --- /dev/null +++ b/py_modules/aiohttp/resolver.py @@ -0,0 +1,301 @@ +import asyncio +import socket +import sys +import weakref +from typing import Any, Final, Optional + +from .abc import AbstractResolver, ResolveResult + +__all__ = ("ThreadedResolver", "AsyncResolver", "DefaultResolver") + + +try: + import aiodns + + aiodns_default = hasattr(aiodns.DNSResolver, "getaddrinfo") +except ImportError: # pragma: no cover + aiodns = None # type: ignore[assignment] + aiodns_default = False + + +_NUMERIC_SOCKET_FLAGS = socket.AI_NUMERICHOST | socket.AI_NUMERICSERV +_NAME_SOCKET_FLAGS = socket.NI_NUMERICHOST | socket.NI_NUMERICSERV +_AI_ADDRCONFIG = socket.AI_ADDRCONFIG +if hasattr(socket, "AI_MASK"): + _AI_ADDRCONFIG &= socket.AI_MASK +_IS_WINDOWS = sys.platform == "win32" + + +def _is_windows_localhost(host: str) -> bool: + return _IS_WINDOWS and host.rstrip(".").casefold() == "localhost" + + +class ThreadedResolver(AbstractResolver): + """Threaded resolver. + + Uses an Executor for synchronous getaddrinfo() calls. + concurrent.futures.ThreadPoolExecutor is used by default. + """ + + def __init__(self, loop: asyncio.AbstractEventLoop | None = None) -> None: + self._loop = loop or asyncio.get_running_loop() + + async def resolve( + self, host: str, port: int = 0, family: socket.AddressFamily = socket.AF_INET + ) -> list[ResolveResult]: + try: + infos = await self._loop.getaddrinfo( + host, + port, + type=socket.SOCK_STREAM, + family=family, + flags=_AI_ADDRCONFIG, + ) + except socket.gaierror: + if not _is_windows_localhost(host): + raise + infos = await self._loop.getaddrinfo( + host, + port, + type=socket.SOCK_STREAM, + family=family, + flags=0, + ) + + hosts: list[ResolveResult] = [] + for family, _, proto, _, address in infos: + if family == socket.AF_INET6: + if len(address) < 3: + # IPv6 is not supported by Python build, + # or IPv6 is not enabled in the host + continue + if address[3]: + # This is essential for link-local IPv6 addresses. + # LL IPv6 is a VERY rare case. Strictly speaking, we should use + # getnameinfo() unconditionally, but performance makes sense. + resolved_host, _port = await self._loop.getnameinfo( + address, _NAME_SOCKET_FLAGS + ) + port = int(_port) + else: + resolved_host, port = address[:2] + else: # IPv4 + assert family == socket.AF_INET + resolved_host, port = address # type: ignore[misc] + hosts.append( + ResolveResult( + hostname=host, + host=resolved_host, + port=port, + family=family, + proto=proto, + flags=_NUMERIC_SOCKET_FLAGS, + ) + ) + + return hosts + + async def close(self) -> None: + pass + + +class AsyncResolver(AbstractResolver): + """Use the `aiodns` package to make asynchronous DNS lookups""" + + def __init__( + self, + loop: asyncio.AbstractEventLoop | None = None, + *args: Any, + **kwargs: Any, + ) -> None: + if aiodns is None: + raise RuntimeError("Resolver requires aiodns library") + + self._loop = loop or asyncio.get_running_loop() + self._manager: _DNSResolverManager | None = None + # If custom args are provided, create a dedicated resolver instance + # This means each AsyncResolver with custom args gets its own + # aiodns.DNSResolver instance + if args or kwargs: + self._resolver = aiodns.DNSResolver(*args, **kwargs) + return + # Use the shared resolver from the manager for default arguments + self._manager = _DNSResolverManager() + self._resolver = self._manager.get_resolver(self, self._loop) + + if not hasattr(self._resolver, "gethostbyname"): + # aiodns 1.1 is not available, fallback to DNSResolver.query + self.resolve = self._resolve_with_query # type: ignore + + async def resolve( + self, host: str, port: int = 0, family: socket.AddressFamily = socket.AF_INET + ) -> list[ResolveResult]: + try: + try: + resp = await self._resolver.getaddrinfo( + host, + port=port, + type=socket.SOCK_STREAM, + family=family, + flags=_AI_ADDRCONFIG, + ) + except aiodns.error.DNSError: + if not _is_windows_localhost(host): + raise + resp = await self._resolver.getaddrinfo( + host, + port=port, + type=socket.SOCK_STREAM, + family=family, + flags=0, + ) + except aiodns.error.DNSError as exc: + msg = exc.args[1] if len(exc.args) >= 1 else "DNS lookup failed" + raise OSError(None, msg) from exc + hosts: list[ResolveResult] = [] + for node in resp.nodes: + address: tuple[bytes, int] | tuple[bytes, int, int, int] = node.addr + if node.family == socket.AF_INET6: + if len(address) > 3 and address[3]: + # This is essential for link-local IPv6 addresses. + # LL IPv6 is a VERY rare case. Strictly speaking, we should use + # getnameinfo() unconditionally, but performance makes sense. + result = await self._resolver.getnameinfo( + (address[0].decode("ascii"), *address[1:]), + _NAME_SOCKET_FLAGS, + ) + resolved_host = result.node + else: + resolved_host = address[0].decode("ascii") + port = address[1] + else: # IPv4 + assert node.family == socket.AF_INET + resolved_host = address[0].decode("ascii") + port = address[1] + hosts.append( + ResolveResult( + hostname=host, + host=resolved_host, + port=port, + family=node.family, + proto=0, + flags=_NUMERIC_SOCKET_FLAGS, + ) + ) + + if not hosts: + raise OSError(None, "DNS lookup failed") + + return hosts + + async def _resolve_with_query( + self, host: str, port: int = 0, family: int = socket.AF_INET + ) -> list[dict[str, Any]]: + qtype: Final = "AAAA" if family == socket.AF_INET6 else "A" + + try: + resp = await self._resolver.query(host, qtype) + except aiodns.error.DNSError as exc: + msg = exc.args[1] if len(exc.args) >= 1 else "DNS lookup failed" + raise OSError(None, msg) from exc + + hosts = [] + for rr in resp: + hosts.append( + { + "hostname": host, + "host": rr.host, + "port": port, + "family": family, + "proto": 0, + "flags": socket.AI_NUMERICHOST, + } + ) + + if not hosts: + raise OSError(None, "DNS lookup failed") + + return hosts + + async def close(self) -> None: + if self._manager: + # Release the resolver from the manager if using the shared resolver + self._manager.release_resolver(self, self._loop) + self._manager = None # Clear reference to manager + self._resolver = None # type: ignore[assignment] # Clear reference to resolver + return + # Otherwise cancel our dedicated resolver + if self._resolver is not None: + self._resolver.cancel() + self._resolver = None # type: ignore[assignment] # Clear reference + + +class _DNSResolverManager: + """Manager for aiodns.DNSResolver objects. + + This class manages shared aiodns.DNSResolver instances + with no custom arguments across different event loops. + """ + + _instance: Optional["_DNSResolverManager"] = None + + def __new__(cls) -> "_DNSResolverManager": + if cls._instance is None: + cls._instance = super().__new__(cls) + cls._instance._init() + return cls._instance + + def _init(self) -> None: + # Use WeakKeyDictionary to allow event loops to be garbage collected + self._loop_data: weakref.WeakKeyDictionary[ + asyncio.AbstractEventLoop, + tuple[aiodns.DNSResolver, weakref.WeakSet[AsyncResolver]], + ] = weakref.WeakKeyDictionary() + + def get_resolver( + self, client: "AsyncResolver", loop: asyncio.AbstractEventLoop + ) -> "aiodns.DNSResolver": + """Get or create the shared aiodns.DNSResolver instance for a specific event loop. + + Args: + client: The AsyncResolver instance requesting the resolver. + This is required to track resolver usage. + loop: The event loop to use for the resolver. + """ + # Create a new resolver and client set for this loop if it doesn't exist + if loop not in self._loop_data: + resolver = aiodns.DNSResolver(loop=loop) + client_set: weakref.WeakSet[AsyncResolver] = weakref.WeakSet() + self._loop_data[loop] = (resolver, client_set) + else: + # Get the existing resolver and client set + resolver, client_set = self._loop_data[loop] + + # Register this client with the loop + client_set.add(client) + return resolver + + def release_resolver( + self, client: "AsyncResolver", loop: asyncio.AbstractEventLoop + ) -> None: + """Release the resolver for an AsyncResolver client when it's closed. + + Args: + client: The AsyncResolver instance to release. + loop: The event loop the resolver was using. + """ + # Remove client from its loop's tracking + current_loop_data = self._loop_data.get(loop) + if current_loop_data is None: + return + resolver, client_set = current_loop_data + client_set.discard(client) + # If no more clients for this loop, cancel and remove its resolver + if not client_set: + if resolver is not None: + resolver.cancel() + del self._loop_data[loop] + + +_DefaultType = type[AsyncResolver | ThreadedResolver] +DefaultResolver: _DefaultType = AsyncResolver if aiodns_default else ThreadedResolver diff --git a/py_modules/aiohttp/streams.py b/py_modules/aiohttp/streams.py new file mode 100644 index 0000000..e1a5b53 --- /dev/null +++ b/py_modules/aiohttp/streams.py @@ -0,0 +1,760 @@ +import asyncio +import collections +import sys +import warnings +from collections.abc import Awaitable, Callable +from typing import Final, Generic, TypeVar + +from .base_protocol import BaseProtocol +from .helpers import ( + _EXC_SENTINEL, + DEFAULT_CHUNK_SIZE, + BaseTimerContext, + TimerNoop, + set_exception, + set_result, +) +from .http_exceptions import LineTooLong +from .log import internal_logger + +__all__ = ( + "EMPTY_PAYLOAD", + "EofStream", + "StreamReader", + "DataQueue", +) + +_T = TypeVar("_T") + + +class EofStream(Exception): + """eof stream indication.""" + + +class AsyncStreamIterator(Generic[_T]): + + __slots__ = ("read_func",) + + def __init__(self, read_func: Callable[[], Awaitable[_T]]) -> None: + self.read_func = read_func + + def __aiter__(self) -> "AsyncStreamIterator[_T]": + return self + + async def __anext__(self) -> _T: + try: + rv = await self.read_func() + except EofStream: + raise StopAsyncIteration + if rv == b"": + raise StopAsyncIteration + return rv + + +class ChunkTupleAsyncStreamIterator: + + __slots__ = ("_stream",) + + def __init__(self, stream: "StreamReader") -> None: + self._stream = stream + + def __aiter__(self) -> "ChunkTupleAsyncStreamIterator": + return self + + async def __anext__(self) -> tuple[bytes, bool]: + rv = await self._stream.readchunk() + if rv == (b"", False): + raise StopAsyncIteration + return rv + + +class StreamReader: + """An enhancement of asyncio.StreamReader. + + Supports asynchronous iteration by line, chunk or as available:: + + async for line in reader: + ... + async for chunk in reader.iter_chunked(1024): + ... + async for slice in reader.iter_any(): + ... + + """ + + __slots__ = ( + "_protocol", + "_low_water", + "_high_water", + "_low_water_chunks", + "_high_water_chunks", + "_loop", + "_size", + "_cursor", + "_http_chunk_splits", + "_buffer", + "_buffer_offset", + "_eof", + "_waiter", + "_eof_waiter", + "_exception", + "_timer", + "_eof_callbacks", + "_eof_counter", + "total_bytes", + "total_compressed_bytes", + ) + + def __init__( + self, + protocol: BaseProtocol, + limit: int, + *, + timer: BaseTimerContext | None = None, + loop: asyncio.AbstractEventLoop | None = None, + ) -> None: + self._protocol = protocol + self._low_water = limit + self._high_water = limit * 2 + if loop is None: + loop = asyncio.get_event_loop() + # Use max(4, ...) because there's always at least 1 chunk split remaining + # (the current position), so we need low_water >= 2 to allow resume. + # limit // 16 gets us a reasonable value of 16k with default 256KiB limit. + self._high_water_chunks = max(4, limit // 16) + self._low_water_chunks = self._high_water_chunks // 2 + self._loop = loop + self._size = 0 + self._cursor = 0 + self._http_chunk_splits: collections.deque[int] | None = None + self._buffer: collections.deque[bytes] = collections.deque() + self._buffer_offset = 0 + self._eof = False + self._waiter: asyncio.Future[None] | None = None + self._eof_waiter: asyncio.Future[None] | None = None + self._exception: BaseException | None = None + self._timer = TimerNoop() if timer is None else timer + self._eof_callbacks: list[Callable[[], None]] = [] + self._eof_counter = 0 + self.total_bytes = 0 + self.total_compressed_bytes: int | None = None + + def __repr__(self) -> str: + info = [self.__class__.__name__] + if self._size: + info.append("%d bytes" % self._size) + if self._eof: + info.append("eof") + if self._low_water != DEFAULT_CHUNK_SIZE: + info.append("low=%d high=%d" % (self._low_water, self._high_water)) + if self._waiter: + info.append("w=%r" % self._waiter) + if self._exception: + info.append("e=%r" % self._exception) + return "<%s>" % " ".join(info) + + def __aiter__(self) -> AsyncStreamIterator[bytes]: + return AsyncStreamIterator(self.readline) + + def iter_chunked(self, n: int) -> AsyncStreamIterator[bytes]: + """Returns an asynchronous iterator that yields chunks of size n.""" + self.set_read_chunk_size(n) + return AsyncStreamIterator(lambda: self.read(n)) + + def iter_any(self) -> AsyncStreamIterator[bytes]: + """Yield all available data as soon as it is received.""" + return AsyncStreamIterator(self.readany) + + def iter_chunks(self) -> ChunkTupleAsyncStreamIterator: + """Yield chunks of data as they are received by the server. + + The yielded objects are tuples + of (bytes, bool) as returned by the StreamReader.readchunk method. + """ + return ChunkTupleAsyncStreamIterator(self) + + def get_read_buffer_limits(self) -> tuple[int, int]: + return (self._low_water, self._high_water) + + def set_read_chunk_size(self, n: int) -> None: + """Raise buffer limits to match the consumer's chunk size.""" + if n > self._low_water: + self._low_water = n + self._high_water = n * 2 + + def exception(self) -> BaseException | None: + return self._exception + + def set_exception( + self, + exc: BaseException, + exc_cause: BaseException = _EXC_SENTINEL, + ) -> None: + self._exception = exc + self._eof_callbacks.clear() + + waiter = self._waiter + if waiter is not None: + self._waiter = None + set_exception(waiter, exc, exc_cause) + + waiter = self._eof_waiter + if waiter is not None: + self._eof_waiter = None + set_exception(waiter, exc, exc_cause) + + def on_eof(self, callback: Callable[[], None]) -> None: + if self._eof: + try: + callback() + except Exception: + internal_logger.exception("Exception in eof callback") + else: + self._eof_callbacks.append(callback) + + def feed_eof(self) -> None: + self._eof = True + + waiter = self._waiter + if waiter is not None: + self._waiter = None + set_result(waiter, None) + + waiter = self._eof_waiter + if waiter is not None: + self._eof_waiter = None + set_result(waiter, None) + + # At EOF the parser is done, there won't be unprocessed data. + self._protocol.resume_reading(resume_parser=False) + + for cb in self._eof_callbacks: + try: + cb() + except Exception: + internal_logger.exception("Exception in eof callback") + + self._eof_callbacks.clear() + + def is_eof(self) -> bool: + """Return True if 'feed_eof' was called.""" + return self._eof + + def at_eof(self) -> bool: + """Return True if the buffer is empty and 'feed_eof' was called.""" + return self._eof and not self._buffer + + async def wait_eof(self) -> None: + if self._eof: + return + + assert self._eof_waiter is None + self._eof_waiter = self._loop.create_future() + try: + await self._eof_waiter + finally: + self._eof_waiter = None + + @property + def total_raw_bytes(self) -> int: + if self.total_compressed_bytes is None: + return self.total_bytes + return self.total_compressed_bytes + + def unread_data(self, data: bytes) -> None: + """rollback reading some data from stream, inserting it to buffer head.""" + warnings.warn( + "unread_data() is deprecated " + "and will be removed in future releases (#3260)", + DeprecationWarning, + stacklevel=2, + ) + if not data: + return + + if self._buffer_offset: + self._buffer[0] = self._buffer[0][self._buffer_offset :] + self._buffer_offset = 0 + self._size += len(data) + self._cursor -= len(data) + self._buffer.appendleft(data) + self._eof_counter = 0 + + # TODO: size is ignored, remove the param later + def feed_data(self, data: bytes, size: int = 0) -> bool: + assert not self._eof, "feed_data after feed_eof" + + if not data: + return False + + data_len = len(data) + self._size += data_len + self._buffer.append(data) + self.total_bytes += data_len + + waiter = self._waiter + if waiter is not None: + self._waiter = None + set_result(waiter, None) + + if self._size > self._high_water: + self._protocol.pause_reading() + return False + + def begin_http_chunk_receiving(self) -> None: + if self._http_chunk_splits is None: + if self.total_bytes: + raise RuntimeError( + "Called begin_http_chunk_receiving when some data was already fed" + ) + self._http_chunk_splits = collections.deque() + + def end_http_chunk_receiving(self) -> None: + if self._http_chunk_splits is None: + raise RuntimeError( + "Called end_chunk_receiving without calling " + "begin_chunk_receiving first" + ) + + # self._http_chunk_splits contains logical byte offsets from start of + # the body transfer. Each offset is the offset of the end of a chunk. + # "Logical" means bytes, accessible for a user. + # If no chunks containing logical data were received, current position + # is difinitely zero. + pos = self._http_chunk_splits[-1] if self._http_chunk_splits else 0 + + if self.total_bytes == pos: + # We should not add empty chunks here. So we check for that. + # Note, when chunked + gzip is used, we can receive a chunk + # of compressed data, but that data may not be enough for gzip FSM + # to yield any uncompressed data. That's why current position may + # not change after receiving a chunk. + return + + self._http_chunk_splits.append(self.total_bytes) + + # If we get too many small chunks before self._high_water is reached, then any + # .read() call becomes computationally expensive, and could block the event loop + # for too long, hence an additional self._high_water_chunks here. + if len(self._http_chunk_splits) > self._high_water_chunks: + self._protocol.pause_reading() + + # wake up readchunk when end of http chunk received + waiter = self._waiter + if waiter is not None: + self._waiter = None + set_result(waiter, None) + + async def _wait(self, func_name: str) -> None: + if not self._protocol.connected: + raise RuntimeError("Connection closed.") + + # StreamReader uses a future to link the protocol feed_data() method + # to a read coroutine. Running two read coroutines at the same time + # would have an unexpected behaviour. It would not possible to know + # which coroutine would get the next data. + if self._waiter is not None: + raise RuntimeError( + "%s() called while another coroutine is " + "already waiting for incoming data" % func_name + ) + + waiter = self._waiter = self._loop.create_future() + try: + with self._timer: + await waiter + finally: + self._waiter = None + + async def readline(self, *, max_line_length: int | None = None) -> bytes: + return await self.readuntil(max_size=max_line_length) + + async def readuntil( + self, separator: bytes = b"\n", *, max_size: int | None = None + ) -> bytes: + seplen = len(separator) + if seplen == 0: + raise ValueError("Separator should be at least one-byte string") + + if self._exception is not None: + raise self._exception + + chunk = b"" + chunk_size = 0 + not_enough = True + max_size = max_size or self._high_water + + while not_enough: + while self._buffer and not_enough: + offset = self._buffer_offset + ichar = self._buffer[0].find(separator, offset) + 1 + # Read from current offset to found separator or to the end. + data = self._read_nowait_chunk( + ichar - offset + seplen - 1 if ichar else -1 + ) + chunk += data + chunk_size += len(data) + if ichar: + not_enough = False + + if chunk_size > max_size: + raise LineTooLong(chunk[:100] + b"...", max_size) + + if self._eof: + break + + if not_enough: + await self._wait("readuntil") + + return chunk + + async def read(self, n: int = -1) -> bytes: + if self._exception is not None: + raise self._exception + + # migration problem; with DataQueue you have to catch + # EofStream exception, so common way is to run payload.read() inside + # infinite loop. what can cause real infinite loop with StreamReader + # lets keep this code one major release. + if __debug__: + if self._eof and not self._buffer: + self._eof_counter = getattr(self, "_eof_counter", 0) + 1 + if self._eof_counter > 5: + internal_logger.warning( + "Multiple access to StreamReader in eof state, " + "might be infinite loop.", + stack_info=True, + ) + + if not n: + return b"" + + if n < 0: + # Reading everything — remove decompression chunk limit. + self.set_read_chunk_size(sys.maxsize) + blocks = [] + while True: + block = await self.readany() + if not block: + break + blocks.append(block) + return b"".join(blocks) + + self.set_read_chunk_size(n) + # TODO: should be `if` instead of `while` + # because waiter maybe triggered on chunk end, + # without feeding any data + while not self._buffer and not self._eof: + await self._wait("read") + + return self._read_nowait(n) + + async def readany(self) -> bytes: + if self._exception is not None: + raise self._exception + + # TODO: should be `if` instead of `while` + # because waiter maybe triggered on chunk end, + # without feeding any data + while not self._buffer and not self._eof: + await self._wait("readany") + + return self._read_nowait(-1) + + async def readchunk(self) -> tuple[bytes, bool]: + """Returns a tuple of (data, end_of_http_chunk). + + When chunked transfer + encoding is used, end_of_http_chunk is a boolean indicating if the end + of the data corresponds to the end of a HTTP chunk , otherwise it is + always False. + """ + while True: + if self._exception is not None: + raise self._exception + + while self._http_chunk_splits: + pos = self._http_chunk_splits.popleft() + if pos == self._cursor: + return (b"", True) + if pos > self._cursor: + return (self._read_nowait(pos - self._cursor), True) + internal_logger.warning( + "Skipping HTTP chunk end due to data " + "consumption beyond chunk boundary" + ) + + if self._buffer: + return (self._read_nowait_chunk(-1), False) + # return (self._read_nowait(-1), False) + + if self._eof: + # Special case for signifying EOF. + # (b'', True) is not a final return value actually. + return (b"", False) + + await self._wait("readchunk") + + async def readexactly(self, n: int) -> bytes: + if self._exception is not None: + raise self._exception + + blocks: list[bytes] = [] + while n > 0: + block = await self.read(n) + if not block: + partial = b"".join(blocks) + raise asyncio.IncompleteReadError(partial, len(partial) + n) + blocks.append(block) + n -= len(block) + + return b"".join(blocks) + + def read_nowait(self, n: int = -1) -> bytes: + # default was changed to be consistent with .read(-1) + # + # I believe the most users don't know about the method and + # they are not affected. + if self._exception is not None: + raise self._exception + + if self._waiter and not self._waiter.done(): + raise RuntimeError( + "Called while some coroutine is waiting for incoming data." + ) + + return self._read_nowait(n) + + def _read_nowait_chunk(self, n: int) -> bytes: + first_buffer = self._buffer[0] + offset = self._buffer_offset + if n != -1 and len(first_buffer) - offset > n: + data = first_buffer[offset : offset + n] + self._buffer_offset += n + + elif offset: + self._buffer.popleft() + data = first_buffer[offset:] + self._buffer_offset = 0 + + else: + data = self._buffer.popleft() + + data_len = len(data) + self._size -= data_len + self._cursor += data_len + + chunk_splits = self._http_chunk_splits + # Prevent memory leak: drop useless chunk splits + while chunk_splits and chunk_splits[0] < self._cursor: + chunk_splits.popleft() + + if self._size < self._low_water and ( + self._http_chunk_splits is None + or len(self._http_chunk_splits) < self._low_water_chunks + ): + self._protocol.resume_reading() + return data + + def _read_nowait(self, n: int) -> bytes: + """Read not more than n bytes, or whole buffer if n == -1""" + self._timer.assert_timeout() + + if n == -1: + # Drain only chunks present now; _read_nowait_chunk() can + # re-entrantly resume_reading() and refill the buffer. + count = len(self._buffer) + if count == 1: + return self._read_nowait_chunk(-1) + return b"".join([self._read_nowait_chunk(-1) for _ in range(count)]) + + chunks: list[bytes] = [] + while self._buffer: + chunk = self._read_nowait_chunk(n) + chunks.append(chunk) + n -= len(chunk) + if n == 0: + break + + return b"".join(chunks) if chunks else b"" + + +class EmptyStreamReader(StreamReader): # lgtm [py/missing-call-to-init] + + __slots__ = ("_read_eof_chunk",) + + def __init__(self) -> None: + self._read_eof_chunk = False + self.total_bytes = 0 + + def __repr__(self) -> str: + return "<%s>" % self.__class__.__name__ + + def exception(self) -> BaseException | None: + return None + + def set_exception( + self, + exc: BaseException, + exc_cause: BaseException = _EXC_SENTINEL, + ) -> None: + pass + + def on_eof(self, callback: Callable[[], None]) -> None: + try: + callback() + except Exception: + internal_logger.exception("Exception in eof callback") + + def feed_eof(self) -> None: + pass + + def is_eof(self) -> bool: + return True + + def at_eof(self) -> bool: + return True + + async def wait_eof(self) -> None: + return + + def feed_data(self, data: bytes, n: int = 0) -> bool: + return False + + def set_read_chunk_size(self, n: int) -> None: + return + + async def readline(self, *, max_line_length: int | None = None) -> bytes: + return b"" + + async def read(self, n: int = -1) -> bytes: + return b"" + + # TODO add async def readuntil + + async def readany(self) -> bytes: + return b"" + + async def readchunk(self) -> tuple[bytes, bool]: + if not self._read_eof_chunk: + self._read_eof_chunk = True + return (b"", False) + + return (b"", True) + + async def readexactly(self, n: int) -> bytes: + raise asyncio.IncompleteReadError(b"", n) + + def read_nowait(self, n: int = -1) -> bytes: + return b"" + + +EMPTY_PAYLOAD: Final[StreamReader] = EmptyStreamReader() + + +class DataQueue(Generic[_T]): + """DataQueue is a general-purpose blocking queue with one reader.""" + + def __init__(self, loop: asyncio.AbstractEventLoop) -> None: + self._loop = loop + self._eof = False + self._waiter: asyncio.Future[None] | None = None + self._exception: BaseException | None = None + self._buffer: collections.deque[tuple[_T, int]] = collections.deque() + + def __len__(self) -> int: + return len(self._buffer) + + def is_eof(self) -> bool: + return self._eof + + def at_eof(self) -> bool: + return self._eof and not self._buffer + + def exception(self) -> BaseException | None: + return self._exception + + def set_exception( + self, + exc: BaseException, + exc_cause: BaseException = _EXC_SENTINEL, + ) -> None: + self._eof = True + self._exception = exc + if (waiter := self._waiter) is not None: + self._waiter = None + set_exception(waiter, exc, exc_cause) + + def feed_data(self, data: _T, size: int = 0) -> None: + self._buffer.append((data, size)) + if (waiter := self._waiter) is not None: + self._waiter = None + set_result(waiter, None) + + def feed_eof(self) -> None: + self._eof = True + if (waiter := self._waiter) is not None: + self._waiter = None + set_result(waiter, None) + + async def read(self) -> _T: + if not self._buffer and not self._eof: + assert not self._waiter + self._waiter = self._loop.create_future() + try: + await self._waiter + except (asyncio.CancelledError, asyncio.TimeoutError): + self._waiter = None + raise + if self._buffer: + data, _ = self._buffer.popleft() + return data + if self._exception is not None: + raise self._exception + raise EofStream + + def __aiter__(self) -> AsyncStreamIterator[_T]: + return AsyncStreamIterator(self.read) + + +class FlowControlDataQueue(DataQueue[_T]): + """FlowControlDataQueue resumes and pauses an underlying stream. + + It is a destination for parsed data. + + This class is deprecated and will be removed in version 4.0. + """ + + def __init__( + self, protocol: BaseProtocol, limit: int, *, loop: asyncio.AbstractEventLoop + ) -> None: + super().__init__(loop=loop) + self._size = 0 + self._protocol = protocol + self._limit = limit * 2 + + def feed_data(self, data: _T, size: int = 0) -> None: + super().feed_data(data, size) + self._size += size + + if self._size > self._limit and not self._protocol._reading_paused: + self._protocol.pause_reading() + + async def read(self) -> _T: + if not self._buffer and not self._eof: + assert not self._waiter + self._waiter = self._loop.create_future() + try: + await self._waiter + except (asyncio.CancelledError, asyncio.TimeoutError): + self._waiter = None + raise + if self._buffer: + data, size = self._buffer.popleft() + self._size -= size + if self._size < self._limit and self._protocol._reading_paused: + self._protocol.resume_reading() + return data + if self._exception is not None: + raise self._exception + raise EofStream diff --git a/py_modules/aiohttp/tcp_helpers.py b/py_modules/aiohttp/tcp_helpers.py new file mode 100644 index 0000000..88b2442 --- /dev/null +++ b/py_modules/aiohttp/tcp_helpers.py @@ -0,0 +1,37 @@ +"""Helper methods to tune a TCP connection""" + +import asyncio +import socket +from contextlib import suppress +from typing import Optional # noqa + +__all__ = ("tcp_keepalive", "tcp_nodelay") + + +if hasattr(socket, "SO_KEEPALIVE"): + + def tcp_keepalive(transport: asyncio.Transport) -> None: + sock = transport.get_extra_info("socket") + if sock is not None: + sock.setsockopt(socket.SOL_SOCKET, socket.SO_KEEPALIVE, 1) + +else: + + def tcp_keepalive(transport: asyncio.Transport) -> None: # pragma: no cover + pass + + +def tcp_nodelay(transport: asyncio.Transport, value: bool) -> None: + sock = transport.get_extra_info("socket") + + if sock is None: + return + + if sock.family not in (socket.AF_INET, socket.AF_INET6): + return + + value = bool(value) + + # socket may be closed already, on windows OSError get raised + with suppress(OSError): + sock.setsockopt(socket.IPPROTO_TCP, socket.TCP_NODELAY, value) diff --git a/py_modules/aiohttp/test_utils.py b/py_modules/aiohttp/test_utils.py new file mode 100644 index 0000000..1d4f59a --- /dev/null +++ b/py_modules/aiohttp/test_utils.py @@ -0,0 +1,808 @@ +"""Utilities shared by tests.""" + +import asyncio +import contextlib +import gc +import inspect +import ipaddress +import os +import socket +import sys +import warnings +from abc import ABC, abstractmethod +from collections.abc import Callable, Iterator +from types import TracebackType +from typing import TYPE_CHECKING, Any, Generic, Literal, TypeVar, cast, overload +from unittest import IsolatedAsyncioTestCase, mock + +from aiosignal import Signal +from multidict import CIMultiDict, CIMultiDictProxy +from yarl import URL + +import aiohttp +from aiohttp.client import ( + _BaseRequestContextManager, + _RequestContextManager, + _RequestOptions, + _WSRequestContextManager, +) + +from . import ClientSession, hdrs +from .abc import AbstractCookieJar +from .client_reqrep import ClientResponse +from .client_ws import ClientWebSocketResponse +from .helpers import sentinel +from .http import HttpVersion, RawRequestMessage +from .streams import EMPTY_PAYLOAD, StreamReader +from .typedefs import StrOrURL +from .web import ( + Application, + AppRunner, + BaseRequest, + BaseRunner, + Request, + Server, + ServerRunner, + SockSite, + UrlMappingMatchInfo, +) +from .web_protocol import _RequestHandler + +if TYPE_CHECKING: + from ssl import SSLContext +else: + SSLContext = Any + +if sys.version_info >= (3, 11) and TYPE_CHECKING: + from typing import Unpack + +if sys.version_info >= (3, 11): + from typing import Self +else: + Self = Any + +_ApplicationNone = TypeVar("_ApplicationNone", Application, None) +_Request = TypeVar("_Request", bound=BaseRequest) + +REUSE_ADDRESS = os.name == "posix" and sys.platform != "cygwin" + + +def get_unused_port_socket( + host: str, family: socket.AddressFamily = socket.AF_INET +) -> socket.socket: + return get_port_socket(host, 0, family) + + +def get_port_socket( + host: str, port: int, family: socket.AddressFamily +) -> socket.socket: + s = socket.socket(family, socket.SOCK_STREAM) + if REUSE_ADDRESS: + # Windows has different semantics for SO_REUSEADDR, + # so don't set it. Ref: + # https://docs.microsoft.com/en-us/windows/win32/winsock/using-so-reuseaddr-and-so-exclusiveaddruse + s.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) + s.bind((host, port)) + return s + + +def unused_port() -> int: + """Return a port that is unused on the current host.""" + with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s: + s.bind(("127.0.0.1", 0)) + return cast(int, s.getsockname()[1]) + + +class BaseTestServer(ABC): + __test__ = False + + def __init__( + self, + *, + scheme: str = "", + loop: asyncio.AbstractEventLoop | None = None, + host: str = "127.0.0.1", + port: int | None = None, + skip_url_asserts: bool = False, + socket_factory: Callable[ + [str, int, socket.AddressFamily], socket.socket + ] = get_port_socket, + **kwargs: Any, + ) -> None: + self._loop = loop + self.runner: BaseRunner | None = None + self._root: URL | None = None + self.host = host + self.port = port + self._closed = False + self.scheme = scheme + self.skip_url_asserts = skip_url_asserts + self.socket_factory = socket_factory + + async def start_server( + self, loop: asyncio.AbstractEventLoop | None = None, **kwargs: Any + ) -> None: + if self.runner: + return + self._loop = loop + self._ssl = kwargs.pop("ssl", None) + self.runner = await self._make_runner(handler_cancellation=True, **kwargs) + await self.runner.setup() + if not self.port: + self.port = 0 + absolute_host = self.host + try: + version = ipaddress.ip_address(self.host).version + except ValueError: + version = 4 + if version == 6: + absolute_host = f"[{self.host}]" + family = socket.AF_INET6 if version == 6 else socket.AF_INET + _sock = self.socket_factory(self.host, self.port, family) + self.host, self.port = _sock.getsockname()[:2] + site = SockSite(self.runner, sock=_sock, ssl_context=self._ssl) + await site.start() + server = site._server + assert server is not None + sockets = server.sockets # type: ignore[attr-defined] + assert sockets is not None + self.port = sockets[0].getsockname()[1] + if not self.scheme: + self.scheme = "https" if self._ssl else "http" + self._root = URL(f"{self.scheme}://{absolute_host}:{self.port}") + + @abstractmethod # pragma: no cover + async def _make_runner(self, **kwargs: Any) -> BaseRunner: + pass + + def make_url(self, path: StrOrURL) -> URL: + assert self._root is not None + url = URL(path) + if not self.skip_url_asserts: + assert not url.absolute + return self._root.join(url) + else: + return URL(str(self._root) + str(path)) + + @property + def started(self) -> bool: + return self.runner is not None + + @property + def closed(self) -> bool: + return self._closed + + @property + def handler(self) -> Server: + # for backward compatibility + # web.Server instance + runner = self.runner + assert runner is not None + assert runner.server is not None + return runner.server + + async def close(self) -> None: + """Close all fixtures created by the test client. + + After that point, the TestClient is no longer usable. + + This is an idempotent function: running close multiple times + will not have any additional effects. + + close is also run when the object is garbage collected, and on + exit when used as a context manager. + + """ + if self.started and not self.closed: + assert self.runner is not None + await self.runner.cleanup() + self._root = None + self.port = None + self._closed = True + + def __enter__(self) -> None: + raise TypeError("Use async with instead") + + def __exit__( + self, + exc_type: type[BaseException] | None, + exc_value: BaseException | None, + traceback: TracebackType | None, + ) -> None: + # __exit__ should exist in pair with __enter__ but never executed + pass # pragma: no cover + + async def __aenter__(self) -> "BaseTestServer": + await self.start_server(loop=self._loop) + return self + + async def __aexit__( + self, + exc_type: type[BaseException] | None, + exc_value: BaseException | None, + traceback: TracebackType | None, + ) -> None: + await self.close() + + +class TestServer(BaseTestServer): + def __init__( + self, + app: Application, + *, + scheme: str = "", + host: str = "127.0.0.1", + port: int | None = None, + **kwargs: Any, + ): + self.app = app + super().__init__(scheme=scheme, host=host, port=port, **kwargs) + + async def _make_runner(self, **kwargs: Any) -> BaseRunner: + return AppRunner(self.app, **kwargs) + + +class RawTestServer(BaseTestServer): + def __init__( + self, + handler: _RequestHandler, + *, + scheme: str = "", + host: str = "127.0.0.1", + port: int | None = None, + **kwargs: Any, + ) -> None: + self._handler = handler + super().__init__(scheme=scheme, host=host, port=port, **kwargs) + + async def _make_runner(self, debug: bool = True, **kwargs: Any) -> ServerRunner: + srv = Server(self._handler, loop=self._loop, debug=debug, **kwargs) + return ServerRunner(srv, debug=debug, **kwargs) + + +class TestClient(Generic[_Request, _ApplicationNone]): + """ + A test client implementation. + + To write functional tests for aiohttp based servers. + + """ + + __test__ = False + + @overload + def __init__( + self: "TestClient[Request, Application]", + server: TestServer, + *, + cookie_jar: AbstractCookieJar | None = None, + **kwargs: Any, + ) -> None: ... + @overload + def __init__( + self: "TestClient[_Request, None]", + server: BaseTestServer, + *, + cookie_jar: AbstractCookieJar | None = None, + **kwargs: Any, + ) -> None: ... + def __init__( + self, + server: BaseTestServer, + *, + cookie_jar: AbstractCookieJar | None = None, + loop: asyncio.AbstractEventLoop | None = None, + **kwargs: Any, + ) -> None: + if not isinstance(server, BaseTestServer): + raise TypeError( + "server must be TestServer instance, found type: %r" % type(server) + ) + self._server = server + self._loop = loop + if cookie_jar is None: + cookie_jar = aiohttp.CookieJar(unsafe=True, loop=loop) + self._session = ClientSession(loop=loop, cookie_jar=cookie_jar, **kwargs) + self._session._retry_connection = False + self._closed = False + self._responses: list[ClientResponse] = [] + self._websockets: list[ClientWebSocketResponse[bool]] = [] + + async def start_server(self) -> None: + await self._server.start_server(loop=self._loop) + + @property + def host(self) -> str: + return self._server.host + + @property + def port(self) -> int | None: + return self._server.port + + @property + def server(self) -> BaseTestServer: + return self._server + + @property + def app(self) -> _ApplicationNone: + return getattr(self._server, "app", None) # type: ignore[return-value] + + @property + def session(self) -> ClientSession: + """An internal aiohttp.ClientSession. + + Unlike the methods on the TestClient, client session requests + do not automatically include the host in the url queried, and + will require an absolute path to the resource. + + """ + return self._session + + def make_url(self, path: StrOrURL) -> URL: + return self._server.make_url(path) + + async def _request( + self, method: str, path: StrOrURL, **kwargs: Any + ) -> ClientResponse: + resp = await self._session.request(method, self.make_url(path), **kwargs) + # save it to close later + self._responses.append(resp) + return resp + + if sys.version_info >= (3, 11) and TYPE_CHECKING: + + def request( + self, method: str, path: StrOrURL, **kwargs: Unpack[_RequestOptions] + ) -> _RequestContextManager: ... + + def get( + self, + path: StrOrURL, + **kwargs: Unpack[_RequestOptions], + ) -> _RequestContextManager: ... + + def options( + self, + path: StrOrURL, + **kwargs: Unpack[_RequestOptions], + ) -> _RequestContextManager: ... + + def head( + self, + path: StrOrURL, + **kwargs: Unpack[_RequestOptions], + ) -> _RequestContextManager: ... + + def post( + self, + path: StrOrURL, + **kwargs: Unpack[_RequestOptions], + ) -> _RequestContextManager: ... + + def put( + self, + path: StrOrURL, + **kwargs: Unpack[_RequestOptions], + ) -> _RequestContextManager: ... + + def patch( + self, + path: StrOrURL, + **kwargs: Unpack[_RequestOptions], + ) -> _RequestContextManager: ... + + def delete( + self, + path: StrOrURL, + **kwargs: Unpack[_RequestOptions], + ) -> _RequestContextManager: ... + + else: + + def request( + self, method: str, path: StrOrURL, **kwargs: Any + ) -> _RequestContextManager: + """Routes a request to tested http server. + + The interface is identical to aiohttp.ClientSession.request, + except the loop kwarg is overridden by the instance used by the + test server. + + """ + return _RequestContextManager(self._request(method, path, **kwargs)) + + def get(self, path: StrOrURL, **kwargs: Any) -> _RequestContextManager: + """Perform an HTTP GET request.""" + return _RequestContextManager(self._request(hdrs.METH_GET, path, **kwargs)) + + def post(self, path: StrOrURL, **kwargs: Any) -> _RequestContextManager: + """Perform an HTTP POST request.""" + return _RequestContextManager(self._request(hdrs.METH_POST, path, **kwargs)) + + def options(self, path: StrOrURL, **kwargs: Any) -> _RequestContextManager: + """Perform an HTTP OPTIONS request.""" + return _RequestContextManager( + self._request(hdrs.METH_OPTIONS, path, **kwargs) + ) + + def head(self, path: StrOrURL, **kwargs: Any) -> _RequestContextManager: + """Perform an HTTP HEAD request.""" + return _RequestContextManager(self._request(hdrs.METH_HEAD, path, **kwargs)) + + def put(self, path: StrOrURL, **kwargs: Any) -> _RequestContextManager: + """Perform an HTTP PUT request.""" + return _RequestContextManager(self._request(hdrs.METH_PUT, path, **kwargs)) + + def patch(self, path: StrOrURL, **kwargs: Any) -> _RequestContextManager: + """Perform an HTTP PATCH request.""" + return _RequestContextManager( + self._request(hdrs.METH_PATCH, path, **kwargs) + ) + + def delete(self, path: StrOrURL, **kwargs: Any) -> _RequestContextManager: + """Perform an HTTP PATCH request.""" + return _RequestContextManager( + self._request(hdrs.METH_DELETE, path, **kwargs) + ) + + @overload + def ws_connect( + self, path: StrOrURL, *, decode_text: Literal[True] = ..., **kwargs: Any + ) -> "_BaseRequestContextManager[ClientWebSocketResponse[Literal[True]]]": ... + + @overload + def ws_connect( + self, path: StrOrURL, *, decode_text: Literal[False], **kwargs: Any + ) -> "_BaseRequestContextManager[ClientWebSocketResponse[Literal[False]]]": ... + + @overload + def ws_connect( + self, path: StrOrURL, *, decode_text: bool = ..., **kwargs: Any + ) -> "_BaseRequestContextManager[ClientWebSocketResponse[bool]]": ... + + def ws_connect( + self, path: StrOrURL, *, decode_text: bool = True, **kwargs: Any + ) -> "_BaseRequestContextManager[ClientWebSocketResponse[bool]]": + """Initiate websocket connection. + + The api corresponds to aiohttp.ClientSession.ws_connect. + + """ + return _WSRequestContextManager( + self._ws_connect(path, decode_text=decode_text, **kwargs) + ) + + @overload + async def _ws_connect( + self, path: StrOrURL, *, decode_text: Literal[True] = ..., **kwargs: Any + ) -> "ClientWebSocketResponse[Literal[True]]": ... + + @overload + async def _ws_connect( + self, path: StrOrURL, *, decode_text: Literal[False], **kwargs: Any + ) -> "ClientWebSocketResponse[Literal[False]]": ... + + @overload + async def _ws_connect( + self, path: StrOrURL, *, decode_text: bool = ..., **kwargs: Any + ) -> "ClientWebSocketResponse[bool]": ... + + async def _ws_connect( + self, path: StrOrURL, *, decode_text: bool = True, **kwargs: Any + ) -> "ClientWebSocketResponse[bool]": + ws = await self._session.ws_connect( + self.make_url(path), decode_text=decode_text, **kwargs + ) + self._websockets.append(ws) + return ws + + async def close(self) -> None: + """Close all fixtures created by the test client. + + After that point, the TestClient is no longer usable. + + This is an idempotent function: running close multiple times + will not have any additional effects. + + close is also run on exit when used as a(n) (asynchronous) + context manager. + + """ + if not self._closed: + for resp in self._responses: + resp.close() + for ws in self._websockets: + await ws.close() + await self._session.close() + await self._server.close() + self._closed = True + + def __enter__(self) -> None: + raise TypeError("Use async with instead") + + def __exit__( + self, + exc_type: type[BaseException] | None, + exc: BaseException | None, + tb: TracebackType | None, + ) -> None: + # __exit__ should exist in pair with __enter__ but never executed + pass # pragma: no cover + + async def __aenter__(self) -> Self: + await self.start_server() + return self + + async def __aexit__( + self, + exc_type: type[BaseException] | None, + exc: BaseException | None, + tb: TracebackType | None, + ) -> None: + await self.close() + + +class AioHTTPTestCase(IsolatedAsyncioTestCase): + """A base class to allow for unittest web applications using aiohttp. + + Provides the following: + + * self.client (aiohttp.test_utils.TestClient): an aiohttp test client. + * self.loop (asyncio.BaseEventLoop): the event loop in which the + application and server are running. + * self.app (aiohttp.web.Application): the application returned by + self.get_application() + + Note that the TestClient's methods are asynchronous: you have to + execute function on the test client using asynchronous methods. + """ + + async def get_application(self) -> Application: + """Get application. + + This method should be overridden + to return the aiohttp.web.Application + object to test. + """ + return self.get_app() + + def get_app(self) -> Application: + """Obsolete method used to constructing web application. + + Use .get_application() coroutine instead. + """ + raise RuntimeError("Did you forget to define get_application()?") + + async def asyncSetUp(self) -> None: + self.loop = asyncio.get_running_loop() + return await self.setUpAsync() + + async def setUpAsync(self) -> None: + self.app = await self.get_application() + self.server = await self.get_server(self.app) + self.client = await self.get_client(self.server) + + await self.client.start_server() + + async def asyncTearDown(self) -> None: + return await self.tearDownAsync() + + async def tearDownAsync(self) -> None: + await self.client.close() + + async def get_server(self, app: Application) -> TestServer: + """Return a TestServer instance.""" + return TestServer(app, loop=self.loop) + + async def get_client(self, server: TestServer) -> TestClient[Request, Application]: + """Return a TestClient instance.""" + return TestClient(server, loop=self.loop) + + +def unittest_run_loop(func: Any, *args: Any, **kwargs: Any) -> Any: + """ + A decorator dedicated to use with asynchronous AioHTTPTestCase test methods. + + In 3.8+, this does nothing. + """ + warnings.warn( + "Decorator `@unittest_run_loop` is no longer needed in aiohttp 3.8+", + DeprecationWarning, + stacklevel=2, + ) + return func + + +_LOOP_FACTORY = Callable[[], asyncio.AbstractEventLoop] + + +@contextlib.contextmanager +def loop_context( + loop_factory: _LOOP_FACTORY = asyncio.new_event_loop, fast: bool = False +) -> Iterator[asyncio.AbstractEventLoop]: + """A contextmanager that creates an event_loop, for test purposes. + + Handles the creation and cleanup of a test loop. + """ + loop = setup_test_loop(loop_factory) + yield loop + teardown_test_loop(loop, fast=fast) + + +def setup_test_loop( + loop_factory: _LOOP_FACTORY = asyncio.new_event_loop, +) -> asyncio.AbstractEventLoop: + """Create and return an asyncio.BaseEventLoop instance. + + The caller should also call teardown_test_loop, + once they are done with the loop. + """ + loop = loop_factory() + asyncio.set_event_loop(loop) + return loop + + +def teardown_test_loop(loop: asyncio.AbstractEventLoop, fast: bool = False) -> None: + """Teardown and cleanup an event_loop created by setup_test_loop.""" + closed = loop.is_closed() + if not closed: + loop.call_soon(loop.stop) + loop.run_forever() + loop.close() + + if not fast: + gc.collect() + + asyncio.set_event_loop(None) + + +def _create_app_mock() -> mock.MagicMock: + def get_dict(app: Any, key: str) -> Any: + return app.__app_dict[key] + + def set_dict(app: Any, key: str, value: Any) -> None: + app.__app_dict[key] = value + + app = mock.MagicMock(spec=Application) + app.__app_dict = {} + app.__getitem__ = get_dict + app.__setitem__ = set_dict + + app._debug = False + app.on_response_prepare = Signal(app) + app.on_response_prepare.freeze() + return app + + +def _create_transport(sslcontext: SSLContext | None = None) -> mock.Mock: + transport = mock.Mock() + + def get_extra_info(key: str) -> SSLContext | tuple[str, int] | None: + if key == "sslcontext": + return sslcontext + return ("127.0.0.1", 80) if key == "sockname" else None + + transport.get_extra_info.side_effect = get_extra_info + return transport + + +def make_mocked_request( + method: str, + path: str, + headers: Any = None, + *, + match_info: Any = sentinel, + version: HttpVersion = HttpVersion(1, 1), + closing: bool = False, + app: Any = None, + writer: Any = sentinel, + protocol: Any = sentinel, + transport: Any = sentinel, + payload: StreamReader = EMPTY_PAYLOAD, + sslcontext: SSLContext | None = None, + client_max_size: int = 1024**2, + loop: Any = ..., +) -> Request: + """Creates mocked web.Request testing purposes. + + Useful in unit tests, when spinning full web server is overkill or + specific conditions and errors are hard to trigger. + """ + task = mock.Mock() + if loop is ...: + # no loop passed, try to get the current one if + # its is running as we need a real loop to create + # executor jobs to be able to do testing + # with a real executor + try: + loop = asyncio.get_running_loop() + except RuntimeError: + loop = mock.Mock() + loop.create_future.return_value = () + + if version < HttpVersion(1, 1): + closing = True + + if headers: + headers = CIMultiDictProxy(CIMultiDict(headers)) + raw_hdrs = tuple( + (k.encode("utf-8"), v.encode("utf-8")) for k, v in headers.items() + ) + else: + headers = CIMultiDictProxy(CIMultiDict()) + raw_hdrs = () + + chunked = "chunked" in headers.get(hdrs.TRANSFER_ENCODING, "").lower() + upgrade = headers.get(hdrs.CONNECTION, "").lower() == "upgrade" and bool( + headers.get(hdrs.UPGRADE) + ) + + message = RawRequestMessage( + method.upper(), + path, + version, + headers, + raw_hdrs, + closing, + None, + upgrade, + chunked, + URL(path), + ) + if app is None: + app = _create_app_mock() + + if transport is sentinel: + transport = _create_transport(sslcontext) + + if protocol is sentinel: + protocol = mock.Mock() + protocol.max_field_size = 8190 + protocol.max_line_length = 8190 + protocol.max_headers = 128 + protocol.transport = transport + type(protocol).peername = mock.PropertyMock( + return_value=transport.get_extra_info("peername") + ) + type(protocol).sockname = mock.PropertyMock( + return_value=transport.get_extra_info("sockname") + ) + type(protocol).ssl_context = mock.PropertyMock(return_value=sslcontext) + + if writer is sentinel: + writer = mock.Mock() + writer.write_headers = make_mocked_coro(None) + writer.write = make_mocked_coro(None) + writer.write_eof = make_mocked_coro(None) + writer.drain = make_mocked_coro(None) + writer.transport = transport + + protocol.transport = transport + protocol.writer = writer + + req = Request( + message, payload, protocol, writer, task, loop, client_max_size=client_max_size + ) + + match_info = UrlMappingMatchInfo( + {} if match_info is sentinel else match_info, mock.Mock() + ) + match_info.add_app(app) + req._match_info = match_info + + return req + + +def make_mocked_coro( + return_value: Any = sentinel, raise_exception: Any = sentinel +) -> Any: + """Creates a coroutine mock.""" + + async def mock_coro(*args: Any, **kwargs: Any) -> Any: + if raise_exception is not sentinel: + raise raise_exception + if not inspect.isawaitable(return_value): + return return_value + await return_value + + return mock.Mock(wraps=mock_coro) diff --git a/py_modules/aiohttp/tracing.py b/py_modules/aiohttp/tracing.py new file mode 100644 index 0000000..cfee054 --- /dev/null +++ b/py_modules/aiohttp/tracing.py @@ -0,0 +1,453 @@ +from types import SimpleNamespace +from typing import TYPE_CHECKING, Any, TypeVar + +import attr +from aiosignal import Signal +from multidict import CIMultiDict +from yarl import URL + +from .client_reqrep import ClientResponse + +if TYPE_CHECKING: + from .client import ClientSession + + _ParamT_contra = TypeVar("_ParamT_contra", contravariant=True) + _TracingSignal = Signal[ClientSession, SimpleNamespace, _ParamT_contra] + + +__all__ = ( + "TraceConfig", + "TraceRequestStartParams", + "TraceRequestEndParams", + "TraceRequestExceptionParams", + "TraceConnectionQueuedStartParams", + "TraceConnectionQueuedEndParams", + "TraceConnectionCreateStartParams", + "TraceConnectionCreateEndParams", + "TraceConnectionReuseconnParams", + "TraceDnsResolveHostStartParams", + "TraceDnsResolveHostEndParams", + "TraceDnsCacheHitParams", + "TraceDnsCacheMissParams", + "TraceRequestRedirectParams", + "TraceRequestChunkSentParams", + "TraceResponseChunkReceivedParams", + "TraceRequestHeadersSentParams", +) + + +class TraceConfig: + """First-class used to trace requests launched via ClientSession objects.""" + + def __init__( + self, trace_config_ctx_factory: type[SimpleNamespace] = SimpleNamespace + ) -> None: + self._on_request_start: _TracingSignal[TraceRequestStartParams] = Signal(self) + self._on_request_chunk_sent: _TracingSignal[TraceRequestChunkSentParams] = ( + Signal(self) + ) + self._on_response_chunk_received: _TracingSignal[ + TraceResponseChunkReceivedParams + ] = Signal(self) + self._on_request_end: _TracingSignal[TraceRequestEndParams] = Signal(self) + self._on_request_exception: _TracingSignal[TraceRequestExceptionParams] = ( + Signal(self) + ) + self._on_request_redirect: _TracingSignal[TraceRequestRedirectParams] = Signal( + self + ) + self._on_connection_queued_start: _TracingSignal[ + TraceConnectionQueuedStartParams + ] = Signal(self) + self._on_connection_queued_end: _TracingSignal[ + TraceConnectionQueuedEndParams + ] = Signal(self) + self._on_connection_create_start: _TracingSignal[ + TraceConnectionCreateStartParams + ] = Signal(self) + self._on_connection_create_end: _TracingSignal[ + TraceConnectionCreateEndParams + ] = Signal(self) + self._on_connection_reuseconn: _TracingSignal[ + TraceConnectionReuseconnParams + ] = Signal(self) + self._on_dns_resolvehost_start: _TracingSignal[ + TraceDnsResolveHostStartParams + ] = Signal(self) + self._on_dns_resolvehost_end: _TracingSignal[TraceDnsResolveHostEndParams] = ( + Signal(self) + ) + self._on_dns_cache_hit: _TracingSignal[TraceDnsCacheHitParams] = Signal(self) + self._on_dns_cache_miss: _TracingSignal[TraceDnsCacheMissParams] = Signal(self) + self._on_request_headers_sent: _TracingSignal[TraceRequestHeadersSentParams] = ( + Signal(self) + ) + + self._trace_config_ctx_factory = trace_config_ctx_factory + + def trace_config_ctx(self, trace_request_ctx: Any = None) -> SimpleNamespace: + """Return a new trace_config_ctx instance""" + return self._trace_config_ctx_factory(trace_request_ctx=trace_request_ctx) + + def freeze(self) -> None: + self._on_request_start.freeze() + self._on_request_chunk_sent.freeze() + self._on_response_chunk_received.freeze() + self._on_request_end.freeze() + self._on_request_exception.freeze() + self._on_request_redirect.freeze() + self._on_connection_queued_start.freeze() + self._on_connection_queued_end.freeze() + self._on_connection_create_start.freeze() + self._on_connection_create_end.freeze() + self._on_connection_reuseconn.freeze() + self._on_dns_resolvehost_start.freeze() + self._on_dns_resolvehost_end.freeze() + self._on_dns_cache_hit.freeze() + self._on_dns_cache_miss.freeze() + self._on_request_headers_sent.freeze() + + @property + def on_request_start(self) -> "_TracingSignal[TraceRequestStartParams]": + return self._on_request_start + + @property + def on_request_chunk_sent( + self, + ) -> "_TracingSignal[TraceRequestChunkSentParams]": + return self._on_request_chunk_sent + + @property + def on_response_chunk_received( + self, + ) -> "_TracingSignal[TraceResponseChunkReceivedParams]": + return self._on_response_chunk_received + + @property + def on_request_end(self) -> "_TracingSignal[TraceRequestEndParams]": + return self._on_request_end + + @property + def on_request_exception( + self, + ) -> "_TracingSignal[TraceRequestExceptionParams]": + return self._on_request_exception + + @property + def on_request_redirect( + self, + ) -> "_TracingSignal[TraceRequestRedirectParams]": + return self._on_request_redirect + + @property + def on_connection_queued_start( + self, + ) -> "_TracingSignal[TraceConnectionQueuedStartParams]": + return self._on_connection_queued_start + + @property + def on_connection_queued_end( + self, + ) -> "_TracingSignal[TraceConnectionQueuedEndParams]": + return self._on_connection_queued_end + + @property + def on_connection_create_start( + self, + ) -> "_TracingSignal[TraceConnectionCreateStartParams]": + return self._on_connection_create_start + + @property + def on_connection_create_end( + self, + ) -> "_TracingSignal[TraceConnectionCreateEndParams]": + return self._on_connection_create_end + + @property + def on_connection_reuseconn( + self, + ) -> "_TracingSignal[TraceConnectionReuseconnParams]": + return self._on_connection_reuseconn + + @property + def on_dns_resolvehost_start( + self, + ) -> "_TracingSignal[TraceDnsResolveHostStartParams]": + return self._on_dns_resolvehost_start + + @property + def on_dns_resolvehost_end( + self, + ) -> "_TracingSignal[TraceDnsResolveHostEndParams]": + return self._on_dns_resolvehost_end + + @property + def on_dns_cache_hit(self) -> "_TracingSignal[TraceDnsCacheHitParams]": + return self._on_dns_cache_hit + + @property + def on_dns_cache_miss(self) -> "_TracingSignal[TraceDnsCacheMissParams]": + return self._on_dns_cache_miss + + @property + def on_request_headers_sent( + self, + ) -> "_TracingSignal[TraceRequestHeadersSentParams]": + return self._on_request_headers_sent + + +@attr.s(auto_attribs=True, frozen=True, slots=True) +class TraceRequestStartParams: + """Parameters sent by the `on_request_start` signal""" + + method: str + url: URL + headers: "CIMultiDict[str]" + + +@attr.s(auto_attribs=True, frozen=True, slots=True) +class TraceRequestChunkSentParams: + """Parameters sent by the `on_request_chunk_sent` signal""" + + method: str + url: URL + chunk: bytes + + +@attr.s(auto_attribs=True, frozen=True, slots=True) +class TraceResponseChunkReceivedParams: + """Parameters sent by the `on_response_chunk_received` signal""" + + method: str + url: URL + chunk: bytes + + +@attr.s(auto_attribs=True, frozen=True, slots=True) +class TraceRequestEndParams: + """Parameters sent by the `on_request_end` signal""" + + method: str + url: URL + headers: "CIMultiDict[str]" + response: ClientResponse + + +@attr.s(auto_attribs=True, frozen=True, slots=True) +class TraceRequestExceptionParams: + """Parameters sent by the `on_request_exception` signal""" + + method: str + url: URL + headers: "CIMultiDict[str]" + exception: BaseException + + +@attr.s(auto_attribs=True, frozen=True, slots=True) +class TraceRequestRedirectParams: + """Parameters sent by the `on_request_redirect` signal""" + + method: str + url: URL + headers: "CIMultiDict[str]" + response: ClientResponse + + +@attr.s(auto_attribs=True, frozen=True, slots=True) +class TraceConnectionQueuedStartParams: + """Parameters sent by the `on_connection_queued_start` signal""" + + +@attr.s(auto_attribs=True, frozen=True, slots=True) +class TraceConnectionQueuedEndParams: + """Parameters sent by the `on_connection_queued_end` signal""" + + +@attr.s(auto_attribs=True, frozen=True, slots=True) +class TraceConnectionCreateStartParams: + """Parameters sent by the `on_connection_create_start` signal""" + + +@attr.s(auto_attribs=True, frozen=True, slots=True) +class TraceConnectionCreateEndParams: + """Parameters sent by the `on_connection_create_end` signal""" + + +@attr.s(auto_attribs=True, frozen=True, slots=True) +class TraceConnectionReuseconnParams: + """Parameters sent by the `on_connection_reuseconn` signal""" + + +@attr.s(auto_attribs=True, frozen=True, slots=True) +class TraceDnsResolveHostStartParams: + """Parameters sent by the `on_dns_resolvehost_start` signal""" + + host: str + + +@attr.s(auto_attribs=True, frozen=True, slots=True) +class TraceDnsResolveHostEndParams: + """Parameters sent by the `on_dns_resolvehost_end` signal""" + + host: str + + +@attr.s(auto_attribs=True, frozen=True, slots=True) +class TraceDnsCacheHitParams: + """Parameters sent by the `on_dns_cache_hit` signal""" + + host: str + + +@attr.s(auto_attribs=True, frozen=True, slots=True) +class TraceDnsCacheMissParams: + """Parameters sent by the `on_dns_cache_miss` signal""" + + host: str + + +@attr.s(auto_attribs=True, frozen=True, slots=True) +class TraceRequestHeadersSentParams: + """Parameters sent by the `on_request_headers_sent` signal""" + + method: str + url: URL + headers: "CIMultiDict[str]" + + +class Trace: + """Internal dependency holder class. + + Used to keep together the main dependencies used + at the moment of send a signal. + """ + + def __init__( + self, + session: "ClientSession", + trace_config: TraceConfig, + trace_config_ctx: SimpleNamespace, + ) -> None: + self._trace_config = trace_config + self._trace_config_ctx = trace_config_ctx + self._session = session + + async def send_request_start( + self, method: str, url: URL, headers: "CIMultiDict[str]" + ) -> None: + return await self._trace_config.on_request_start.send( + self._session, + self._trace_config_ctx, + TraceRequestStartParams(method, url, headers), + ) + + async def send_request_chunk_sent( + self, method: str, url: URL, chunk: bytes + ) -> None: + return await self._trace_config.on_request_chunk_sent.send( + self._session, + self._trace_config_ctx, + TraceRequestChunkSentParams(method, url, chunk), + ) + + async def send_response_chunk_received( + self, method: str, url: URL, chunk: bytes + ) -> None: + return await self._trace_config.on_response_chunk_received.send( + self._session, + self._trace_config_ctx, + TraceResponseChunkReceivedParams(method, url, chunk), + ) + + async def send_request_end( + self, + method: str, + url: URL, + headers: "CIMultiDict[str]", + response: ClientResponse, + ) -> None: + return await self._trace_config.on_request_end.send( + self._session, + self._trace_config_ctx, + TraceRequestEndParams(method, url, headers, response), + ) + + async def send_request_exception( + self, + method: str, + url: URL, + headers: "CIMultiDict[str]", + exception: BaseException, + ) -> None: + return await self._trace_config.on_request_exception.send( + self._session, + self._trace_config_ctx, + TraceRequestExceptionParams(method, url, headers, exception), + ) + + async def send_request_redirect( + self, + method: str, + url: URL, + headers: "CIMultiDict[str]", + response: ClientResponse, + ) -> None: + return await self._trace_config._on_request_redirect.send( + self._session, + self._trace_config_ctx, + TraceRequestRedirectParams(method, url, headers, response), + ) + + async def send_connection_queued_start(self) -> None: + return await self._trace_config.on_connection_queued_start.send( + self._session, self._trace_config_ctx, TraceConnectionQueuedStartParams() + ) + + async def send_connection_queued_end(self) -> None: + return await self._trace_config.on_connection_queued_end.send( + self._session, self._trace_config_ctx, TraceConnectionQueuedEndParams() + ) + + async def send_connection_create_start(self) -> None: + return await self._trace_config.on_connection_create_start.send( + self._session, self._trace_config_ctx, TraceConnectionCreateStartParams() + ) + + async def send_connection_create_end(self) -> None: + return await self._trace_config.on_connection_create_end.send( + self._session, self._trace_config_ctx, TraceConnectionCreateEndParams() + ) + + async def send_connection_reuseconn(self) -> None: + return await self._trace_config.on_connection_reuseconn.send( + self._session, self._trace_config_ctx, TraceConnectionReuseconnParams() + ) + + async def send_dns_resolvehost_start(self, host: str) -> None: + return await self._trace_config.on_dns_resolvehost_start.send( + self._session, self._trace_config_ctx, TraceDnsResolveHostStartParams(host) + ) + + async def send_dns_resolvehost_end(self, host: str) -> None: + return await self._trace_config.on_dns_resolvehost_end.send( + self._session, self._trace_config_ctx, TraceDnsResolveHostEndParams(host) + ) + + async def send_dns_cache_hit(self, host: str) -> None: + return await self._trace_config.on_dns_cache_hit.send( + self._session, self._trace_config_ctx, TraceDnsCacheHitParams(host) + ) + + async def send_dns_cache_miss(self, host: str) -> None: + return await self._trace_config.on_dns_cache_miss.send( + self._session, self._trace_config_ctx, TraceDnsCacheMissParams(host) + ) + + async def send_request_headers( + self, method: str, url: URL, headers: "CIMultiDict[str]" + ) -> None: + return await self._trace_config._on_request_headers_sent.send( + self._session, + self._trace_config_ctx, + TraceRequestHeadersSentParams(method, url, headers), + ) diff --git a/py_modules/aiohttp/typedefs.py b/py_modules/aiohttp/typedefs.py new file mode 100644 index 0000000..ef27aff --- /dev/null +++ b/py_modules/aiohttp/typedefs.py @@ -0,0 +1,61 @@ +import json +import os +from collections.abc import Awaitable, Callable, Iterable, Mapping +from typing import TYPE_CHECKING, Any, Protocol, Union + +from multidict import CIMultiDict, CIMultiDictProxy, MultiDict, MultiDictProxy, istr +from yarl import URL, Query as _Query + +Query = _Query + +DEFAULT_JSON_ENCODER = json.dumps +DEFAULT_JSON_DECODER = json.loads + +if TYPE_CHECKING: + _CIMultiDict = CIMultiDict[str] + _CIMultiDictProxy = CIMultiDictProxy[str] + _MultiDict = MultiDict[str] + _MultiDictProxy = MultiDictProxy[str] + from http.cookies import BaseCookie, Morsel + + from .web import Request, StreamResponse +else: + _CIMultiDict = CIMultiDict + _CIMultiDictProxy = CIMultiDictProxy + _MultiDict = MultiDict + _MultiDictProxy = MultiDictProxy + +Byteish = Union[bytes, bytearray, memoryview] +JSONEncoder = Callable[[Any], str] +JSONBytesEncoder = Callable[[Any], bytes] +JSONDecoder = Callable[[str], Any] +LooseHeaders = Union[ + Mapping[str, str], + Mapping[istr, str], + _CIMultiDict, + _CIMultiDictProxy, + Iterable[tuple[str | istr, str]], +] +RawHeaders = tuple[tuple[bytes, bytes], ...] +StrOrURL = Union[str, URL] + +LooseCookiesMappings = Mapping[str, Union[str, "BaseCookie[str]", "Morsel[Any]"]] +LooseCookiesIterables = Iterable[ + tuple[str, Union[str, "BaseCookie[str]", "Morsel[Any]"]] +] +LooseCookies = Union[ + LooseCookiesMappings, + LooseCookiesIterables, + "BaseCookie[str]", +] + +Handler = Callable[["Request"], Awaitable["StreamResponse"]] + + +class Middleware(Protocol): + def __call__( + self, request: "Request", handler: Handler + ) -> Awaitable["StreamResponse"]: ... + + +PathLike = Union[str, "os.PathLike[str]"] diff --git a/py_modules/aiohttp/web.py b/py_modules/aiohttp/web.py new file mode 100644 index 0000000..f717d94 --- /dev/null +++ b/py_modules/aiohttp/web.py @@ -0,0 +1,593 @@ +import asyncio +import logging +import os +import socket +import sys +import warnings +from argparse import ArgumentParser +from collections.abc import Awaitable, Callable, Iterable, Iterable as TypingIterable +from contextlib import suppress +from importlib import import_module +from typing import TYPE_CHECKING, Any, cast + +from .abc import AbstractAccessLogger +from .helpers import AppKey, RequestKey, ResponseKey +from .log import access_logger +from .typedefs import PathLike +from .web_app import Application as Application, CleanupError as CleanupError +from .web_exceptions import ( + HTTPAccepted as HTTPAccepted, + HTTPBadGateway as HTTPBadGateway, + HTTPBadRequest as HTTPBadRequest, + HTTPClientError as HTTPClientError, + HTTPConflict as HTTPConflict, + HTTPCreated as HTTPCreated, + HTTPError as HTTPError, + HTTPException as HTTPException, + HTTPExpectationFailed as HTTPExpectationFailed, + HTTPFailedDependency as HTTPFailedDependency, + HTTPForbidden as HTTPForbidden, + HTTPFound as HTTPFound, + HTTPGatewayTimeout as HTTPGatewayTimeout, + HTTPGone as HTTPGone, + HTTPInsufficientStorage as HTTPInsufficientStorage, + HTTPInternalServerError as HTTPInternalServerError, + HTTPLengthRequired as HTTPLengthRequired, + HTTPMethodNotAllowed as HTTPMethodNotAllowed, + HTTPMisdirectedRequest as HTTPMisdirectedRequest, + HTTPMove as HTTPMove, + HTTPMovedPermanently as HTTPMovedPermanently, + HTTPMultipleChoices as HTTPMultipleChoices, + HTTPNetworkAuthenticationRequired as HTTPNetworkAuthenticationRequired, + HTTPNoContent as HTTPNoContent, + HTTPNonAuthoritativeInformation as HTTPNonAuthoritativeInformation, + HTTPNotAcceptable as HTTPNotAcceptable, + HTTPNotExtended as HTTPNotExtended, + HTTPNotFound as HTTPNotFound, + HTTPNotImplemented as HTTPNotImplemented, + HTTPNotModified as HTTPNotModified, + HTTPOk as HTTPOk, + HTTPPartialContent as HTTPPartialContent, + HTTPPaymentRequired as HTTPPaymentRequired, + HTTPPermanentRedirect as HTTPPermanentRedirect, + HTTPPreconditionFailed as HTTPPreconditionFailed, + HTTPPreconditionRequired as HTTPPreconditionRequired, + HTTPProxyAuthenticationRequired as HTTPProxyAuthenticationRequired, + HTTPRedirection as HTTPRedirection, + HTTPRequestEntityTooLarge as HTTPRequestEntityTooLarge, + HTTPRequestHeaderFieldsTooLarge as HTTPRequestHeaderFieldsTooLarge, + HTTPRequestRangeNotSatisfiable as HTTPRequestRangeNotSatisfiable, + HTTPRequestTimeout as HTTPRequestTimeout, + HTTPRequestURITooLong as HTTPRequestURITooLong, + HTTPResetContent as HTTPResetContent, + HTTPSeeOther as HTTPSeeOther, + HTTPServerError as HTTPServerError, + HTTPServiceUnavailable as HTTPServiceUnavailable, + HTTPSuccessful as HTTPSuccessful, + HTTPTemporaryRedirect as HTTPTemporaryRedirect, + HTTPTooManyRequests as HTTPTooManyRequests, + HTTPUnauthorized as HTTPUnauthorized, + HTTPUnavailableForLegalReasons as HTTPUnavailableForLegalReasons, + HTTPUnprocessableEntity as HTTPUnprocessableEntity, + HTTPUnsupportedMediaType as HTTPUnsupportedMediaType, + HTTPUpgradeRequired as HTTPUpgradeRequired, + HTTPUseProxy as HTTPUseProxy, + HTTPVariantAlsoNegotiates as HTTPVariantAlsoNegotiates, + HTTPVersionNotSupported as HTTPVersionNotSupported, + NotAppKeyWarning as NotAppKeyWarning, +) +from .web_fileresponse import FileResponse as FileResponse +from .web_log import AccessLogger +from .web_middlewares import ( + middleware as middleware, + normalize_path_middleware as normalize_path_middleware, +) +from .web_protocol import ( + PayloadAccessError as PayloadAccessError, + RequestHandler as RequestHandler, + RequestPayloadError as RequestPayloadError, +) +from .web_request import ( + BaseRequest as BaseRequest, + FileField as FileField, + Request as Request, +) +from .web_response import ( + ContentCoding as ContentCoding, + Response as Response, + StreamResponse as StreamResponse, + json_bytes_response as json_bytes_response, + json_response as json_response, +) +from .web_routedef import ( + AbstractRouteDef as AbstractRouteDef, + RouteDef as RouteDef, + RouteTableDef as RouteTableDef, + StaticDef as StaticDef, + delete as delete, + get as get, + head as head, + options as options, + patch as patch, + post as post, + put as put, + route as route, + static as static, + view as view, +) +from .web_runner import ( + AppRunner as AppRunner, + BaseRunner as BaseRunner, + BaseSite as BaseSite, + GracefulExit as GracefulExit, + NamedPipeSite as NamedPipeSite, + ServerRunner as ServerRunner, + SockSite as SockSite, + TCPSite as TCPSite, + UnixSite as UnixSite, +) +from .web_server import Server as Server +from .web_urldispatcher import ( + AbstractResource as AbstractResource, + AbstractRoute as AbstractRoute, + DynamicResource as DynamicResource, + PlainResource as PlainResource, + PrefixedSubAppResource as PrefixedSubAppResource, + Resource as Resource, + ResourceRoute as ResourceRoute, + StaticResource as StaticResource, + UrlDispatcher as UrlDispatcher, + UrlMappingMatchInfo as UrlMappingMatchInfo, + View as View, +) +from .web_ws import ( + WebSocketReady as WebSocketReady, + WebSocketResponse as WebSocketResponse, + WSMsgType as WSMsgType, +) + +__all__ = ( + # web_app + "AppKey", + "Application", + "CleanupError", + # web_exceptions + "NotAppKeyWarning", + "HTTPAccepted", + "HTTPBadGateway", + "HTTPBadRequest", + "HTTPClientError", + "HTTPConflict", + "HTTPCreated", + "HTTPError", + "HTTPException", + "HTTPExpectationFailed", + "HTTPFailedDependency", + "HTTPForbidden", + "HTTPFound", + "HTTPGatewayTimeout", + "HTTPGone", + "HTTPInsufficientStorage", + "HTTPInternalServerError", + "HTTPLengthRequired", + "HTTPMethodNotAllowed", + "HTTPMisdirectedRequest", + "HTTPMove", + "HTTPMovedPermanently", + "HTTPMultipleChoices", + "HTTPNetworkAuthenticationRequired", + "HTTPNoContent", + "HTTPNonAuthoritativeInformation", + "HTTPNotAcceptable", + "HTTPNotExtended", + "HTTPNotFound", + "HTTPNotImplemented", + "HTTPNotModified", + "HTTPOk", + "HTTPPartialContent", + "HTTPPaymentRequired", + "HTTPPermanentRedirect", + "HTTPPreconditionFailed", + "HTTPPreconditionRequired", + "HTTPProxyAuthenticationRequired", + "HTTPRedirection", + "HTTPRequestEntityTooLarge", + "HTTPRequestHeaderFieldsTooLarge", + "HTTPRequestRangeNotSatisfiable", + "HTTPRequestTimeout", + "HTTPRequestURITooLong", + "HTTPResetContent", + "HTTPSeeOther", + "HTTPServerError", + "HTTPServiceUnavailable", + "HTTPSuccessful", + "HTTPTemporaryRedirect", + "HTTPTooManyRequests", + "HTTPUnauthorized", + "HTTPUnavailableForLegalReasons", + "HTTPUnprocessableEntity", + "HTTPUnsupportedMediaType", + "HTTPUpgradeRequired", + "HTTPUseProxy", + "HTTPVariantAlsoNegotiates", + "HTTPVersionNotSupported", + # web_fileresponse + "FileResponse", + # web_middlewares + "middleware", + "normalize_path_middleware", + # web_protocol + "PayloadAccessError", + "RequestHandler", + "RequestPayloadError", + # web_request + "BaseRequest", + "FileField", + "Request", + "RequestKey", + # web_response + "ContentCoding", + "Response", + "StreamResponse", + "json_bytes_response", + "json_response", + "ResponseKey", + # web_routedef + "AbstractRouteDef", + "RouteDef", + "RouteTableDef", + "StaticDef", + "delete", + "get", + "head", + "options", + "patch", + "post", + "put", + "route", + "static", + "view", + # web_runner + "AppRunner", + "BaseRunner", + "BaseSite", + "GracefulExit", + "ServerRunner", + "SockSite", + "TCPSite", + "UnixSite", + "NamedPipeSite", + # web_server + "Server", + # web_urldispatcher + "AbstractResource", + "AbstractRoute", + "DynamicResource", + "PlainResource", + "PrefixedSubAppResource", + "Resource", + "ResourceRoute", + "StaticResource", + "UrlDispatcher", + "UrlMappingMatchInfo", + "View", + # web_ws + "WebSocketReady", + "WebSocketResponse", + "WSMsgType", + # web + "run_app", +) + + +if TYPE_CHECKING: + from ssl import SSLContext +else: + try: + from ssl import SSLContext + except ImportError: # pragma: no cover + SSLContext = object # type: ignore[misc,assignment] + +# Only display warning when using -Wdefault, -We, -X dev or similar. +warnings.filterwarnings("ignore", category=NotAppKeyWarning, append=True) + +HostSequence = TypingIterable[str] + + +async def _run_app( + app: Application | Awaitable[Application], + *, + host: str | HostSequence | None = None, + port: int | None = None, + path: PathLike | TypingIterable[PathLike] | None = None, + sock: socket.socket | TypingIterable[socket.socket] | None = None, + ssl_context: SSLContext | None = None, + print: Callable[..., None] | None = print, + backlog: int = 128, + reuse_address: bool | None = None, + reuse_port: bool | None = None, + **kwargs: Any, # TODO(PY311): Use Unpack +) -> None: + # An internal function to actually do all dirty job for application running + if asyncio.iscoroutine(app): + app = await app + + app = cast(Application, app) + + runner = AppRunner(app, **kwargs) + + await runner.setup() + + sites: list[BaseSite] = [] + + try: + if host is not None: + if isinstance(host, str): + sites.append( + TCPSite( + runner, + host, + port, + ssl_context=ssl_context, + backlog=backlog, + reuse_address=reuse_address, + reuse_port=reuse_port, + ) + ) + else: + for h in host: + sites.append( + TCPSite( + runner, + h, + port, + ssl_context=ssl_context, + backlog=backlog, + reuse_address=reuse_address, + reuse_port=reuse_port, + ) + ) + elif path is None and sock is None or port is not None: + sites.append( + TCPSite( + runner, + port=port, + ssl_context=ssl_context, + backlog=backlog, + reuse_address=reuse_address, + reuse_port=reuse_port, + ) + ) + + if path is not None: + if isinstance(path, (str, os.PathLike)): + sites.append( + UnixSite( + runner, + path, + ssl_context=ssl_context, + backlog=backlog, + ) + ) + else: + for p in path: + sites.append( + UnixSite( + runner, + p, + ssl_context=ssl_context, + backlog=backlog, + ) + ) + + if sock is not None: + if not isinstance(sock, Iterable): + sites.append( + SockSite( + runner, + sock, + ssl_context=ssl_context, + backlog=backlog, + ) + ) + else: + for s in sock: + sites.append( + SockSite( + runner, + s, + ssl_context=ssl_context, + backlog=backlog, + ) + ) + for site in sites: + await site.start() + + if print: # pragma: no branch + names = sorted(str(s.name) for s in runner.sites) + print( + "======== Running on {} ========\n" + "(Press CTRL+C to quit)".format(", ".join(names)) + ) + + # sleep forever by 1 hour intervals, + while True: + await asyncio.sleep(3600) + finally: + await runner.cleanup() + + +def _cancel_tasks( + to_cancel: set["asyncio.Task[Any]"], loop: asyncio.AbstractEventLoop +) -> None: + if not to_cancel: + return + + for task in to_cancel: + task.cancel() + + loop.run_until_complete(asyncio.gather(*to_cancel, return_exceptions=True)) + + for task in to_cancel: + if task.cancelled(): + continue + if task.exception() is not None: + loop.call_exception_handler( + { + "message": "unhandled exception during asyncio.run() shutdown", + "exception": task.exception(), + "task": task, + } + ) + + +def run_app( + app: Application | Awaitable[Application], + *, + host: str | HostSequence | None = None, + port: int | None = None, + path: PathLike | TypingIterable[PathLike] | None = None, + sock: socket.socket | TypingIterable[socket.socket] | None = None, + shutdown_timeout: float = 60.0, + keepalive_timeout: float = 75.0, + ssl_context: SSLContext | None = None, + print: Callable[..., None] | None = print, + backlog: int = 128, + access_log_class: type[AbstractAccessLogger] = AccessLogger, + access_log_format: str = AccessLogger.LOG_FORMAT, + access_log: logging.Logger | None = access_logger, + handle_signals: bool = True, + reuse_address: bool | None = None, + reuse_port: bool | None = None, + handler_cancellation: bool = False, + loop: asyncio.AbstractEventLoop | None = None, + **kwargs: Any, +) -> None: + """Run an app locally""" + if loop is None: + loop = asyncio.new_event_loop() + + # Configure if and only if in debugging mode and using the default logger + if loop.get_debug() and access_log and access_log.name == "aiohttp.access": + if access_log.level == logging.NOTSET: + access_log.setLevel(logging.DEBUG) + if not access_log.hasHandlers(): + access_log.addHandler(logging.StreamHandler()) + + main_task = loop.create_task( + _run_app( + app, + host=host, + port=port, + path=path, + sock=sock, + shutdown_timeout=shutdown_timeout, + keepalive_timeout=keepalive_timeout, + ssl_context=ssl_context, + print=print, + backlog=backlog, + access_log_class=access_log_class, + access_log_format=access_log_format, + access_log=access_log, + handle_signals=handle_signals, + reuse_address=reuse_address, + reuse_port=reuse_port, + handler_cancellation=handler_cancellation, + **kwargs, + ) + ) + + try: + asyncio.set_event_loop(loop) + loop.run_until_complete(main_task) + except (GracefulExit, KeyboardInterrupt): # pragma: no cover + pass + finally: + try: + # Skip when ``main_task`` is already done (e.g. raised during startup). + # Re-running ``loop.run_until_complete`` on a finished task calls + # ``Future.result`` again, which does + # ``raise self._exception.with_traceback(self._exception_tb)`` and + # resets ``exc.__traceback__`` to the originally saved tb — by then + # shallow — clobbering the deep traceback the caller would otherwise + # see (frames from ``cleanup_ctx`` / ``on_startup`` and the user code + # that actually raised). + if not main_task.done(): + main_task.cancel() + with suppress(asyncio.CancelledError): + loop.run_until_complete(main_task) + finally: + _cancel_tasks(asyncio.all_tasks(loop), loop) + loop.run_until_complete(loop.shutdown_asyncgens()) + loop.close() + + +def main(argv: list[str]) -> None: + arg_parser = ArgumentParser( + description="aiohttp.web Application server", prog="aiohttp.web" + ) + arg_parser.add_argument( + "entry_func", + help=( + "Callable returning the `aiohttp.web.Application` instance to " + "run. Should be specified in the 'module:function' syntax." + ), + metavar="entry-func", + ) + arg_parser.add_argument( + "-H", + "--hostname", + help="TCP/IP hostname to serve on (default: localhost)", + default=None, + ) + arg_parser.add_argument( + "-P", + "--port", + help="TCP/IP port to serve on (default: %(default)r)", + type=int, + default=8080, + ) + arg_parser.add_argument( + "-U", + "--path", + help="Unix file system path to serve on. Can be combined with hostname " + "to serve on both Unix and TCP.", + ) + args, extra_argv = arg_parser.parse_known_args(argv) + + # Import logic + mod_str, _, func_str = args.entry_func.partition(":") + if not func_str or not mod_str: + arg_parser.error("'entry-func' not in 'module:function' syntax") + if mod_str.startswith("."): + arg_parser.error("relative module names not supported") + try: + module = import_module(mod_str) + except ImportError as ex: + arg_parser.error(f"unable to import {mod_str}: {ex}") + try: + func = getattr(module, func_str) + except AttributeError: + arg_parser.error(f"module {mod_str!r} has no attribute {func_str!r}") + + # Compatibility logic + if args.path is not None and not hasattr(socket, "AF_UNIX"): + arg_parser.error( + "file system paths not supported by your operating environment" + ) + + logging.basicConfig(level=logging.DEBUG) + + if args.path and args.hostname is None: + host = port = None + else: + host = args.hostname or "localhost" + port = args.port + + app = func(extra_argv) + run_app(app, host=host, port=port, path=args.path) + arg_parser.exit(message="Stopped\n") + + +if __name__ == "__main__": # pragma: no branch + main(sys.argv[1:]) # pragma: no cover diff --git a/py_modules/aiohttp/web_app.py b/py_modules/aiohttp/web_app.py new file mode 100644 index 0000000..dc748af --- /dev/null +++ b/py_modules/aiohttp/web_app.py @@ -0,0 +1,610 @@ +import asyncio +import logging +import warnings +from collections.abc import ( + AsyncIterator, + Awaitable, + Callable, + Iterable, + Iterator, + Mapping, + MutableMapping, + Sequence, +) +from contextlib import AbstractAsyncContextManager, asynccontextmanager +from functools import lru_cache, partial, update_wrapper +from typing import TYPE_CHECKING, Any, Optional, TypeVar, cast, overload + +from aiosignal import Signal +from frozenlist import FrozenList + +from . import hdrs +from .abc import ( + AbstractAccessLogger, + AbstractMatchInfo, + AbstractRouter, + AbstractStreamWriter, +) +from .helpers import DEBUG, AppKey +from .http_parser import RawRequestMessage +from .log import web_logger +from .streams import StreamReader +from .typedefs import Handler, Middleware +from .web_exceptions import NotAppKeyWarning +from .web_log import AccessLogger +from .web_middlewares import _fix_request_current_app +from .web_protocol import RequestHandler +from .web_request import Request +from .web_response import StreamResponse +from .web_routedef import AbstractRouteDef +from .web_server import Server +from .web_urldispatcher import ( + AbstractResource, + AbstractRoute, + Domain, + MaskDomain, + MatchedSubAppResource, + PrefixedSubAppResource, + SystemRoute, + UrlDispatcher, +) + +__all__ = ("Application", "CleanupError") + + +if TYPE_CHECKING: + _AppSignal = Signal["Application"] + _RespPrepareSignal = Signal[Request, StreamResponse] + _Middlewares = FrozenList[Middleware] + _MiddlewaresHandlers = Optional[Sequence[tuple[Middleware, bool]]] + _Subapps = list["Application"] +else: + # No type checker mode, skip types + _AppSignal = Signal + _RespPrepareSignal = Signal + _Middlewares = FrozenList + _MiddlewaresHandlers = Optional[Sequence] + _Subapps = list + +_T = TypeVar("_T") +_U = TypeVar("_U") +_Resource = TypeVar("_Resource", bound=AbstractResource) + + +def _build_middlewares( + handler: Handler, apps: tuple["Application", ...] +) -> Callable[[Request], Awaitable[StreamResponse]]: + """Apply middlewares to handler.""" + for app in apps[::-1]: + for m, _ in app._middlewares_handlers: # type: ignore[union-attr] + handler = update_wrapper(partial(m, handler=handler), handler) + return handler + + +_cached_build_middleware = lru_cache(maxsize=1024)(_build_middlewares) + + +class Application(MutableMapping[str | AppKey[Any], Any]): + ATTRS = frozenset( + [ + "logger", + "_debug", + "_router", + "_loop", + "_handler_args", + "_middlewares", + "_middlewares_handlers", + "_has_legacy_middlewares", + "_run_middlewares", + "_state", + "_frozen", + "_pre_frozen", + "_subapps", + "_on_response_prepare", + "_on_startup", + "_on_shutdown", + "_on_cleanup", + "_client_max_size", + "_cleanup_ctx", + ] + ) + + def __init__( + self, + *, + logger: logging.Logger = web_logger, + router: UrlDispatcher | None = None, + middlewares: Iterable[Middleware] = (), + handler_args: Mapping[str, Any] | None = None, + client_max_size: int = 1024**2, + loop: asyncio.AbstractEventLoop | None = None, + debug: Any = ..., # mypy doesn't support ellipsis + ) -> None: + if router is None: + router = UrlDispatcher() + else: + warnings.warn( + "router argument is deprecated", DeprecationWarning, stacklevel=2 + ) + assert isinstance(router, AbstractRouter), router + + if loop is not None: + warnings.warn( + "loop argument is deprecated", DeprecationWarning, stacklevel=2 + ) + + if debug is not ...: + warnings.warn( + "debug argument is deprecated", DeprecationWarning, stacklevel=2 + ) + self._debug = debug + self._router: UrlDispatcher = router + self._loop = loop + self._handler_args = handler_args + self.logger = logger + + self._middlewares: _Middlewares = FrozenList(middlewares) + + # initialized on freezing + self._middlewares_handlers: _MiddlewaresHandlers = None + # initialized on freezing + self._run_middlewares: bool | None = None + self._has_legacy_middlewares: bool = True + + self._state: dict[AppKey[Any] | str, object] = {} + self._frozen = False + self._pre_frozen = False + self._subapps: _Subapps = [] + + self._on_response_prepare: _RespPrepareSignal = Signal(self) + self._on_startup: _AppSignal = Signal(self) + self._on_shutdown: _AppSignal = Signal(self) + self._on_cleanup: _AppSignal = Signal(self) + self._cleanup_ctx = CleanupContext() + self._on_startup.append(self._cleanup_ctx._on_startup) + self._on_cleanup.append(self._cleanup_ctx._on_cleanup) + self._client_max_size = client_max_size + + def __init_subclass__(cls: type["Application"]) -> None: + warnings.warn( + f"Inheritance class {cls.__name__} from web.Application is discouraged", + DeprecationWarning, + stacklevel=3, + ) + + if DEBUG: # pragma: no cover + + def __setattr__(self, name: str, val: Any) -> None: + if name not in self.ATTRS: + warnings.warn( + f"Setting custom web.Application.{name} attribute " + "is discouraged", + DeprecationWarning, + stacklevel=2, + ) + super().__setattr__(name, val) + + # MutableMapping API + + def __eq__(self, other: object) -> bool: + return self is other + + @overload # type: ignore[override] + def __getitem__(self, key: AppKey[_T]) -> _T: ... + + @overload + def __getitem__(self, key: str) -> Any: ... + + def __getitem__(self, key: str | AppKey[_T]) -> Any: + return self._state[key] + + def _check_frozen(self) -> None: + if self._frozen: + warnings.warn( + "Changing state of started or joined application is deprecated", + DeprecationWarning, + stacklevel=3, + ) + + @overload # type: ignore[override] + def __setitem__(self, key: AppKey[_T], value: _T) -> None: ... + + @overload + def __setitem__(self, key: str, value: Any) -> None: ... + + def __setitem__(self, key: str | AppKey[_T], value: Any) -> None: + self._check_frozen() + if not isinstance(key, AppKey): + warnings.warn( + "It is recommended to use web.AppKey instances for keys.\n" + + "https://docs.aiohttp.org/en/stable/web_advanced.html" + + "#application-s-config", + category=NotAppKeyWarning, + stacklevel=2, + ) + self._state[key] = value + + def __delitem__(self, key: str | AppKey[_T]) -> None: + self._check_frozen() + del self._state[key] + + def __len__(self) -> int: + return len(self._state) + + def __iter__(self) -> Iterator[str | AppKey[Any]]: + return iter(self._state) + + def __hash__(self) -> int: + return id(self) + + @overload # type: ignore[override] + def get(self, key: AppKey[_T], default: None = ...) -> _T | None: ... + + @overload + def get(self, key: AppKey[_T], default: _U) -> _T | _U: ... + + @overload + def get(self, key: str, default: Any = ...) -> Any: ... + + def get(self, key: str | AppKey[_T], default: Any = None) -> Any: + return self._state.get(key, default) + + ######## + @property + def loop(self) -> asyncio.AbstractEventLoop: + # Technically the loop can be None + # but we mask it by explicit type cast + # to provide more convenient type annotation + warnings.warn("loop property is deprecated", DeprecationWarning, stacklevel=2) + return cast(asyncio.AbstractEventLoop, self._loop) + + def _set_loop(self, loop: asyncio.AbstractEventLoop | None) -> None: + if loop is None: + loop = asyncio.get_event_loop() + if self._loop is not None and self._loop is not loop: + raise RuntimeError( + "web.Application instance initialized with different loop" + ) + + self._loop = loop + + # set loop debug + if self._debug is ...: + self._debug = loop.get_debug() + + # set loop to sub applications + for subapp in self._subapps: + subapp._set_loop(loop) + + @property + def pre_frozen(self) -> bool: + return self._pre_frozen + + def pre_freeze(self) -> None: + if self._pre_frozen: + return + + self._pre_frozen = True + self._middlewares.freeze() + self._router.freeze() + self._on_response_prepare.freeze() + self._cleanup_ctx.freeze() + self._on_startup.freeze() + self._on_shutdown.freeze() + self._on_cleanup.freeze() + self._middlewares_handlers = tuple(self._prepare_middleware()) + self._has_legacy_middlewares = any( + not new_style for _, new_style in self._middlewares_handlers + ) + + # If current app and any subapp do not have middlewares avoid run all + # of the code footprint that it implies, which have a middleware + # hardcoded per app that sets up the current_app attribute. If no + # middlewares are configured the handler will receive the proper + # current_app without needing all of this code. + self._run_middlewares = True if self.middlewares else False + + for subapp in self._subapps: + subapp.pre_freeze() + self._run_middlewares = self._run_middlewares or subapp._run_middlewares + + @property + def frozen(self) -> bool: + return self._frozen + + def freeze(self) -> None: + if self._frozen: + return + + self.pre_freeze() + self._frozen = True + for subapp in self._subapps: + subapp.freeze() + + @property + def debug(self) -> bool: + warnings.warn("debug property is deprecated", DeprecationWarning, stacklevel=2) + return self._debug # type: ignore[no-any-return] + + def _reg_subapp_signals(self, subapp: "Application") -> None: + def reg_handler(signame: str) -> None: + subsig = getattr(subapp, signame) + + async def handler(app: "Application") -> None: + await subsig.send(subapp) + + appsig = getattr(self, signame) + appsig.append(handler) + + reg_handler("on_startup") + reg_handler("on_shutdown") + reg_handler("on_cleanup") + + def add_subapp(self, prefix: str, subapp: "Application") -> PrefixedSubAppResource: + if not isinstance(prefix, str): + raise TypeError("Prefix must be str") + prefix = prefix.rstrip("/") + if not prefix: + raise ValueError("Prefix cannot be empty") + factory = partial(PrefixedSubAppResource, prefix, subapp) + return self._add_subapp(factory, subapp) + + def _add_subapp( + self, resource_factory: Callable[[], _Resource], subapp: "Application" + ) -> _Resource: + if self.frozen: + raise RuntimeError("Cannot add sub application to frozen application") + if subapp.frozen: + raise RuntimeError("Cannot add frozen application") + resource = resource_factory() + self.router.register_resource(resource) + self._reg_subapp_signals(subapp) + self._subapps.append(subapp) + subapp.pre_freeze() + if self._loop is not None: + subapp._set_loop(self._loop) + return resource + + def add_domain(self, domain: str, subapp: "Application") -> MatchedSubAppResource: + if not isinstance(domain, str): + raise TypeError("Domain must be str") + elif "*" in domain: + rule: Domain = MaskDomain(domain) + else: + rule = Domain(domain) + factory = partial(MatchedSubAppResource, rule, subapp) + return self._add_subapp(factory, subapp) + + def add_routes(self, routes: Iterable[AbstractRouteDef]) -> list[AbstractRoute]: + return self.router.add_routes(routes) + + @property + def on_response_prepare(self) -> _RespPrepareSignal: + return self._on_response_prepare + + @property + def on_startup(self) -> _AppSignal: + return self._on_startup + + @property + def on_shutdown(self) -> _AppSignal: + return self._on_shutdown + + @property + def on_cleanup(self) -> _AppSignal: + return self._on_cleanup + + @property + def cleanup_ctx(self) -> "CleanupContext": + return self._cleanup_ctx + + @property + def router(self) -> UrlDispatcher: + return self._router + + @property + def middlewares(self) -> _Middlewares: + return self._middlewares + + def _make_handler( + self, + *, + loop: asyncio.AbstractEventLoop | None = None, + access_log_class: type[AbstractAccessLogger] = AccessLogger, + **kwargs: Any, + ) -> Server: + + if not issubclass(access_log_class, AbstractAccessLogger): + raise TypeError( + "access_log_class must be subclass of " + f"aiohttp.abc.AbstractAccessLogger, got {access_log_class}" + ) + + self._set_loop(loop) + self.freeze() + + kwargs["debug"] = self._debug + kwargs["access_log_class"] = access_log_class + if self._handler_args: + for k, v in self._handler_args.items(): + kwargs[k] = v + + return Server( + self._handle, # type: ignore[arg-type] + request_factory=self._make_request, + loop=self._loop, + **kwargs, + ) + + def make_handler( + self, + *, + loop: asyncio.AbstractEventLoop | None = None, + access_log_class: type[AbstractAccessLogger] = AccessLogger, + **kwargs: Any, + ) -> Server: + + warnings.warn( + "Application.make_handler(...) is deprecated, use AppRunner API instead", + DeprecationWarning, + stacklevel=2, + ) + + return self._make_handler( + loop=loop, access_log_class=access_log_class, **kwargs + ) + + async def startup(self) -> None: + """Causes on_startup signal + + Should be called in the event loop along with the request handler. + """ + await self.on_startup.send(self) + + async def shutdown(self) -> None: + """Causes on_shutdown signal + + Should be called before cleanup() + """ + await self.on_shutdown.send(self) + + async def cleanup(self) -> None: + """Causes on_cleanup signal + + Should be called after shutdown() + """ + if self.on_cleanup.frozen: + await self.on_cleanup.send(self) + else: + # If an exception occurs in startup, ensure cleanup contexts are completed. + await self._cleanup_ctx._on_cleanup(self) + + def _make_request( + self, + message: RawRequestMessage, + payload: StreamReader, + protocol: RequestHandler, + writer: AbstractStreamWriter, + task: "asyncio.Task[None]", + _cls: type[Request] = Request, + ) -> Request: + if TYPE_CHECKING: + assert self._loop is not None + return _cls( + message, + payload, + protocol, + writer, + task, + self._loop, + client_max_size=self._client_max_size, + ) + + def _prepare_middleware(self) -> Iterator[tuple[Middleware, bool]]: + for m in reversed(self._middlewares): + if getattr(m, "__middleware_version__", None) == 1: + yield m, True + else: + warnings.warn( + f'old-style middleware "{m!r}" deprecated, see #2252', + DeprecationWarning, + stacklevel=2, + ) + yield m, False + + yield _fix_request_current_app(self), True + + async def _handle(self, request: Request) -> StreamResponse: + loop = asyncio.get_event_loop() + debug = loop.get_debug() + match_info = await self._router.resolve(request) + if debug: # pragma: no cover + if not isinstance(match_info, AbstractMatchInfo): + raise TypeError( + "match_info should be AbstractMatchInfo " + f"instance, not {match_info!r}" + ) + match_info.add_app(self) + + match_info.freeze() + + request._match_info = match_info + + if request.headers.get(hdrs.EXPECT): + resp = await match_info.expect_handler(request) + await request.writer.drain() + if resp is not None: + return resp + + handler = match_info.handler + + if self._run_middlewares: + # If its a SystemRoute, don't cache building the middlewares since + # they are constructed for every MatchInfoError as a new handler + # is made each time. + if not self._has_legacy_middlewares and not isinstance( + match_info.route, SystemRoute + ): + handler = _cached_build_middleware(handler, match_info.apps) + else: + for app in match_info.apps[::-1]: + for m, new_style in app._middlewares_handlers: # type: ignore[union-attr] + if new_style: + handler = update_wrapper( + partial(m, handler=handler), handler + ) + else: + handler = await m(app, handler) # type: ignore[arg-type,assignment] + + return await handler(request) + + def __call__(self) -> "Application": + """gunicorn compatibility""" + return self + + def __repr__(self) -> str: + return f"" + + def __bool__(self) -> bool: + return True + + +class CleanupError(RuntimeError): + @property + def exceptions(self) -> list[BaseException]: + return cast(list[BaseException], self.args[1]) + + +_CleanupContextCallable = ( + Callable[[Application], AbstractAsyncContextManager[None]] + | Callable[[Application], AsyncIterator[None]] +) + + +class CleanupContext(FrozenList[_CleanupContextCallable]): + def __init__(self) -> None: + super().__init__() + self._exits: list[AbstractAsyncContextManager[None]] = [] + + async def _on_startup(self, app: Application) -> None: + for cb in self: + ctx = cb(app) + + if not isinstance(ctx, AbstractAsyncContextManager): + ctx = asynccontextmanager(cb)(app) # type: ignore[arg-type] + + await ctx.__aenter__() + self._exits.append(ctx) + + async def _on_cleanup(self, app: Application) -> None: + errors = [] + for it in reversed(self._exits): + try: + await it.__aexit__(None, None, None) + except (Exception, asyncio.CancelledError) as exc: + errors.append(exc) + if errors: + if len(errors) == 1: + raise errors[0] + else: + raise CleanupError("Multiple errors on cleanup stage", errors) diff --git a/py_modules/aiohttp/web_exceptions.py b/py_modules/aiohttp/web_exceptions.py new file mode 100644 index 0000000..b5b3b30 --- /dev/null +++ b/py_modules/aiohttp/web_exceptions.py @@ -0,0 +1,450 @@ +import warnings +from typing import Any, Dict, Iterable, List, Optional, Set # noqa + +from yarl import URL + +from .typedefs import LooseHeaders, StrOrURL +from .web_response import Response + +__all__ = ( + "HTTPException", + "HTTPError", + "HTTPRedirection", + "HTTPSuccessful", + "HTTPOk", + "HTTPCreated", + "HTTPAccepted", + "HTTPNonAuthoritativeInformation", + "HTTPNoContent", + "HTTPResetContent", + "HTTPPartialContent", + "HTTPMove", + "HTTPMultipleChoices", + "HTTPMovedPermanently", + "HTTPFound", + "HTTPSeeOther", + "HTTPNotModified", + "HTTPUseProxy", + "HTTPTemporaryRedirect", + "HTTPPermanentRedirect", + "HTTPClientError", + "HTTPBadRequest", + "HTTPUnauthorized", + "HTTPPaymentRequired", + "HTTPForbidden", + "HTTPNotFound", + "HTTPMethodNotAllowed", + "HTTPNotAcceptable", + "HTTPProxyAuthenticationRequired", + "HTTPRequestTimeout", + "HTTPConflict", + "HTTPGone", + "HTTPLengthRequired", + "HTTPPreconditionFailed", + "HTTPRequestEntityTooLarge", + "HTTPRequestURITooLong", + "HTTPUnsupportedMediaType", + "HTTPRequestRangeNotSatisfiable", + "HTTPExpectationFailed", + "HTTPMisdirectedRequest", + "HTTPUnprocessableEntity", + "HTTPFailedDependency", + "HTTPUpgradeRequired", + "HTTPPreconditionRequired", + "HTTPTooManyRequests", + "HTTPRequestHeaderFieldsTooLarge", + "HTTPUnavailableForLegalReasons", + "HTTPServerError", + "HTTPInternalServerError", + "HTTPNotImplemented", + "HTTPBadGateway", + "HTTPServiceUnavailable", + "HTTPGatewayTimeout", + "HTTPVersionNotSupported", + "HTTPVariantAlsoNegotiates", + "HTTPInsufficientStorage", + "HTTPNotExtended", + "HTTPNetworkAuthenticationRequired", +) + + +class NotAppKeyWarning(UserWarning): + """Warning when not using AppKey in Application.""" + + +############################################################ +# HTTP Exceptions +############################################################ + + +class HTTPException(Response, Exception): + + # You should set in subclasses: + # status = 200 + + status_code = -1 + empty_body = False + + __http_exception__ = True + + def __init__( + self, + *, + headers: LooseHeaders | None = None, + reason: str | None = None, + body: Any = None, + text: str | None = None, + content_type: str | None = None, + ) -> None: + if body is not None: + warnings.warn( + "body argument is deprecated for http web exceptions", + DeprecationWarning, + ) + if reason is not None and ("\r" in reason or "\n" in reason): + raise ValueError("Reason cannot contain \\r or \\n") + Response.__init__( + self, + status=self.status_code, + headers=headers, + reason=reason, + body=body, + text=text, + content_type=content_type, + ) + Exception.__init__(self, self.reason) + if self.body is None and not self.empty_body: + self.text = f"{self.status}: {self.reason}" + + def __bool__(self) -> bool: + return True + + +class HTTPError(HTTPException): + """Base class for exceptions with status codes in the 400s and 500s.""" + + +class HTTPRedirection(HTTPException): + """Base class for exceptions with status codes in the 300s.""" + + +class HTTPSuccessful(HTTPException): + """Base class for exceptions with status codes in the 200s.""" + + +class HTTPOk(HTTPSuccessful): + status_code = 200 + + +class HTTPCreated(HTTPSuccessful): + status_code = 201 + + +class HTTPAccepted(HTTPSuccessful): + status_code = 202 + + +class HTTPNonAuthoritativeInformation(HTTPSuccessful): + status_code = 203 + + +class HTTPNoContent(HTTPSuccessful): + status_code = 204 + empty_body = True + + +class HTTPResetContent(HTTPSuccessful): + status_code = 205 + empty_body = True + + +class HTTPPartialContent(HTTPSuccessful): + status_code = 206 + + +############################################################ +# 3xx redirection +############################################################ + + +class HTTPMove(HTTPRedirection): + def __init__( + self, + location: StrOrURL, + *, + headers: LooseHeaders | None = None, + reason: str | None = None, + body: Any = None, + text: str | None = None, + content_type: str | None = None, + ) -> None: + if not location: + raise ValueError("HTTP redirects need a location to redirect to.") + super().__init__( + headers=headers, + reason=reason, + body=body, + text=text, + content_type=content_type, + ) + self.headers["Location"] = str(URL(location)) + self.location = location + + +class HTTPMultipleChoices(HTTPMove): + status_code = 300 + + +class HTTPMovedPermanently(HTTPMove): + status_code = 301 + + +class HTTPFound(HTTPMove): + status_code = 302 + + +# This one is safe after a POST (the redirected location will be +# retrieved with GET): +class HTTPSeeOther(HTTPMove): + status_code = 303 + + +class HTTPNotModified(HTTPRedirection): + # FIXME: this should include a date or etag header + status_code = 304 + empty_body = True + + +class HTTPUseProxy(HTTPMove): + # Not a move, but looks a little like one + status_code = 305 + + +class HTTPTemporaryRedirect(HTTPMove): + status_code = 307 + + +class HTTPPermanentRedirect(HTTPMove): + status_code = 308 + + +############################################################ +# 4xx client error +############################################################ + + +class HTTPClientError(HTTPError): + pass + + +class HTTPBadRequest(HTTPClientError): + status_code = 400 + + +class HTTPUnauthorized(HTTPClientError): + status_code = 401 + + +class HTTPPaymentRequired(HTTPClientError): + status_code = 402 + + +class HTTPForbidden(HTTPClientError): + status_code = 403 + + +class HTTPNotFound(HTTPClientError): + status_code = 404 + + +class HTTPMethodNotAllowed(HTTPClientError): + status_code = 405 + + def __init__( + self, + method: str, + allowed_methods: Iterable[str], + *, + headers: LooseHeaders | None = None, + reason: str | None = None, + body: Any = None, + text: str | None = None, + content_type: str | None = None, + ) -> None: + allow = ",".join(sorted(allowed_methods)) + super().__init__( + headers=headers, + reason=reason, + body=body, + text=text, + content_type=content_type, + ) + self.headers["Allow"] = allow + self.allowed_methods: set[str] = set(allowed_methods) + self.method = method.upper() + + +class HTTPNotAcceptable(HTTPClientError): + status_code = 406 + + +class HTTPProxyAuthenticationRequired(HTTPClientError): + status_code = 407 + + +class HTTPRequestTimeout(HTTPClientError): + status_code = 408 + + +class HTTPConflict(HTTPClientError): + status_code = 409 + + +class HTTPGone(HTTPClientError): + status_code = 410 + + +class HTTPLengthRequired(HTTPClientError): + status_code = 411 + + +class HTTPPreconditionFailed(HTTPClientError): + status_code = 412 + + +class HTTPRequestEntityTooLarge(HTTPClientError): + status_code = 413 + + def __init__(self, max_size: float, actual_size: float = 0, **kwargs: Any) -> None: + kwargs.setdefault("text", f"Maximum request body size {max_size} exceeded.") + super().__init__(**kwargs) + + +class HTTPRequestURITooLong(HTTPClientError): + status_code = 414 + + +class HTTPUnsupportedMediaType(HTTPClientError): + status_code = 415 + + +class HTTPRequestRangeNotSatisfiable(HTTPClientError): + status_code = 416 + + +class HTTPExpectationFailed(HTTPClientError): + status_code = 417 + + +class HTTPMisdirectedRequest(HTTPClientError): + status_code = 421 + + +class HTTPUnprocessableEntity(HTTPClientError): + status_code = 422 + + +class HTTPFailedDependency(HTTPClientError): + status_code = 424 + + +class HTTPUpgradeRequired(HTTPClientError): + status_code = 426 + + +class HTTPPreconditionRequired(HTTPClientError): + status_code = 428 + + +class HTTPTooManyRequests(HTTPClientError): + status_code = 429 + + +class HTTPRequestHeaderFieldsTooLarge(HTTPClientError): + status_code = 431 + + +class HTTPUnavailableForLegalReasons(HTTPClientError): + status_code = 451 + + def __init__( + self, + link: StrOrURL | None, + *, + headers: LooseHeaders | None = None, + reason: str | None = None, + body: Any = None, + text: str | None = None, + content_type: str | None = None, + ) -> None: + super().__init__( + headers=headers, + reason=reason, + body=body, + text=text, + content_type=content_type, + ) + self._link = None + if link: + self._link = URL(link) + self.headers["Link"] = f'<{str(self._link)}>; rel="blocked-by"' + + @property + def link(self) -> URL | None: + return self._link + + +############################################################ +# 5xx Server Error +############################################################ +# Response status codes beginning with the digit "5" indicate cases in +# which the server is aware that it has erred or is incapable of +# performing the request. Except when responding to a HEAD request, the +# server SHOULD include an entity containing an explanation of the error +# situation, and whether it is a temporary or permanent condition. User +# agents SHOULD display any included entity to the user. These response +# codes are applicable to any request method. + + +class HTTPServerError(HTTPError): + pass + + +class HTTPInternalServerError(HTTPServerError): + status_code = 500 + + +class HTTPNotImplemented(HTTPServerError): + status_code = 501 + + +class HTTPBadGateway(HTTPServerError): + status_code = 502 + + +class HTTPServiceUnavailable(HTTPServerError): + status_code = 503 + + +class HTTPGatewayTimeout(HTTPServerError): + status_code = 504 + + +class HTTPVersionNotSupported(HTTPServerError): + status_code = 505 + + +class HTTPVariantAlsoNegotiates(HTTPServerError): + status_code = 506 + + +class HTTPInsufficientStorage(HTTPServerError): + status_code = 507 + + +class HTTPNotExtended(HTTPServerError): + status_code = 510 + + +class HTTPNetworkAuthenticationRequired(HTTPServerError): + status_code = 511 diff --git a/py_modules/aiohttp/web_fileresponse.py b/py_modules/aiohttp/web_fileresponse.py new file mode 100644 index 0000000..8aa67ce --- /dev/null +++ b/py_modules/aiohttp/web_fileresponse.py @@ -0,0 +1,419 @@ +import asyncio +import io +import os +import pathlib +import sys +from contextlib import suppress +from enum import Enum, auto +from mimetypes import MimeTypes +from stat import S_ISREG +from types import MappingProxyType +from typing import ( # noqa + IO, + TYPE_CHECKING, + Any, + Awaitable, + Callable, + Final, + Iterator, + List, + Optional, + Set, + Tuple, + Union, + cast, +) + +from . import hdrs +from .abc import AbstractStreamWriter +from .helpers import DEFAULT_CHUNK_SIZE, ETAG_ANY, ETag, must_be_empty_body +from .typedefs import LooseHeaders, PathLike +from .web_exceptions import ( + HTTPForbidden, + HTTPNotFound, + HTTPNotModified, + HTTPPartialContent, + HTTPPreconditionFailed, + HTTPRequestRangeNotSatisfiable, +) +from .web_response import StreamResponse + +__all__ = ("FileResponse",) + +if TYPE_CHECKING: + from .web_request import BaseRequest + + +_T_OnChunkSent = Optional[Callable[[bytes], Awaitable[None]]] + + +NOSENDFILE: Final[bool] = bool(os.environ.get("AIOHTTP_NOSENDFILE")) + +CONTENT_TYPES: Final[MimeTypes] = MimeTypes() + +# File extension to IANA encodings map that will be checked in the order defined. +ENCODING_EXTENSIONS = MappingProxyType( + {ext: CONTENT_TYPES.encodings_map[ext] for ext in (".br", ".gz")} +) + +FALLBACK_CONTENT_TYPE = "application/octet-stream" + +# Provide additional MIME type/extension pairs to be recognized. +# https://en.wikipedia.org/wiki/List_of_archive_formats#Compression_only +ADDITIONAL_CONTENT_TYPES = MappingProxyType( + { + "application/gzip": ".gz", + "application/x-brotli": ".br", + "application/x-bzip2": ".bz2", + "application/x-compress": ".Z", + "application/x-xz": ".xz", + } +) + + +class _FileResponseResult(Enum): + """The result of the file response.""" + + SEND_FILE = auto() # Ie a regular file to send + NOT_ACCEPTABLE = auto() # Ie a socket, or non-regular file + PRE_CONDITION_FAILED = auto() # Ie If-Match or If-None-Match failed + NOT_MODIFIED = auto() # 304 Not Modified + + +# Add custom pairs and clear the encodings map so guess_type ignores them. +CONTENT_TYPES.encodings_map.clear() +for content_type, extension in ADDITIONAL_CONTENT_TYPES.items(): + CONTENT_TYPES.add_type(content_type, extension) + + +_CLOSE_FUTURES: set[asyncio.Future[None]] = set() + + +class FileResponse(StreamResponse): + """A response object can be used to send files.""" + + def __init__( + self, + path: PathLike, + chunk_size: int = DEFAULT_CHUNK_SIZE, + status: int = 200, + reason: str | None = None, + headers: LooseHeaders | None = None, + ) -> None: + super().__init__(status=status, reason=reason, headers=headers) + + self._path = pathlib.Path(path) + self._chunk_size = chunk_size + + def _seek_and_read(self, fobj: IO[Any], offset: int, chunk_size: int) -> bytes: + fobj.seek(offset) + return fobj.read(chunk_size) # type: ignore[no-any-return] + + async def _sendfile_fallback( + self, writer: AbstractStreamWriter, fobj: IO[Any], offset: int, count: int + ) -> AbstractStreamWriter: + # To keep memory usage low,fobj is transferred in chunks + # controlled by the constructor's chunk_size argument. + + chunk_size = self._chunk_size + loop = asyncio.get_event_loop() + chunk = await loop.run_in_executor( + None, self._seek_and_read, fobj, offset, min(chunk_size, count) + ) + while chunk: + await writer.write(chunk) + count = count - len(chunk) + if count <= 0: + break + chunk = await loop.run_in_executor(None, fobj.read, min(chunk_size, count)) + + await writer.drain() + return writer + + async def _sendfile( + self, request: "BaseRequest", fobj: IO[Any], offset: int, count: int + ) -> AbstractStreamWriter: + writer = await super().prepare(request) + assert writer is not None + + if NOSENDFILE or self.compression: + return await self._sendfile_fallback(writer, fobj, offset, count) + + loop = request._loop + transport = request.transport + if transport is None: + raise ConnectionResetError("Connection lost") + + try: + await loop.sendfile(transport, fobj, offset, count) + except NotImplementedError: + return await self._sendfile_fallback(writer, fobj, offset, count) + + await super().write_eof() + return writer + + @staticmethod + def _etag_match(etag_value: str, etags: tuple[ETag, ...], *, weak: bool) -> bool: + if len(etags) == 1 and etags[0].value == ETAG_ANY: + return True + return any( + etag.value == etag_value for etag in etags if weak or not etag.is_weak + ) + + async def _not_modified( + self, request: "BaseRequest", etag_value: str, last_modified: float + ) -> AbstractStreamWriter | None: + self.set_status(HTTPNotModified.status_code) + self._length_check = False + self.etag = etag_value + self.last_modified = last_modified + # Delete any Content-Length headers provided by user. HTTP 304 + # should always have empty response body + return await super().prepare(request) + + async def _precondition_failed( + self, request: "BaseRequest" + ) -> AbstractStreamWriter | None: + self.set_status(HTTPPreconditionFailed.status_code) + self.content_length = 0 + return await super().prepare(request) + + def _make_response( + self, request: "BaseRequest", accept_encoding: str + ) -> tuple[ + _FileResponseResult, io.BufferedReader | None, os.stat_result, str | None + ]: + """Return the response result, io object, stat result, and encoding. + + If an uncompressed file is returned, the encoding is set to + :py:data:`None`. + + This method should be called from a thread executor + since it calls os.stat which may block. + """ + file_path, st, file_encoding = self._get_file_path_stat_encoding( + accept_encoding + ) + if not file_path: + return _FileResponseResult.NOT_ACCEPTABLE, None, st, None + + etag_value = f"{st.st_mtime_ns:x}-{st.st_size:x}" + + # https://www.rfc-editor.org/rfc/rfc9110#section-13.1.1-2 + if (ifmatch := request.if_match) is not None and not self._etag_match( + etag_value, ifmatch, weak=False + ): + return _FileResponseResult.PRE_CONDITION_FAILED, None, st, file_encoding + + if ( + (unmodsince := request.if_unmodified_since) is not None + and ifmatch is None + and st.st_mtime > unmodsince.timestamp() + ): + return _FileResponseResult.PRE_CONDITION_FAILED, None, st, file_encoding + + # https://www.rfc-editor.org/rfc/rfc9110#section-13.1.2-2 + if (ifnonematch := request.if_none_match) is not None and self._etag_match( + etag_value, ifnonematch, weak=True + ): + return _FileResponseResult.NOT_MODIFIED, None, st, file_encoding + + if ( + (modsince := request.if_modified_since) is not None + and ifnonematch is None + and st.st_mtime <= modsince.timestamp() + ): + return _FileResponseResult.NOT_MODIFIED, None, st, file_encoding + + fobj = file_path.open("rb") + with suppress(OSError): + # fstat() may not be available on all platforms + # Once we open the file, we want the fstat() to ensure + # the file has not changed between the first stat() + # and the open(). + st = os.stat(fobj.fileno()) + return _FileResponseResult.SEND_FILE, fobj, st, file_encoding + + def _get_file_path_stat_encoding( + self, accept_encoding: str + ) -> tuple[pathlib.Path | None, os.stat_result, str | None]: + file_path = self._path + for file_extension, file_encoding in ENCODING_EXTENSIONS.items(): + if file_encoding not in accept_encoding: + continue + + compressed_path = file_path.with_suffix(file_path.suffix + file_extension) + with suppress(OSError): + # Do not follow symlinks and ignore any non-regular files. + st = compressed_path.lstat() + if S_ISREG(st.st_mode): + return compressed_path, st, file_encoding + + # Fallback to the uncompressed file + st = file_path.stat() + return file_path if S_ISREG(st.st_mode) else None, st, None + + async def prepare(self, request: "BaseRequest") -> AbstractStreamWriter | None: + loop = asyncio.get_running_loop() + # Encoding comparisons should be case-insensitive + # https://www.rfc-editor.org/rfc/rfc9110#section-8.4.1 + accept_encoding = request.headers.get(hdrs.ACCEPT_ENCODING, "").lower() + try: + response_result, fobj, st, file_encoding = await loop.run_in_executor( + None, self._make_response, request, accept_encoding + ) + except PermissionError: + self.set_status(HTTPForbidden.status_code) + return await super().prepare(request) + except OSError: + # Most likely to be FileNotFoundError or OSError for circular + # symlinks in python >= 3.13, so respond with 404. + self.set_status(HTTPNotFound.status_code) + return await super().prepare(request) + + # Forbid special files like sockets, pipes, devices, etc. + if response_result is _FileResponseResult.NOT_ACCEPTABLE: + self.set_status(HTTPForbidden.status_code) + return await super().prepare(request) + + if response_result is _FileResponseResult.PRE_CONDITION_FAILED: + return await self._precondition_failed(request) + + if response_result is _FileResponseResult.NOT_MODIFIED: + etag_value = f"{st.st_mtime_ns:x}-{st.st_size:x}" + last_modified = st.st_mtime + return await self._not_modified(request, etag_value, last_modified) + + assert fobj is not None + try: + return await self._prepare_open_file(request, fobj, st, file_encoding) + finally: + # We do not await here because we do not want to wait + # for the executor to finish before returning the response + # so the connection can begin servicing another request + # as soon as possible. + close_future = loop.run_in_executor(None, fobj.close) + # Hold a strong reference to the future to prevent it from being + # garbage collected before it completes. + _CLOSE_FUTURES.add(close_future) + close_future.add_done_callback(_CLOSE_FUTURES.remove) + + async def _prepare_open_file( + self, + request: "BaseRequest", + fobj: io.BufferedReader, + st: os.stat_result, + file_encoding: str | None, + ) -> AbstractStreamWriter | None: + status = self._status + file_size: int = st.st_size + file_mtime: float = st.st_mtime + count: int = file_size + start: int | None = None + + if (ifrange := request.if_range) is None or file_mtime <= ifrange.timestamp(): + # If-Range header check: + # condition = cached date >= last modification date + # return 206 if True else 200. + # if False: + # Range header would not be processed, return 200 + # if True but Range header missing + # return 200 + try: + rng = request.http_range + start = rng.start + end: int | None = rng.stop + except ValueError: + # https://tools.ietf.org/html/rfc7233: + # A server generating a 416 (Range Not Satisfiable) response to + # a byte-range request SHOULD send a Content-Range header field + # with an unsatisfied-range value. + # The complete-length in a 416 response indicates the current + # length of the selected representation. + # + # Will do the same below. Many servers ignore this and do not + # send a Content-Range header with HTTP 416 + self._headers[hdrs.CONTENT_RANGE] = f"bytes */{file_size}" + self.set_status(HTTPRequestRangeNotSatisfiable.status_code) + return await super().prepare(request) + + # If a range request has been made, convert start, end slice + # notation into file pointer offset and count + if start is not None: + if start < 0 and end is None: # return tail of file + start += file_size + if start < 0: + # if Range:bytes=-1000 in request header but file size + # is only 200, there would be trouble without this + start = 0 + count = file_size - start + else: + # rfc7233:If the last-byte-pos value is + # absent, or if the value is greater than or equal to + # the current length of the representation data, + # the byte range is interpreted as the remainder + # of the representation (i.e., the server replaces the + # value of last-byte-pos with a value that is one less than + # the current length of the selected representation). + count = ( + min(end if end is not None else file_size, file_size) - start + ) + + if start >= file_size: + # HTTP 416 should be returned in this case. + # + # According to https://tools.ietf.org/html/rfc7233: + # If a valid byte-range-set includes at least one + # byte-range-spec with a first-byte-pos that is less than + # the current length of the representation, or at least one + # suffix-byte-range-spec with a non-zero suffix-length, + # then the byte-range-set is satisfiable. Otherwise, the + # byte-range-set is unsatisfiable. + self._headers[hdrs.CONTENT_RANGE] = f"bytes */{file_size}" + self.set_status(HTTPRequestRangeNotSatisfiable.status_code) + return await super().prepare(request) + + status = HTTPPartialContent.status_code + # Even though you are sending the whole file, you should still + # return a HTTP 206 for a Range request. + self.set_status(status) + + # If the Content-Type header is not already set, guess it based on the + # extension of the request path. The encoding returned by guess_type + # can be ignored since the map was cleared above. + if hdrs.CONTENT_TYPE not in self._headers: + if sys.version_info >= (3, 13): + guesser = CONTENT_TYPES.guess_file_type + else: + guesser = CONTENT_TYPES.guess_type + self.content_type = guesser(self._path)[0] or FALLBACK_CONTENT_TYPE + + if file_encoding: + self._headers[hdrs.CONTENT_ENCODING] = file_encoding + self._headers[hdrs.VARY] = hdrs.ACCEPT_ENCODING + # Disable compression if we are already sending + # a compressed file since we don't want to double + # compress. + self._compression = False + + self.etag = f"{st.st_mtime_ns:x}-{st.st_size:x}" + self.last_modified = file_mtime + self.content_length = count + + self._headers[hdrs.ACCEPT_RANGES] = "bytes" + + if status == HTTPPartialContent.status_code: + real_start = start + assert real_start is not None + self._headers[hdrs.CONTENT_RANGE] = ( + f"bytes {real_start}-{real_start + count - 1}/{file_size}" + ) + + # If we are sending 0 bytes calling sendfile() will throw a ValueError + if count == 0 or must_be_empty_body(request.method, status): + return await super().prepare(request) + + # be aware that start could be None or int=0 here. + offset = start or 0 + + return await self._sendfile(request, fobj, offset, count) diff --git a/py_modules/aiohttp/web_log.py b/py_modules/aiohttp/web_log.py new file mode 100644 index 0000000..9283cc0 --- /dev/null +++ b/py_modules/aiohttp/web_log.py @@ -0,0 +1,231 @@ +import datetime +import functools +import logging +import os +import re +import time as time_mod +from collections.abc import Iterable +from typing import Callable, ClassVar, NamedTuple + +from .abc import AbstractAccessLogger +from .web_request import BaseRequest +from .web_response import StreamResponse + + +class KeyMethod(NamedTuple): + key: str | tuple[str, str] + method: Callable[[BaseRequest, StreamResponse, float], str] + + +class AccessLogger(AbstractAccessLogger): + """Helper object to log access. + + Usage: + log = logging.getLogger("spam") + log_format = "%a %{User-Agent}i" + access_logger = AccessLogger(log, log_format) + access_logger.log(request, response, time) + + Format: + %% The percent sign + %a Remote IP-address (IP-address of proxy if using reverse proxy) + %t Time when the request was started to process + %P The process ID of the child that serviced the request + %r First line of request + %s Response status code + %b Size of response in bytes, including HTTP headers + %T Time taken to serve the request, in seconds + %Tf Time taken to serve the request, in seconds with floating fraction + in .06f format + %D Time taken to serve the request, in microseconds + %{FOO}i request.headers['FOO'] + %{FOO}o response.headers['FOO'] + %{FOO}e os.environ['FOO'] + + """ + + LOG_FORMAT_MAP = { + "a": "remote_address", + "t": "request_start_time", + "P": "process_id", + "r": "first_request_line", + "s": "response_status", + "b": "response_size", + "T": "request_time", + "Tf": "request_time_frac", + "D": "request_time_micro", + "i": "request_header", + "o": "response_header", + } + + LOG_FORMAT = '%a %t "%r" %s %b "%{Referer}i" "%{User-Agent}i"' + FORMAT_RE = re.compile(r"%(\{([A-Za-z0-9\-_]+)\}([ioe])|[atPrsbOD]|Tf?)") + CLEANUP_RE = re.compile(r"(%[^s])") + _FORMAT_CACHE: dict[str, tuple[str, list[KeyMethod]]] = {} + + _cached_tz: ClassVar[datetime.timezone | None] = None + _cached_tz_expires: ClassVar[float] = 0.0 + + def __init__(self, logger: logging.Logger, log_format: str = LOG_FORMAT) -> None: + """Initialise the logger. + + logger is a logger object to be used for logging. + log_format is a string with apache compatible log format description. + + """ + super().__init__(logger, log_format=log_format) + + _compiled_format = AccessLogger._FORMAT_CACHE.get(log_format) + if not _compiled_format: + _compiled_format = self.compile_format(log_format) + AccessLogger._FORMAT_CACHE[log_format] = _compiled_format + + self._log_format, self._methods = _compiled_format + + def compile_format(self, log_format: str) -> tuple[str, list[KeyMethod]]: + """Translate log_format into form usable by modulo formatting + + All known atoms will be replaced with %s + Also methods for formatting of those atoms will be added to + _methods in appropriate order + + For example we have log_format = "%a %t" + This format will be translated to "%s %s" + Also contents of _methods will be + [self._format_a, self._format_t] + These method will be called and results will be passed + to translated string format. + + Each _format_* method receive 'args' which is list of arguments + given to self.log + + Exceptions are _format_e, _format_i and _format_o methods which + also receive key name (by functools.partial) + + """ + # list of (key, method) tuples, we don't use an OrderedDict as users + # can repeat the same key more than once + methods = list() + + for atom in self.FORMAT_RE.findall(log_format): + if atom[1] == "": + format_key1 = self.LOG_FORMAT_MAP[atom[0]] + m = getattr(AccessLogger, "_format_%s" % atom[0]) + key_method = KeyMethod(format_key1, m) + else: + format_key2 = (self.LOG_FORMAT_MAP[atom[2]], atom[1]) + m = getattr(AccessLogger, "_format_%s" % atom[2]) + key_method = KeyMethod(format_key2, functools.partial(m, atom[1])) + + methods.append(key_method) + + log_format = self.FORMAT_RE.sub(r"%s", log_format) + log_format = self.CLEANUP_RE.sub(r"%\1", log_format) + return log_format, methods + + @staticmethod + def _format_i( + key: str, request: BaseRequest, response: StreamResponse, time: float + ) -> str: + if request is None: + return "(no headers)" + + # suboptimal, make istr(key) once + return request.headers.get(key, "-") + + @staticmethod + def _format_o( + key: str, request: BaseRequest, response: StreamResponse, time: float + ) -> str: + # suboptimal, make istr(key) once + return response.headers.get(key, "-") + + @staticmethod + def _format_a(request: BaseRequest, response: StreamResponse, time: float) -> str: + if request is None: + return "-" + ip = request.remote + return ip if ip is not None else "-" + + @classmethod + def _get_local_time(cls) -> datetime.datetime: + if cls._cached_tz is None or time_mod.time() >= cls._cached_tz_expires: + gmtoff = time_mod.localtime().tm_gmtoff + cls._cached_tz = tz = datetime.timezone(datetime.timedelta(seconds=gmtoff)) + + now = datetime.datetime.now(tz) + # Expire at every 30 mins, as any DST change should occur at 0/30 mins past. + d = now + datetime.timedelta(minutes=30) + d = d.replace(minute=30 if d.minute >= 30 else 0, second=0, microsecond=0) + cls._cached_tz_expires = d.timestamp() + return now + + return datetime.datetime.now(cls._cached_tz) + + @staticmethod + def _format_t(request: BaseRequest, response: StreamResponse, time: float) -> str: + now = AccessLogger._get_local_time() + start_time = now - datetime.timedelta(seconds=time) + return start_time.strftime("[%d/%b/%Y:%H:%M:%S %z]") + + @staticmethod + def _format_P(request: BaseRequest, response: StreamResponse, time: float) -> str: + return "<%s>" % os.getpid() + + @staticmethod + def _format_r(request: BaseRequest, response: StreamResponse, time: float) -> str: + if request is None: + return "-" + return f"{request.method} {request.path_qs} HTTP/{request.version.major}.{request.version.minor}" + + @staticmethod + def _format_s(request: BaseRequest, response: StreamResponse, time: float) -> int: + return response.status + + @staticmethod + def _format_b(request: BaseRequest, response: StreamResponse, time: float) -> int: + return response.body_length + + @staticmethod + def _format_T(request: BaseRequest, response: StreamResponse, time: float) -> str: + return str(round(time)) + + @staticmethod + def _format_Tf(request: BaseRequest, response: StreamResponse, time: float) -> str: + return "%06f" % time + + @staticmethod + def _format_D(request: BaseRequest, response: StreamResponse, time: float) -> str: + return str(round(time * 1000000)) + + def _format_line( + self, request: BaseRequest, response: StreamResponse, time: float + ) -> Iterable[tuple[str | tuple[str, str], str]]: + return [(key, method(request, response, time)) for key, method in self._methods] + + @property + def enabled(self) -> bool: + """Check if logger is enabled.""" + # Avoid formatting the log line if it will not be emitted. + return self.logger.isEnabledFor(logging.INFO) + + def log(self, request: BaseRequest, response: StreamResponse, time: float) -> None: + try: + fmt_info = self._format_line(request, response, time) + + values = list() + extra: dict[str, str | dict[str, str]] = dict() + for key, value in fmt_info: + values.append(value) + + if isinstance(key, str): + extra[key] = value + else: + k1, k2 = key + dct: dict[str, str] = extra.get(k1, {}) # type: ignore[assignment] + dct[k2] = value + extra[k1] = dct + + self.logger.info(self._log_format % tuple(values), extra=extra) + except Exception: + self.logger.exception("Error in logging") diff --git a/py_modules/aiohttp/web_middlewares.py b/py_modules/aiohttp/web_middlewares.py new file mode 100644 index 0000000..9816a38 --- /dev/null +++ b/py_modules/aiohttp/web_middlewares.py @@ -0,0 +1,121 @@ +import re +from typing import TYPE_CHECKING, TypeVar + +from .typedefs import Handler, Middleware +from .web_exceptions import HTTPMove, HTTPPermanentRedirect +from .web_request import Request +from .web_response import StreamResponse +from .web_urldispatcher import SystemRoute + +__all__ = ( + "middleware", + "normalize_path_middleware", +) + +if TYPE_CHECKING: + from .web_app import Application + +_Func = TypeVar("_Func") + + +async def _check_request_resolves(request: Request, path: str) -> tuple[bool, Request]: + alt_request = request.clone(rel_url=path) + + match_info = await request.app.router.resolve(alt_request) + alt_request._match_info = match_info + + if match_info.http_exception is None: + return True, alt_request + + return False, request + + +def middleware(f: _Func) -> _Func: + f.__middleware_version__ = 1 # type: ignore[attr-defined] + return f + + +def normalize_path_middleware( + *, + append_slash: bool = True, + remove_slash: bool = False, + merge_slashes: bool = True, + redirect_class: type[HTTPMove] = HTTPPermanentRedirect, +) -> Middleware: + """Factory for producing a middleware that normalizes the path of a request. + + Normalizing means: + - Add or remove a trailing slash to the path. + - Double slashes are replaced by one. + + The middleware returns as soon as it finds a path that resolves + correctly. The order if both merge and append/remove are enabled is + 1) merge slashes + 2) append/remove slash + 3) both merge slashes and append/remove slash. + If the path resolves with at least one of those conditions, it will + redirect to the new path. + + Only one of `append_slash` and `remove_slash` can be enabled. If both + are `True` the factory will raise an assertion error + + If `append_slash` is `True` the middleware will append a slash when + needed. If a resource is defined with trailing slash and the request + comes without it, it will append it automatically. + + If `remove_slash` is `True`, `append_slash` must be `False`. When enabled + the middleware will remove trailing slashes and redirect if the resource + is defined + + If merge_slashes is True, merge multiple consecutive slashes in the + path into one. + """ + correct_configuration = not (append_slash and remove_slash) + assert correct_configuration, "Cannot both remove and append slash" + + @middleware + async def impl(request: Request, handler: Handler) -> StreamResponse: + if isinstance(request.match_info.route, SystemRoute): + paths_to_check = [] + if "?" in request.raw_path: + path, query = request.raw_path.split("?", 1) + query = "?" + query + else: + query = "" + path = request.raw_path + + if merge_slashes: + paths_to_check.append(re.sub("//+", "/", path)) + if append_slash and not request.path.endswith("/"): + paths_to_check.append(path + "/") + if remove_slash and request.path.endswith("/"): + paths_to_check.append(path[:-1]) + if merge_slashes and append_slash: + paths_to_check.append(re.sub("//+", "/", path + "/")) + if merge_slashes and remove_slash: + merged_slashes = re.sub("//+", "/", path) + paths_to_check.append(merged_slashes[:-1]) + + for path in paths_to_check: + path = re.sub("^//+", "/", path) # SECURITY: GHSA-v6wp-4m6f-gcjg + resolves, request = await _check_request_resolves(request, path) + if resolves: + raise redirect_class(request.raw_path + query) + + return await handler(request) + + return impl + + +def _fix_request_current_app(app: "Application") -> Middleware: + @middleware + async def impl(request: Request, handler: Handler) -> StreamResponse: + match_info = request.match_info + prev = match_info.current_app + match_info.current_app = app + try: + return await handler(request) + finally: + match_info.current_app = prev + + return impl diff --git a/py_modules/aiohttp/web_protocol.py b/py_modules/aiohttp/web_protocol.py new file mode 100644 index 0000000..f1b3844 --- /dev/null +++ b/py_modules/aiohttp/web_protocol.py @@ -0,0 +1,874 @@ +import asyncio +import asyncio.streams +import sys +import traceback +import warnings +from collections import deque +from collections.abc import Awaitable, Callable, Sequence +from contextlib import suppress +from html import escape as html_escape +from http import HTTPStatus +from logging import Logger +from typing import TYPE_CHECKING, Any, Optional, cast + +import attr +import yarl +from propcache import under_cached_property + +from .abc import AbstractAccessLogger, AbstractStreamWriter +from .base_protocol import PAUSE_RESUME_READING_ERRORS, BaseProtocol +from .helpers import DEFAULT_CHUNK_SIZE, ceil_timeout +from .http import ( + HttpProcessingError, + HttpRequestParser, + HttpVersion10, + RawRequestMessage, + StreamWriter, + WebSocketReader, +) +from .http_exceptions import BadHttpMethod +from .log import access_logger, server_logger +from .streams import EMPTY_PAYLOAD, StreamReader +from .tcp_helpers import tcp_keepalive +from .web_exceptions import HTTPException, HTTPInternalServerError +from .web_log import AccessLogger +from .web_request import BaseRequest +from .web_response import Response, StreamResponse + +__all__ = ("RequestHandler", "RequestPayloadError", "PayloadAccessError") + +# Max parsed-but-unhandled pipelined requests buffered per connection before +# reading is paused. Bounds memory a client can pin by keeping one handler busy +# and pipelining behind it; reading resumes as the queue drains. +MAX_MSG_QUEUE_SIZE = 32 + +if TYPE_CHECKING: + import ssl + + from .web_server import Server + + +_RequestFactory = Callable[ + [ + RawRequestMessage, + StreamReader, + "RequestHandler", + AbstractStreamWriter, + "asyncio.Task[None]", + ], + BaseRequest, +] + +_RequestHandler = Callable[[BaseRequest], Awaitable[StreamResponse]] + +ERROR = RawRequestMessage( + "UNKNOWN", + "/", + HttpVersion10, + {}, # type: ignore[arg-type] + {}, # type: ignore[arg-type] + True, + None, + False, + False, + yarl.URL("/"), +) + + +class RequestPayloadError(Exception): + """Payload parsing error.""" + + +class PayloadAccessError(Exception): + """Payload was accessed after response was sent.""" + + +_PAYLOAD_ACCESS_ERROR = PayloadAccessError() + + +@attr.s(auto_attribs=True, frozen=True, slots=True) +class _ErrInfo: + status: int + exc: BaseException + message: str + + +_MsgType = tuple[RawRequestMessage | _ErrInfo, StreamReader] + + +class RequestHandler(BaseProtocol): + """HTTP protocol implementation. + + RequestHandler handles incoming HTTP request. It reads request line, + request headers and request payload and calls handle_request() method. + By default it always returns with 404 response. + + RequestHandler handles errors in incoming request, like bad + status line, bad headers or incomplete payload. If any error occurs, + connection gets closed. + + keepalive_timeout -- number of seconds before closing + keep-alive connection + + tcp_keepalive -- TCP keep-alive is on, default is on + + debug -- enable debug mode + + logger -- custom logger object + + access_log_class -- custom class for access_logger + + access_log -- custom logging object + + access_log_format -- access log format string + + loop -- Optional event loop + + max_line_size -- Optional maximum header line size + + max_field_size -- Optional maximum header field size + + max_headers -- Optional maximum header size + + timeout_ceil_threshold -- Optional value to specify + threshold to ceil() timeout + values + + """ + + __slots__ = ( + "max_field_size", + "max_headers", + "max_line_size", + "_request_count", + "_keepalive", + "_manager", + "_request_handler", + "_request_factory", + "_tcp_keepalive", + "_next_keepalive_close_time", + "_keepalive_handle", + "_keepalive_timeout", + "_lingering_time", + "_messages", + "_max_msg_queue_size", + "_msg_queue_resume_size", + "_msg_queue_paused", + "_message_tail", + "_handler_waiter", + "_waiter", + "_task_handler", + "_payload_parser", + "_data_received_cb", + "logger", + "debug", + "access_log", + "access_logger", + "_close", + "_force_close", + "_current_request", + "_timeout_ceil_threshold", + "_request_in_progress", + "_logging_enabled", + "_cache", + ) + + def __init__( + self, + manager: "Server", + *, + loop: asyncio.AbstractEventLoop, + # Default should be high enough that it's likely longer than a reverse proxy. + keepalive_timeout: float = 3630, + tcp_keepalive: bool = True, + logger: Logger = server_logger, + access_log_class: type[AbstractAccessLogger] = AccessLogger, + access_log: Logger = access_logger, + access_log_format: str = AccessLogger.LOG_FORMAT, + debug: bool = False, + max_line_size: int = 8190, + max_headers: int = 128, + max_field_size: int = 8190, + lingering_time: float = 10.0, + read_bufsize: int = DEFAULT_CHUNK_SIZE, + auto_decompress: bool = True, + timeout_ceil_threshold: float = 5, + ): + self._max_msg_queue_size = MAX_MSG_QUEUE_SIZE + # Low-water mark: resume reading once the queue drains to half the limit + # so we refill in batches instead of churning pause/resume per request. + self._msg_queue_resume_size = MAX_MSG_QUEUE_SIZE // 2 + # Set before super().__init__ so _reading_paused_for_msg_queue() is safe + # if BaseProtocol ever triggers a resume during init. + self._msg_queue_paused = False + parser = HttpRequestParser( + self, + loop, + read_bufsize, + max_line_size=max_line_size, + max_field_size=max_field_size, + max_headers=max_headers, + payload_exception=RequestPayloadError, + auto_decompress=auto_decompress, + max_msg_queue_size=MAX_MSG_QUEUE_SIZE, + ) + super().__init__(loop, parser) + + # _request_count is the number of requests processed with the same connection. + self._request_count = 0 + self._keepalive = False + self._current_request: BaseRequest | None = None + self._manager: Server | None = manager + self._request_handler: _RequestHandler | None = manager.request_handler + self._request_factory: _RequestFactory | None = manager.request_factory + + self.max_line_size = max_line_size + self.max_headers = max_headers + self.max_field_size = max_field_size + + self._tcp_keepalive = tcp_keepalive + # placeholder to be replaced on keepalive timeout setup + self._next_keepalive_close_time = 0.0 + self._keepalive_handle: asyncio.Handle | None = None + self._keepalive_timeout = keepalive_timeout + self._lingering_time = float(lingering_time) + + self._messages: deque[_MsgType] = deque() + self._message_tail = b"" + self._data_received_cb: Callable[[], None] | None = None + + self._waiter: asyncio.Future[None] | None = None + self._handler_waiter: asyncio.Future[None] | None = None + self._task_handler: asyncio.Task[None] | None = None + self._payload_parser: Any = None + + self._timeout_ceil_threshold: float = 5 + try: + self._timeout_ceil_threshold = float(timeout_ceil_threshold) + except (TypeError, ValueError): + pass + + self.logger = logger + self.debug = debug + self.access_log = access_log + if access_log: + self.access_logger: AbstractAccessLogger | None = access_log_class( + access_log, access_log_format + ) + self._logging_enabled = self.access_logger.enabled + else: + self.access_logger = None + self._logging_enabled = False + + self._close = False + self._force_close = False + self._request_in_progress = False + self._cache: dict[str, Any] = {} + + def __repr__(self) -> str: + return "<{} {}>".format( + self.__class__.__name__, + "connected" if self.transport is not None else "disconnected", + ) + + @under_cached_property + def ssl_context(self) -> Optional["ssl.SSLContext"]: + """Return SSLContext if available.""" + return ( + None + if self.transport is None + else self.transport.get_extra_info("sslcontext") + ) + + @under_cached_property + def peername( + self, + ) -> str | tuple[str, int, int, int] | tuple[str, int] | None: + """Return peername if available.""" + return ( + None + if self.transport is None + else self.transport.get_extra_info("peername") + ) + + @under_cached_property + def sockname( + self, + ) -> str | tuple[str, int, int, int] | tuple[str, int] | None: + """Return sockname if available.""" + return ( + None + if self.transport is None + else self.transport.get_extra_info("sockname") + ) + + @property + def keepalive_timeout(self) -> float: + return self._keepalive_timeout + + async def shutdown(self, timeout: float | None = 15.0) -> None: + """Do worker process exit preparations. + + We need to clean up everything and stop accepting requests. + It is especially important for keep-alive connections. + """ + self._force_close = True + + if self._keepalive_handle is not None: + self._keepalive_handle.cancel() + + # Wait for graceful handler completion + if self._request_in_progress: + # The future is only created when we are shutting + # down while the handler is still processing a request + # to avoid creating a future for every request. + self._handler_waiter = self._loop.create_future() + try: + async with ceil_timeout(timeout): + await self._handler_waiter + except (asyncio.CancelledError, asyncio.TimeoutError): + self._handler_waiter = None + if ( + sys.version_info >= (3, 11) + and (task := asyncio.current_task()) + and task.cancelling() + ): + raise + # Then cancel handler and wait + try: + async with ceil_timeout(timeout): + if self._current_request is not None: + self._current_request._cancel(asyncio.CancelledError()) + + if self._task_handler is not None and not self._task_handler.done(): + await asyncio.shield(self._task_handler) + except (asyncio.CancelledError, asyncio.TimeoutError): + if ( + sys.version_info >= (3, 11) + and (task := asyncio.current_task()) + and task.cancelling() + ): + raise + + # force-close non-idle handler + if self._task_handler is not None: + self._task_handler.cancel() + + self.force_close() + + def connection_made(self, transport: asyncio.BaseTransport) -> None: + super().connection_made(transport) + + real_transport = cast(asyncio.Transport, transport) + if self._tcp_keepalive: + tcp_keepalive(real_transport) + + assert self._manager is not None + self._manager.connection_made(self, real_transport) + + loop = self._loop + if sys.version_info >= (3, 12): + task = asyncio.Task(self.start(), loop=loop, eager_start=True) + else: + task = loop.create_task(self.start()) + self._task_handler = task + + def connection_lost(self, exc: BaseException | None) -> None: + if self._manager is None: + return + self._manager.connection_lost(self, exc) + + # Grab value before setting _manager to None. + handler_cancellation = self._manager.handler_cancellation + + self.force_close() + super().connection_lost(exc) + self._manager = None + self._request_factory = None + self._request_handler = None + self._parser = None + + if self._keepalive_handle is not None: + self._keepalive_handle.cancel() + + if self._current_request is not None: + if exc is None: + exc = ConnectionResetError("Connection lost") + self._current_request._cancel(exc) + + if handler_cancellation and self._task_handler is not None: + self._task_handler.cancel() + + self._task_handler = None + + if self._payload_parser is not None: + self._payload_parser.feed_eof() + self._payload_parser = None + + def set_parser( + self, + parser: WebSocketReader, + data_received_cb: Callable[[], None] | None = None, + ) -> None: + assert self._payload_parser is None + + self._payload_parser = parser + self._data_received_cb = data_received_cb + + if self._message_tail: + self._payload_parser.feed_data(self._message_tail) + self._message_tail = b"" + + def eof_received(self) -> None: + pass + + def data_received(self, data: bytes) -> None: + if self._force_close or self._close: + return + # parse http messages + messages: Sequence[_MsgType] + if self._payload_parser is None and not self._upgraded: + assert self._parser is not None + try: + messages, upgraded, tail = self._parser.feed_data(data) + except HttpProcessingError as exc: + messages = [ + (_ErrInfo(status=400, exc=exc, message=exc.message), EMPTY_PAYLOAD) + ] + upgraded = False + tail = b"" + + for msg, payload in messages: + self._request_count += 1 + self._messages.append((msg, payload)) + + waiter = self._waiter + if messages and waiter is not None and not waiter.done(): + # don't set result twice + waiter.set_result(None) + + # Queue full: pause the transport (the parser already stopped + # emitting). start() resumes as it drains the queue. + if ( + not self._msg_queue_paused + and len(self._messages) >= self._max_msg_queue_size + ): + self._pause_msg_queue_reading() + + self._upgraded = upgraded + if upgraded and tail: + self._message_tail = tail + + # no parser, just store + elif self._payload_parser is None and self._upgraded and data: + self._message_tail += data + + # feed payload + elif data: + if self._data_received_cb is not None: + self._data_received_cb() + eof, tail = self._payload_parser.feed_data(data) + if eof: + self.close() + + def _reading_paused_for_msg_queue(self) -> bool: + return self._msg_queue_paused + + def _pause_msg_queue_reading(self) -> None: + self._msg_queue_paused = True + if self.transport is not None: + try: + self.transport.pause_reading() + except PAUSE_RESUME_READING_ERRORS: + # Transport lacks flow control; nothing to pause. Intentionally + # ignored (see PAUSE_RESUME_READING_ERRORS; do not use suppress). + pass + + def _resume_msg_queue_reading(self) -> None: + if not self._upgraded: + # Reparse buffered pipelined requests while still marked paused so + # a refill past the limit does not re-pause an already-paused + # transport; only resume below once it stayed under the limit. + self.data_received(b"") + if len(self._messages) >= self._max_msg_queue_size: + return + self._msg_queue_paused = False + if not self._reading_paused and self.transport is not None: + try: + self.transport.resume_reading() + except PAUSE_RESUME_READING_ERRORS: + # Transport lacks flow control; nothing to resume. Intentionally + # ignored (see PAUSE_RESUME_READING_ERRORS; do not use suppress). + pass + + def keep_alive(self, val: bool) -> None: + """Set keep-alive connection mode. + + :param bool val: new state. + """ + self._keepalive = val + if self._keepalive_handle: + self._keepalive_handle.cancel() + self._keepalive_handle = None + + def close(self) -> None: + """Close connection. + + Stop accepting new pipelining messages and close + connection when handlers done processing messages. + """ + self._close = True + if self._waiter: + self._waiter.cancel() + + def force_close(self) -> None: + """Forcefully close connection.""" + self._force_close = True + if self._waiter: + self._waiter.cancel() + if self.transport is not None: + self.transport.close() + self.transport = None + + def log_access( + self, request: BaseRequest, response: StreamResponse, time: float | None + ) -> None: + if self._logging_enabled and self.access_logger is not None: + if TYPE_CHECKING: + assert time is not None + self.access_logger.log(request, response, self._loop.time() - time) + + def log_debug(self, *args: Any, **kw: Any) -> None: + if self.debug: + self.logger.debug(*args, **kw) + + def log_exception(self, *args: Any, **kw: Any) -> None: + self.logger.exception(*args, **kw) + + def _process_keepalive(self) -> None: + self._keepalive_handle = None + if self._force_close or not self._keepalive: + return + + loop = self._loop + now = loop.time() + close_time = self._next_keepalive_close_time + if now < close_time: + # Keep alive close check fired too early, reschedule + self._keepalive_handle = loop.call_at(close_time, self._process_keepalive) + return + + # handler in idle state + if self._waiter and not self._waiter.done(): + self.force_close() + + async def _handle_request( + self, + request: BaseRequest, + start_time: float | None, + request_handler: Callable[[BaseRequest], Awaitable[StreamResponse]], + ) -> tuple[StreamResponse, bool]: + self._request_in_progress = True + try: + try: + self._current_request = request + resp = await request_handler(request) + finally: + self._current_request = None + except HTTPException as exc: + resp = exc + resp, reset = await self.finish_response(request, resp, start_time) + except asyncio.CancelledError: + raise + except asyncio.TimeoutError as exc: + self.log_debug("Request handler timed out.", exc_info=exc) + resp = self.handle_error(request, 504) + resp, reset = await self.finish_response(request, resp, start_time) + except Exception as exc: + resp = self.handle_error(request, 500, exc) + resp, reset = await self.finish_response(request, resp, start_time) + else: + # Deprecation warning (See #2415) + if getattr(resp, "__http_exception__", False): + warnings.warn( + "returning HTTPException object is deprecated " + "(#2415) and will be removed, " + "please raise the exception instead", + DeprecationWarning, + ) + + resp, reset = await self.finish_response(request, resp, start_time) + finally: + self._request_in_progress = False + if self._handler_waiter is not None: + self._handler_waiter.set_result(None) + + return resp, reset + + async def start(self) -> None: + """Process incoming request. + + It reads request line, request headers and request payload, then + calls handle_request() method. Subclass has to override + handle_request(). start() handles various exceptions in request + or response handling. Connection is being closed always unless + keep_alive(True) specified. + """ + loop = self._loop + manager = self._manager + assert manager is not None + keepalive_timeout = self._keepalive_timeout + resp = None + assert self._request_factory is not None + assert self._request_handler is not None + + while not self._force_close: + if not self._messages: + try: + # wait for next request + self._waiter = loop.create_future() + await self._waiter + finally: + self._waiter = None + + message, payload = self._messages.popleft() + + # Free a parser slot; resume reading once drained to low water so + # pipelining keeps flowing while this request is handled. + # no branch: _parser is only None after connection_lost, whose path + # exits this loop, so the None case is not reachably exercisable. + if self._parser is not None: # pragma: no branch + self._parser.message_consumed() + if ( + self._msg_queue_paused + and len(self._messages) <= self._msg_queue_resume_size + ): + self._resume_msg_queue_reading() + + # time is only fetched if logging is enabled as otherwise + # its thrown away and never used. + start = loop.time() if self._logging_enabled else None + + manager.requests_count += 1 + writer = StreamWriter(self, loop) + if isinstance(message, _ErrInfo): + # make request_factory work + request_handler = self._make_error_handler(message) + message = ERROR + else: + request_handler = self._request_handler + + # Important don't hold a reference to the current task + # as on traceback it will prevent the task from being + # collected and will cause a memory leak. + request = self._request_factory( + message, + payload, + self, + writer, + self._task_handler or asyncio.current_task(loop), # type: ignore[arg-type] + ) + try: + # a new task is used for copy context vars (#3406) + coro = self._handle_request(request, start, request_handler) + if sys.version_info >= (3, 12): + task = asyncio.Task(coro, loop=loop, eager_start=True) + else: + task = loop.create_task(coro) + try: + resp, reset = await task + except ConnectionError: + self.log_debug("Ignored premature client disconnection") + break + + # Drop the processed task from asyncio.Task.all_tasks() early + del task + if reset: + self.log_debug("Ignored premature client disconnection 2") + break + + # notify server about keep-alive + self._keepalive = bool(resp.keep_alive) + + # check payload + if not payload.is_eof(): + lingering_time = self._lingering_time + if not self._force_close and lingering_time: + self.log_debug( + "Start lingering close timer for %s sec.", lingering_time + ) + + now = loop.time() + end_t = now + lingering_time + + try: + while not payload.is_eof() and now < end_t: + async with ceil_timeout(end_t - now): + # read and ignore + await payload.readany() + now = loop.time() + except (asyncio.CancelledError, asyncio.TimeoutError): + if ( + sys.version_info >= (3, 11) + and (t := asyncio.current_task()) + and t.cancelling() + ): + raise + + # if payload still uncompleted + if not payload.is_eof() and not self._force_close: + self.log_debug("Uncompleted request.") + self.close() + + payload.set_exception(_PAYLOAD_ACCESS_ERROR) + + except asyncio.CancelledError: + self.log_debug("Ignored premature client disconnection") + self.force_close() + raise + except Exception as exc: + self.log_exception("Unhandled exception", exc_info=exc) + self.force_close() + except BaseException: + self.force_close() + raise + finally: + request._task = None # type: ignore[assignment] # Break reference cycle in case of exception + if self.transport is None and resp is not None: + self.log_debug("Ignored premature client disconnection.") + + if self._keepalive and not self._close and not self._force_close: + # start keep-alive timer + close_time = loop.time() + keepalive_timeout + self._next_keepalive_close_time = close_time + if self._keepalive_handle is None: + self._keepalive_handle = loop.call_at( + close_time, self._process_keepalive + ) + else: + break + + # remove handler, close transport if no handlers left + if not self._force_close: + self._task_handler = None + if self.transport is not None: + self.transport.close() + + async def finish_response( + self, request: BaseRequest, resp: StreamResponse, start_time: float | None + ) -> tuple[StreamResponse, bool]: + """Prepare the response and write_eof, then log access. + + This has to + be called within the context of any exception so the access logger + can get exception information. Returns True if the client disconnects + prematurely. + """ + request._finish() + if self._parser is not None: + self._parser.set_upgraded(False) + self._upgraded = False + if self._message_tail: + messages, _upgraded, tail = self._parser.feed_data(self._message_tail) + self._message_tail = tail + for msg, payload in messages: + self._request_count += 1 + self._messages.append((msg, payload)) + # This shouldn't be possible. If a future refactor results in this + # failing, then the code may need to be updated to set the waiter. + assert self._waiter is None + try: + prepare_meth = resp.prepare + except AttributeError: + if resp is None: + self.log_exception("Missing return statement on request handler") + else: + self.log_exception( + f"Web-handler should return a response instance, got {resp!r}" + ) + exc = HTTPInternalServerError() + resp = Response( + status=exc.status, reason=exc.reason, text=exc.text, headers=exc.headers + ) + prepare_meth = resp.prepare + try: + await prepare_meth(request) + await resp.write_eof() + except ConnectionError: + self.log_access(request, resp, start_time) + return resp, True + + self.log_access(request, resp, start_time) + return resp, False + + def handle_error( + self, + request: BaseRequest, + status: int = 500, + exc: BaseException | None = None, + message: str | None = None, + ) -> StreamResponse: + """Handle errors. + + Returns HTTP response with specific status code. Logs additional + information. It always closes current connection. + """ + if self._request_count == 1 and isinstance(exc, BadHttpMethod): + # BadHttpMethod is common when a client sends non-HTTP + # or encrypted traffic to an HTTP port. This is expected + # to happen when connected to the public internet so we log + # it at the debug level as to not fill logs with noise. + self.logger.debug( + "Error handling request from %s", request.remote, exc_info=exc + ) + else: + self.log_exception( + "Error handling request from %s", request.remote, exc_info=exc + ) + + # some data already got sent, connection is broken + if request.writer.output_size > 0: + raise ConnectionError( + "Response is sent already, cannot send another response " + "with the error message" + ) + + ct = "text/plain" + if status == HTTPStatus.INTERNAL_SERVER_ERROR: + title = f"{HTTPStatus.INTERNAL_SERVER_ERROR.value} {HTTPStatus.INTERNAL_SERVER_ERROR.phrase}" + msg = HTTPStatus.INTERNAL_SERVER_ERROR.description + tb = None + if self.debug: + with suppress(Exception): + tb = traceback.format_exc() + + if "text/html" in request.headers.get("Accept", ""): + if tb: + tb = html_escape(tb) + msg = f"

Traceback:

\n
{tb}
" + message = ( + "" + f"{title}" + f"\n

{title}

" + f"\n{msg}\n\n" + ) + ct = "text/html" + else: + if tb: + msg = tb + message = title + "\n\n" + msg + + resp = Response(status=status, text=message, content_type=ct) + resp.force_close() + + return resp + + def _make_error_handler( + self, err_info: _ErrInfo + ) -> Callable[[BaseRequest], Awaitable[StreamResponse]]: + async def handler(request: BaseRequest) -> StreamResponse: + return self.handle_error( + request, err_info.status, err_info.exc, err_info.message + ) + + return handler diff --git a/py_modules/aiohttp/web_request.py b/py_modules/aiohttp/web_request.py new file mode 100644 index 0000000..ca925e2 --- /dev/null +++ b/py_modules/aiohttp/web_request.py @@ -0,0 +1,949 @@ +import asyncio +import datetime +import io +import re +import string +import tempfile +import types +import warnings +from collections.abc import Iterator, Mapping, MutableMapping +from re import Pattern +from types import MappingProxyType +from typing import TYPE_CHECKING, Any, Final, Optional, TypeVar, cast, overload +from urllib.parse import parse_qsl + +import attr +from multidict import ( + CIMultiDict, + CIMultiDictProxy, + MultiDict, + MultiDictProxy, + MultiMapping, +) +from yarl import URL + +from . import hdrs +from ._cookie_helpers import parse_cookie_header +from .abc import AbstractStreamWriter +from .helpers import ( + _SENTINEL, + DEBUG, + DEFAULT_CHUNK_SIZE, + ETAG_ANY, + LIST_QUOTED_ETAG_RE, + ChainMapProxy, + ETag, + HeadersMixin, + RequestKey, + parse_http_date, + reify, + sentinel, + set_exception, +) +from .http_parser import RawRequestMessage +from .http_writer import HttpVersion +from .multipart import BodyPartReader, MultipartReader +from .streams import EmptyStreamReader, StreamReader +from .typedefs import ( + DEFAULT_JSON_DECODER, + JSONDecoder, + LooseHeaders, + RawHeaders, + StrOrURL, +) +from .web_exceptions import HTTPRequestEntityTooLarge, NotAppKeyWarning +from .web_response import StreamResponse + +__all__ = ("BaseRequest", "FileField", "Request") + + +if TYPE_CHECKING: + from .web_app import Application + from .web_protocol import RequestHandler + from .web_urldispatcher import UrlMappingMatchInfo + + +_T = TypeVar("_T") + + +@attr.s(auto_attribs=True, frozen=True, slots=True) +class FileField: + name: str + filename: str + file: io.BufferedReader + content_type: str + headers: CIMultiDictProxy[str] + + +_Post = str | bytes | bytearray | FileField +_TCHAR: Final[str] = string.digits + string.ascii_letters + r"!#$%&'*+.^_`|~-" +# '-' at the end to prevent interpretation as range in a char class + +_TOKEN: Final[str] = rf"[{_TCHAR}]+" + +_QDTEXT: Final[str] = r"[{}]".format( + r"".join(chr(c) for c in (0x09, 0x20, 0x21) + tuple(range(0x23, 0x7F))) +) +# qdtext includes 0x5C to escape 0x5D ('\]') +# qdtext excludes obs-text (because obsoleted, and encoding not specified) + +_QUOTED_PAIR: Final[str] = r"\\[\t !-~]" + +_QUOTED_STRING: Final[str] = rf'"(?:{_QUOTED_PAIR}|{_QDTEXT})*"' + +# This does not have a ReDOS/performance concern as long as it used with re.match(). +_FORWARDED_PAIR: Final[str] = rf"({_TOKEN})=({_TOKEN}|{_QUOTED_STRING})(:\d{{1,4}})?" + +_QUOTED_PAIR_REPLACE_RE: Final[Pattern[str]] = re.compile(r"\\([\t !-~])") +# same pattern as _QUOTED_PAIR but contains a capture group + +_FORWARDED_PAIR_RE: Final[Pattern[str]] = re.compile(_FORWARDED_PAIR) + +############################################################ +# HTTP Request +############################################################ + + +class BaseRequest(MutableMapping[str | RequestKey[Any], Any], HeadersMixin): + POST_METHODS = { + hdrs.METH_PATCH, + hdrs.METH_POST, + hdrs.METH_PUT, + hdrs.METH_TRACE, + hdrs.METH_DELETE, + } + + ATTRS = HeadersMixin.ATTRS | frozenset( + [ + "_message", + "_protocol", + "_payload_writer", + "_payload", + "_headers", + "_method", + "_version", + "_rel_url", + "_post", + "_read_bytes", + "_state", + "_cache", + "_task", + "_client_max_size", + "_loop", + "_transport_sslcontext", + "_transport_peername", + "_transport_sockname", + ] + ) + _post: MultiDictProxy[_Post] | None = None + _read_bytes: bytes | None = None + _seen_str_keys: set[str] = set() + + def __init__( + self, + message: RawRequestMessage, + payload: StreamReader, + protocol: "RequestHandler", + payload_writer: AbstractStreamWriter, + task: "asyncio.Task[None]", + loop: asyncio.AbstractEventLoop, + *, + client_max_size: int = 1024**2, + state: dict[RequestKey[Any] | str, Any] | None = None, + scheme: str | None = None, + host: str | None = None, + remote: str | None = None, + ) -> None: + self._message = message + self._protocol = protocol + self._payload_writer = payload_writer + + self._payload = payload + self._headers: CIMultiDictProxy[str] = message.headers + self._method = message.method + self._version = message.version + self._cache: dict[str, Any] = {} + url = message.url + if url.absolute: + if scheme is not None: + url = url.with_scheme(scheme) + if host is not None: + url = url.with_host(host) + # absolute URL is given, + # override auto-calculating url, host, and scheme + # all other properties should be good + self._cache["url"] = url + self._cache["host"] = url.host + self._cache["scheme"] = url.scheme + self._rel_url = url.relative() + else: + self._rel_url = url + if scheme is not None: + self._cache["scheme"] = scheme + if host is not None: + self._cache["host"] = host + + self._state = {} if state is None else state + self._task = task + self._client_max_size = client_max_size + self._loop = loop + + self._transport_sslcontext = protocol.ssl_context + self._transport_peername = protocol.peername + self._transport_sockname = protocol.sockname + + if remote is not None: + self._cache["remote"] = remote + + def clone( + self, + *, + method: str | _SENTINEL = sentinel, + rel_url: StrOrURL | _SENTINEL = sentinel, + headers: LooseHeaders | _SENTINEL = sentinel, + scheme: str | _SENTINEL = sentinel, + host: str | _SENTINEL = sentinel, + remote: str | _SENTINEL = sentinel, + client_max_size: int | _SENTINEL = sentinel, + ) -> "BaseRequest": + """Clone itself with replacement some attributes. + + Creates and returns a new instance of Request object. If no parameters + are given, an exact copy is returned. If a parameter is not passed, it + will reuse the one from the current request object. + """ + if self._read_bytes: + raise RuntimeError("Cannot clone request after reading its content") + + dct: dict[str, Any] = {} + if method is not sentinel: + dct["method"] = method + if rel_url is not sentinel: + new_url: URL = URL(rel_url) + dct["url"] = new_url + dct["path"] = str(new_url) + if headers is not sentinel: + # a copy semantic + dct["headers"] = CIMultiDictProxy(CIMultiDict(headers)) + dct["raw_headers"] = tuple( + (k.encode("utf-8"), v.encode("utf-8")) + for k, v in dct["headers"].items() + ) + + message = self._message._replace(**dct) + + kwargs = {} + if scheme is not sentinel: + kwargs["scheme"] = scheme + if host is not sentinel: + kwargs["host"] = host + if remote is not sentinel: + kwargs["remote"] = remote + if client_max_size is sentinel: + client_max_size = self._client_max_size + + return self.__class__( + message, + self._payload, + self._protocol, + self._payload_writer, + self._task, + self._loop, + client_max_size=client_max_size, + state=self._state.copy(), + **kwargs, + ) + + @property + def task(self) -> "asyncio.Task[None]": + return self._task + + @property + def protocol(self) -> "RequestHandler": + return self._protocol + + @property + def transport(self) -> asyncio.Transport | None: + if self._protocol is None: + return None + return self._protocol.transport + + @property + def writer(self) -> AbstractStreamWriter: + return self._payload_writer + + @property + def client_max_size(self) -> int: + return self._client_max_size + + @reify + def message(self) -> RawRequestMessage: + warnings.warn("Request.message is deprecated", DeprecationWarning, stacklevel=3) + return self._message + + @reify + def rel_url(self) -> URL: + return self._rel_url + + @reify + def loop(self) -> asyncio.AbstractEventLoop: + warnings.warn( + "request.loop property is deprecated", DeprecationWarning, stacklevel=2 + ) + return self._loop + + # MutableMapping API + + @overload # type: ignore[override] + def __getitem__(self, key: RequestKey[_T]) -> _T: ... + + @overload + def __getitem__(self, key: str) -> Any: ... + + def __getitem__(self, key: str | RequestKey[_T]) -> Any: + return self._state[key] + + @overload # type: ignore[override] + def __setitem__(self, key: RequestKey[_T], value: _T) -> None: ... + + @overload + def __setitem__(self, key: str, value: Any) -> None: ... + + def __setitem__(self, key: str | RequestKey[_T], value: Any) -> None: + if not isinstance(key, RequestKey) and key not in BaseRequest._seen_str_keys: + BaseRequest._seen_str_keys.add(key) + warnings.warn( + "It is recommended to use web.RequestKey instances for keys.\n" + + "https://docs.aiohttp.org/en/stable/web_advanced.html" + + "#request-s-storage", + category=NotAppKeyWarning, + stacklevel=2, + ) + self._state[key] = value + + def __delitem__(self, key: str | RequestKey[_T]) -> None: + del self._state[key] + + def __len__(self) -> int: + return len(self._state) + + def __iter__(self) -> Iterator[str | RequestKey[Any]]: + return iter(self._state) + + ######## + + @reify + def secure(self) -> bool: + """A bool indicating if the request is handled with SSL.""" + return self.scheme == "https" + + @reify + def forwarded(self) -> tuple[Mapping[str, str], ...]: + """A tuple containing all parsed Forwarded header(s). + + Makes an effort to parse Forwarded headers as specified by RFC 7239: + + - It adds one (immutable) dictionary per Forwarded 'field-value', ie + per proxy. The element corresponds to the data in the Forwarded + field-value added by the first proxy encountered by the client. Each + subsequent item corresponds to those added by later proxies. + - It checks that every value has valid syntax in general as specified + in section 4: either a 'token' or a 'quoted-string'. + - It un-escapes found escape sequences. + - It does NOT validate 'by' and 'for' contents as specified in section + 6. + - It does NOT validate 'host' contents (Host ABNF). + - It does NOT validate 'proto' contents for valid URI scheme names. + + Returns a tuple containing one or more immutable dicts + """ + elems = [] + for field_value in self._message.headers.getall(hdrs.FORWARDED, ()): + length = len(field_value) + pos = 0 + need_separator = False + elem: dict[str, str] = {} + elems.append(types.MappingProxyType(elem)) + while 0 <= pos < length: + match = _FORWARDED_PAIR_RE.match(field_value, pos) + if match is not None: # got a valid forwarded-pair + if need_separator: + # bad syntax here, skip to next comma + pos = field_value.find(",", pos) + else: + name, value, port = match.groups() + if value[0] == '"': + # quoted string: remove quotes and unescape + value = _QUOTED_PAIR_REPLACE_RE.sub(r"\1", value[1:-1]) + if port: + value += port + elem[name.lower()] = value + pos += len(match.group(0)) + need_separator = True + elif field_value[pos] == ",": # next forwarded-element + need_separator = False + elem = {} + elems.append(types.MappingProxyType(elem)) + pos += 1 + elif field_value[pos] == ";": # next forwarded-pair + need_separator = False + pos += 1 + elif field_value[pos] in " \t": + # Allow whitespace even between forwarded-pairs, though + # RFC 7239 doesn't. This simplifies code and is in line + # with Postel's law. + pos += 1 + else: + # bad syntax here, skip to next comma + pos = field_value.find(",", pos) + return tuple(elems) + + @reify + def scheme(self) -> str: + """A string representing the scheme of the request. + + Hostname is resolved in this order: + + - overridden value by .clone(scheme=new_scheme) call. + - type of connection to peer: HTTPS if socket is SSL, HTTP otherwise. + + 'http' or 'https'. + """ + if self._transport_sslcontext: + return "https" + else: + return "http" + + @reify + def method(self) -> str: + """Read only property for getting HTTP method. + + The value is upper-cased str like 'GET', 'POST', 'PUT' etc. + """ + return self._method + + @reify + def version(self) -> HttpVersion: + """Read only property for getting HTTP version of request. + + Returns aiohttp.protocol.HttpVersion instance. + """ + return self._version + + @reify + def host(self) -> str: + """Hostname of the request. + + Hostname is resolved in this order: + + - overridden value by .clone(host=new_host) call. + - HOST HTTP header + - local socket address the request arrived on + (transport ``sockname``) + - empty string if no transport information is available + + For example, 'example.com' or 'localhost:8080'. + + For historical reasons, the port number may be included. + """ + host = self._message.headers.get(hdrs.HOST) + if host is not None: + return host + sockname = self._transport_sockname + if sockname is None: + return "" + if isinstance(sockname, tuple): + # AF_INET6 returns a 4-tuple (host, port, flowinfo, scopeid); + # bracket the bare address so it matches the Host-header shape + # and is a valid URL authority component. + if len(sockname) == 4: + return f"[{sockname[0]}]" + return str(sockname[0]) + return str(sockname) + + @reify + def remote(self) -> str | None: + """Remote IP of client initiated HTTP request. + + The IP is resolved in this order: + + - overridden value by .clone(remote=new_remote) call. + - peername of opened socket + """ + if self._transport_peername is None: + return None + if isinstance(self._transport_peername, (list, tuple)): + return str(self._transport_peername[0]) + return str(self._transport_peername) + + @reify + def url(self) -> URL: + """The full URL of the request.""" + # authority is used here because it may include the port number + # and we want yarl to parse it correctly + return URL.build(scheme=self.scheme, authority=self.host).join(self._rel_url) + + @reify + def path(self) -> str: + """The URL including *PATH INFO* without the host or scheme. + + E.g., ``/app/blog`` + """ + return self._rel_url.path + + @reify + def path_qs(self) -> str: + """The URL including PATH_INFO and the query string. + + E.g, /app/blog?id=10 + """ + return str(self._rel_url) + + @reify + def raw_path(self) -> str: + """The URL including raw *PATH INFO* without the host or scheme. + + Warning, the path is unquoted and may contains non valid URL characters + + E.g., ``/my%2Fpath%7Cwith%21some%25strange%24characters`` + """ + return self._message.path + + @reify + def query(self) -> "MultiMapping[str]": + """A multidict with all the variables in the query string.""" + return self._rel_url.query + + @reify + def query_string(self) -> str: + """The query string in the URL. + + E.g., id=10 + """ + return self._rel_url.query_string + + @reify + def headers(self) -> CIMultiDictProxy[str]: + """A case-insensitive multidict proxy with all headers.""" + return self._headers + + @reify + def raw_headers(self) -> RawHeaders: + """A sequence of pairs for all headers.""" + return self._message.raw_headers + + @reify + def if_modified_since(self) -> datetime.datetime | None: + """The value of If-Modified-Since HTTP header, or None. + + This header is represented as a `datetime` object. + """ + return parse_http_date(self.headers.get(hdrs.IF_MODIFIED_SINCE)) + + @reify + def if_unmodified_since(self) -> datetime.datetime | None: + """The value of If-Unmodified-Since HTTP header, or None. + + This header is represented as a `datetime` object. + """ + return parse_http_date(self.headers.get(hdrs.IF_UNMODIFIED_SINCE)) + + @staticmethod + def _etag_values(etag_header: str) -> Iterator[ETag]: + """Extract `ETag` objects from raw header.""" + if etag_header == ETAG_ANY: + yield ETag( + is_weak=False, + value=ETAG_ANY, + ) + else: + for match in LIST_QUOTED_ETAG_RE.finditer(etag_header): + is_weak, value, garbage = match.group(2, 3, 4) + # Any symbol captured by 4th group means + # that the following sequence is invalid. + if garbage: + break + + yield ETag( + is_weak=bool(is_weak), + value=value, + ) + + @classmethod + def _if_match_or_none_impl( + cls, header_value: str | None + ) -> tuple[ETag, ...] | None: + if not header_value: + return None + + return tuple(cls._etag_values(header_value)) + + @reify + def if_match(self) -> tuple[ETag, ...] | None: + """The value of If-Match HTTP header, or None. + + This header is represented as a `tuple` of `ETag` objects. + """ + return self._if_match_or_none_impl(self.headers.get(hdrs.IF_MATCH)) + + @reify + def if_none_match(self) -> tuple[ETag, ...] | None: + """The value of If-None-Match HTTP header, or None. + + This header is represented as a `tuple` of `ETag` objects. + """ + return self._if_match_or_none_impl(self.headers.get(hdrs.IF_NONE_MATCH)) + + @reify + def if_range(self) -> datetime.datetime | None: + """The value of If-Range HTTP header, or None. + + This header is represented as a `datetime` object. + """ + return parse_http_date(self.headers.get(hdrs.IF_RANGE)) + + @reify + def keep_alive(self) -> bool: + """Is keepalive enabled by client?""" + return not self._message.should_close + + @reify + def cookies(self) -> Mapping[str, str]: + """Return request cookies. + + A read-only dictionary-like object. + """ + # Use parse_cookie_header for RFC 6265 compliant Cookie header parsing + # that accepts special characters in cookie names (fixes #2683) + parsed = parse_cookie_header(self.headers.get(hdrs.COOKIE, "")) + # Extract values from Morsel objects + return MappingProxyType({name: morsel.value for name, morsel in parsed}) + + @reify + def http_range(self) -> slice: + """The content of Range HTTP header. + + Return a slice instance. + + """ + rng = self._headers.get(hdrs.RANGE) + start, end = None, None + if rng is not None: + try: + pattern = r"^bytes=(\d*)-(\d*)$" + start, end = re.findall(pattern, rng, re.ASCII)[0] + except IndexError: # pattern was not found in header + raise ValueError("range not in acceptable format") + + end = int(end) if end else None + start = int(start) if start else None + + if start is None and end is not None: + # end with no start is to return tail of content + start = -end + end = None + + if start is not None and end is not None: + # end is inclusive in range header, exclusive for slice + end += 1 + + if start >= end: + raise ValueError("start cannot be after end") + + if start is end is None: # No valid range supplied + raise ValueError("No start or end of range specified") + + return slice(start, end, 1) + + @reify + def content(self) -> StreamReader: + """Return raw payload stream.""" + return self._payload + + @property + def has_body(self) -> bool: + """Return True if request's HTTP BODY can be read, False otherwise.""" + warnings.warn( + "Deprecated, use .can_read_body #2005", DeprecationWarning, stacklevel=2 + ) + return not self._payload.at_eof() + + @property + def can_read_body(self) -> bool: + """Return True if request's HTTP BODY can be read, False otherwise.""" + return not self._payload.at_eof() + + @reify + def body_exists(self) -> bool: + """Return True if request has HTTP BODY, False otherwise.""" + return type(self._payload) is not EmptyStreamReader + + async def release(self) -> None: + """Release request. + + Eat unread part of HTTP BODY if present. + """ + while not self._payload.at_eof(): + await self._payload.readany() + + async def read(self) -> bytes: + """Read request body if present. + + Returns bytes object with full request content. + """ + if self._read_bytes is None: + # Raise the buffer limits so compressed payloads decompress in + # larger chunks instead of many small pause/resume cycles. + if self._client_max_size: + self._payload.set_read_chunk_size(self._client_max_size) + body = bytearray() + while True: + chunk = await self._payload.readany() + body.extend(chunk) + if self._client_max_size: + body_size = len(body) + if body_size > self._client_max_size: + raise HTTPRequestEntityTooLarge(self._client_max_size) + if not chunk: + break + self._read_bytes = bytes(body) + return self._read_bytes + + async def text(self) -> str: + """Return BODY as text using encoding from .charset.""" + bytes_body = await self.read() + encoding = self.charset or "utf-8" + return bytes_body.decode(encoding) + + async def json(self, *, loads: JSONDecoder = DEFAULT_JSON_DECODER) -> Any: + """Return BODY as JSON.""" + body = await self.text() + return loads(body) + + async def multipart(self) -> MultipartReader: + """Return async iterator to process BODY as multipart.""" + return MultipartReader( + self._headers, + self._payload, + client_max_size=self._client_max_size, + max_field_size=self._protocol.max_field_size, + max_headers=self._protocol.max_headers, + max_size_error_cls=HTTPRequestEntityTooLarge, + ) + + async def post(self) -> "MultiDictProxy[_Post]": + """Return POST parameters.""" + if self._post is not None: + return self._post + if self._method not in self.POST_METHODS: + self._post = MultiDictProxy(MultiDict()) + return self._post + + content_type = self.content_type + if content_type not in ( + "", + "application/x-www-form-urlencoded", + "multipart/form-data", + ): + self._post = MultiDictProxy(MultiDict()) + return self._post + + out: MultiDict[_Post] = MultiDict() + + if content_type == "multipart/form-data": + multipart = await self.multipart() + max_size = self._client_max_size + + size = 0 + while (field := await multipart.next()) is not None: + field_ct = field.headers.get(hdrs.CONTENT_TYPE) + + if isinstance(field, BodyPartReader): + if field.name is None: + raise ValueError("Multipart field missing name.") + + # Note that according to RFC 7578, the Content-Type header + # is optional, even for files, so we can't assume it's + # present. + # https://tools.ietf.org/html/rfc7578#section-4.4 + if field.filename: + # store file in temp file + tmp = await self._loop.run_in_executor( + None, tempfile.TemporaryFile + ) + while chunk := await field.read_chunk(size=DEFAULT_CHUNK_SIZE): + async for decoded_chunk in field.decode_iter(chunk): + await self._loop.run_in_executor( + None, tmp.write, decoded_chunk + ) + size += len(decoded_chunk) + if 0 < max_size < size: + await self._loop.run_in_executor(None, tmp.close) + raise HTTPRequestEntityTooLarge(max_size) + await self._loop.run_in_executor(None, tmp.seek, 0) + + if field_ct is None: + field_ct = "application/octet-stream" + + ff = FileField( + field.name, + field.filename, + cast(io.BufferedReader, tmp), + field_ct, + field.headers, + ) + out.add(field.name, ff) + else: + # deal with ordinary data + raw_data = bytearray() + while chunk := await field.read_chunk(): + size += len(chunk) + if 0 < max_size < size: + raise HTTPRequestEntityTooLarge(max_size) + raw_data.extend(chunk) + + value = bytearray() + # form-data doesn't support compression, so don't need to check size again. + async for d in field.decode_iter(raw_data): + value.extend(d) + + if field_ct is None or field_ct.startswith("text/"): + charset = field.get_charset(default="utf-8") + out.add(field.name, value.decode(charset)) + else: + out.add(field.name, value) + else: + raise ValueError( + "To decode nested multipart you need to use custom reader", + ) + else: + data = await self.read() + if data: + charset = self.charset or "utf-8" + out.extend( + parse_qsl( + data.rstrip().decode(charset), + keep_blank_values=True, + encoding=charset, + ) + ) + + self._post = MultiDictProxy(out) + return self._post + + def get_extra_info(self, name: str, default: Any = None) -> Any: + """Extra info from protocol transport""" + protocol = self._protocol + if protocol is None: + return default + + transport = protocol.transport + if transport is None: + return default + + return transport.get_extra_info(name, default) + + def __repr__(self) -> str: + ascii_encodable_path = self.path.encode("ascii", "backslashreplace").decode( + "ascii" + ) + return f"<{self.__class__.__name__} {self._method} {ascii_encodable_path} >" + + def __eq__(self, other: object) -> bool: + return id(self) == id(other) + + def __bool__(self) -> bool: + return True + + async def _prepare_hook(self, response: StreamResponse) -> None: + return + + def _cancel(self, exc: BaseException) -> None: + set_exception(self._payload, exc) + + def _finish(self) -> None: + if self._post is None or self.content_type != "multipart/form-data": + return + + # NOTE: Release file descriptors for the + # NOTE: `tempfile.Temporaryfile`-created `_io.BufferedRandom` + # NOTE: instances of files sent within multipart request body + # NOTE: via HTTP POST request. + for file_name, file_field_object in self._post.items(): + if isinstance(file_field_object, FileField): + file_field_object.file.close() + + +class Request(BaseRequest): + + ATTRS = BaseRequest.ATTRS | frozenset(["_match_info"]) + + _match_info: Optional["UrlMappingMatchInfo"] = None + + if DEBUG: + + def __setattr__(self, name: str, val: Any) -> None: + if name not in self.ATTRS: + warnings.warn( + f"Setting custom {self.__class__.__name__}.{name} attribute " + "is discouraged", + DeprecationWarning, + stacklevel=2, + ) + super().__setattr__(name, val) + + def clone( + self, + *, + method: str | _SENTINEL = sentinel, + rel_url: StrOrURL | _SENTINEL = sentinel, + headers: LooseHeaders | _SENTINEL = sentinel, + scheme: str | _SENTINEL = sentinel, + host: str | _SENTINEL = sentinel, + remote: str | _SENTINEL = sentinel, + client_max_size: int | _SENTINEL = sentinel, + ) -> "Request": + ret = super().clone( + method=method, + rel_url=rel_url, + headers=headers, + scheme=scheme, + host=host, + remote=remote, + client_max_size=client_max_size, + ) + new_ret = cast(Request, ret) + new_ret._match_info = self._match_info + return new_ret + + @reify + def match_info(self) -> "UrlMappingMatchInfo": + """Result of route resolving.""" + match_info = self._match_info + assert match_info is not None + return match_info + + @property + def app(self) -> "Application": + """Application instance.""" + match_info = self._match_info + assert match_info is not None + return match_info.current_app + + @property + def config_dict(self) -> ChainMapProxy: + match_info = self._match_info + assert match_info is not None + lst = match_info.apps + app = self.app + idx = lst.index(app) + sublist = list(reversed(lst[: idx + 1])) + return ChainMapProxy(sublist) + + async def _prepare_hook(self, response: StreamResponse) -> None: + match_info = self._match_info + if match_info is None: + return + for app in match_info._apps: + if on_response_prepare := app.on_response_prepare: + await on_response_prepare.send(self, response) diff --git a/py_modules/aiohttp/web_response.py b/py_modules/aiohttp/web_response.py new file mode 100644 index 0000000..ec0f9a4 --- /dev/null +++ b/py_modules/aiohttp/web_response.py @@ -0,0 +1,911 @@ +import asyncio +import collections.abc +import datetime +import enum +import json +import math +import time +import warnings +from collections.abc import Iterator, MutableMapping +from concurrent.futures import Executor +from http import HTTPStatus +from http.cookies import SimpleCookie +from typing import TYPE_CHECKING, Any, Optional, TypeVar, cast, overload + +from multidict import CIMultiDict, istr + +from . import hdrs, payload +from .abc import AbstractStreamWriter +from .compression_utils import MAX_SYNC_CHUNK_SIZE, ZLibCompressor +from .helpers import ( + ETAG_ANY, + QUOTED_ETAG_RE, + ETag, + HeadersMixin, + ResponseKey, + must_be_empty_body, + parse_http_date, + rfc822_formatted_time, + sentinel, + should_remove_content_length, + validate_etag_value, +) +from .http import SERVER_SOFTWARE, HttpVersion10, HttpVersion11 +from .payload import Payload +from .typedefs import JSONBytesEncoder, JSONEncoder, LooseHeaders + +REASON_PHRASES = {http_status.value: http_status.phrase for http_status in HTTPStatus} + +__all__ = ( + "ContentCoding", + "StreamResponse", + "Response", + "json_response", + "json_bytes_response", +) + + +if TYPE_CHECKING: + from .web_request import BaseRequest + + BaseClass = MutableMapping[str, Any] +else: + BaseClass = collections.abc.MutableMapping + + +_T = TypeVar("_T") + + +# TODO(py311): Convert to StrEnum for wider use +class ContentCoding(enum.Enum): + # The content codings that we have support for. + # + # Additional registered codings are listed at: + # https://www.iana.org/assignments/http-parameters/http-parameters.xhtml#content-coding + deflate = "deflate" + gzip = "gzip" + identity = "identity" + + +CONTENT_CODINGS = {coding.value: coding for coding in ContentCoding} + +############################################################ +# HTTP Response classes +############################################################ + + +class StreamResponse(MutableMapping[str | ResponseKey[Any], Any], HeadersMixin): + + _body: None | bytes | bytearray | Payload + _length_check = True + _body = None + _keep_alive: bool | None = None + _chunked: bool = False + _compression: bool = False + _compression_strategy: int | None = None + _compression_force: ContentCoding | None = None + _req: Optional["BaseRequest"] = None + _payload_writer: AbstractStreamWriter | None = None + _eof_sent: bool = False + _must_be_empty_body: bool | None = None + _body_length = 0 + _cookies: SimpleCookie | None = None + _send_headers_immediately = True + _seen_str_keys: set[str] = set() + + def __init__( + self, + *, + status: int = 200, + reason: str | None = None, + headers: LooseHeaders | None = None, + _real_headers: CIMultiDict[str] | None = None, + ) -> None: + """Initialize a new stream response object. + + _real_headers is an internal parameter used to pass a pre-populated + headers object. It is used by the `Response` class to avoid copying + the headers when creating a new response object. It is not intended + to be used by external code. + """ + self._state: dict[str | ResponseKey[Any], Any] = {} + + if _real_headers is not None: + self._headers = _real_headers + elif headers is not None: + self._headers: CIMultiDict[str] = CIMultiDict(headers) + else: + self._headers = CIMultiDict() + + self._set_status(status, reason) + + @property + def prepared(self) -> bool: + return self._eof_sent or self._payload_writer is not None + + @property + def task(self) -> "asyncio.Task[None] | None": + if self._req: + return self._req.task + else: + return None + + @property + def status(self) -> int: + return self._status + + @property + def chunked(self) -> bool: + return self._chunked + + @property + def compression(self) -> bool: + return self._compression + + @property + def reason(self) -> str: + return self._reason + + def set_status( + self, + status: int, + reason: str | None = None, + ) -> None: + assert ( + not self.prepared + ), "Cannot change the response status code after the headers have been sent" + self._set_status(status, reason) + + def _set_status(self, status: int, reason: str | None) -> None: + self._status = int(status) + if reason is None: + reason = REASON_PHRASES.get(self._status, "") + elif "\r" in reason or "\n" in reason: + raise ValueError("Reason cannot contain \\r or \\n") + self._reason = reason + + @property + def keep_alive(self) -> bool | None: + return self._keep_alive + + def force_close(self) -> None: + self._keep_alive = False + + @property + def body_length(self) -> int: + return self._body_length + + @property + def output_length(self) -> int: + warnings.warn("output_length is deprecated", DeprecationWarning) + assert self._payload_writer + return self._payload_writer.buffer_size + + def enable_chunked_encoding(self, chunk_size: int | None = None) -> None: + """Enables automatic chunked transfer encoding.""" + if hdrs.CONTENT_LENGTH in self._headers: + raise RuntimeError( + "You can't enable chunked encoding when a content length is set" + ) + if chunk_size is not None: + warnings.warn("Chunk size is deprecated #1615", DeprecationWarning) + self._chunked = True + + def enable_compression( + self, + force: bool | ContentCoding | None = None, + strategy: int | None = None, + ) -> None: + """Enables response compression encoding.""" + # Backwards compatibility for when force was a bool <0.17. + if isinstance(force, bool): + force = ContentCoding.deflate if force else ContentCoding.identity + warnings.warn( + "Using boolean for force is deprecated #3318", DeprecationWarning + ) + elif force is not None: + assert isinstance( + force, ContentCoding + ), "force should one of None, bool or ContentEncoding" + + self._compression = True + self._compression_force = force + self._compression_strategy = strategy + + @property + def headers(self) -> "CIMultiDict[str]": + return self._headers + + @property + def cookies(self) -> SimpleCookie: + if self._cookies is None: + self._cookies = SimpleCookie() + return self._cookies + + def set_cookie( + self, + name: str, + value: str, + *, + expires: str | None = None, + domain: str | None = None, + max_age: int | str | None = None, + path: str = "/", + secure: bool | None = None, + httponly: bool | None = None, + version: str | None = None, + samesite: str | None = None, + partitioned: bool | None = None, + ) -> None: + """Set or update response cookie. + + Sets new cookie or updates existent with new value. + Also updates only those params which are not None. + """ + if self._cookies is None: + self._cookies = SimpleCookie() + + self._cookies[name] = value + c = self._cookies[name] + + if expires is not None: + c["expires"] = expires + elif c.get("expires") == "Thu, 01 Jan 1970 00:00:00 GMT": + del c["expires"] + + if domain is not None: + c["domain"] = domain + + if max_age is not None: + c["max-age"] = str(max_age) + elif "max-age" in c: + del c["max-age"] + + c["path"] = path + + if secure is not None: + c["secure"] = secure + if httponly is not None: + c["httponly"] = httponly + if version is not None: + c["version"] = version + if samesite is not None: + c["samesite"] = samesite + + if partitioned is not None: + c["partitioned"] = partitioned + + def del_cookie( + self, + name: str, + *, + domain: str | None = None, + path: str = "/", + secure: bool | None = None, + httponly: bool | None = None, + samesite: str | None = None, + ) -> None: + """Delete cookie. + + Creates new empty expired cookie. + """ + # TODO: do we need domain/path here? + if self._cookies is not None: + self._cookies.pop(name, None) + self.set_cookie( + name, + "", + max_age=0, + expires="Thu, 01 Jan 1970 00:00:00 GMT", + domain=domain, + path=path, + secure=secure, + httponly=httponly, + samesite=samesite, + ) + + @property + def content_length(self) -> int | None: + # Just a placeholder for adding setter + return super().content_length + + @content_length.setter + def content_length(self, value: int | None) -> None: + if value is not None: + value = int(value) + if self._chunked: + raise RuntimeError( + "You can't set content length when chunked encoding is enable" + ) + self._headers[hdrs.CONTENT_LENGTH] = str(value) + else: + self._headers.pop(hdrs.CONTENT_LENGTH, None) + + @property + def content_type(self) -> str: + # Just a placeholder for adding setter + return super().content_type + + @content_type.setter + def content_type(self, value: str) -> None: + self.content_type # read header values if needed + self._content_type = str(value) + self._generate_content_type_header() + + @property + def charset(self) -> str | None: + # Just a placeholder for adding setter + return super().charset + + @charset.setter + def charset(self, value: str | None) -> None: + ctype = self.content_type # read header values if needed + if ctype == "application/octet-stream": + raise RuntimeError( + "Setting charset for application/octet-stream " + "doesn't make sense, setup content_type first" + ) + assert self._content_dict is not None + if value is None: + self._content_dict.pop("charset", None) + else: + self._content_dict["charset"] = str(value).lower() + self._generate_content_type_header() + + @property + def last_modified(self) -> datetime.datetime | None: + """The value of Last-Modified HTTP header, or None. + + This header is represented as a `datetime` object. + """ + return parse_http_date(self._headers.get(hdrs.LAST_MODIFIED)) + + @last_modified.setter + def last_modified( + self, value: int | float | datetime.datetime | str | None + ) -> None: + if value is None: + self._headers.pop(hdrs.LAST_MODIFIED, None) + elif isinstance(value, (int, float)): + self._headers[hdrs.LAST_MODIFIED] = time.strftime( + "%a, %d %b %Y %H:%M:%S GMT", time.gmtime(math.ceil(value)) + ) + elif isinstance(value, datetime.datetime): + if value.microsecond: + value = value.replace(microsecond=0) + datetime.timedelta(seconds=1) + self._headers[hdrs.LAST_MODIFIED] = time.strftime( + "%a, %d %b %Y %H:%M:%S GMT", value.utctimetuple() + ) + elif isinstance(value, str): + self._headers[hdrs.LAST_MODIFIED] = value + else: + msg = f"Unsupported type for last_modified: {type(value).__name__}" + raise TypeError(msg) + + @property + def etag(self) -> ETag | None: + quoted_value = self._headers.get(hdrs.ETAG) + if not quoted_value: + return None + elif quoted_value == ETAG_ANY: + return ETag(value=ETAG_ANY) + match = QUOTED_ETAG_RE.fullmatch(quoted_value) + if not match: + return None + is_weak, value = match.group(1, 2) + return ETag( + is_weak=bool(is_weak), + value=value, + ) + + @etag.setter + def etag(self, value: ETag | str | None) -> None: + if value is None: + self._headers.pop(hdrs.ETAG, None) + elif (isinstance(value, str) and value == ETAG_ANY) or ( + isinstance(value, ETag) and value.value == ETAG_ANY + ): + self._headers[hdrs.ETAG] = ETAG_ANY + elif isinstance(value, str): + validate_etag_value(value) + self._headers[hdrs.ETAG] = f'"{value}"' + elif isinstance(value, ETag) and isinstance(value.value, str): + validate_etag_value(value.value) + hdr_value = f'W/"{value.value}"' if value.is_weak else f'"{value.value}"' + self._headers[hdrs.ETAG] = hdr_value + else: + raise ValueError( + f"Unsupported etag type: {type(value)}. " + f"etag must be str, ETag or None" + ) + + def _generate_content_type_header( + self, CONTENT_TYPE: istr = hdrs.CONTENT_TYPE + ) -> None: + assert self._content_dict is not None + assert self._content_type is not None + params = "; ".join(f"{k}={v}" for k, v in self._content_dict.items()) + if params: + ctype = self._content_type + "; " + params + else: + ctype = self._content_type + self._headers[CONTENT_TYPE] = ctype + + async def _do_start_compression(self, coding: ContentCoding) -> None: + if coding is ContentCoding.identity: + return + assert self._payload_writer is not None + self._headers[hdrs.CONTENT_ENCODING] = coding.value + self._payload_writer.enable_compression( + coding.value, self._compression_strategy + ) + # Compressed payload may have different content length, + # remove the header + self._headers.popall(hdrs.CONTENT_LENGTH, None) + + async def _start_compression(self, request: "BaseRequest") -> None: + if self._compression_force: + await self._do_start_compression(self._compression_force) + return + # Encoding comparisons should be case-insensitive + # https://www.rfc-editor.org/rfc/rfc9110#section-8.4.1 + accept_encoding = request.headers.get(hdrs.ACCEPT_ENCODING, "").lower() + for value, coding in CONTENT_CODINGS.items(): + if value in accept_encoding: + await self._do_start_compression(coding) + return + + async def prepare(self, request: "BaseRequest") -> AbstractStreamWriter | None: + if self._eof_sent: + return None + if self._payload_writer is not None: + return self._payload_writer + self._must_be_empty_body = must_be_empty_body(request.method, self.status) + return await self._start(request) + + async def _start(self, request: "BaseRequest") -> AbstractStreamWriter: + self._req = request + writer = self._payload_writer = request._payload_writer + + await self._prepare_headers() + await request._prepare_hook(self) + await self._write_headers() + + return writer + + async def _prepare_headers(self) -> None: + request = self._req + assert request is not None + writer = self._payload_writer + assert writer is not None + keep_alive = self._keep_alive + if keep_alive is None: + keep_alive = request.keep_alive + self._keep_alive = keep_alive + + version = request.version + + headers = self._headers + if self._cookies: + for cookie in self._cookies.values(): + value = cookie.output(header="")[1:] + headers.add(hdrs.SET_COOKIE, value) + + if self._compression: + await self._start_compression(request) + + if self._chunked: + if version != HttpVersion11: + raise RuntimeError( + "Using chunked encoding is forbidden " + f"for HTTP/{request.version.major}.{request.version.minor}" + ) + if not self._must_be_empty_body: + writer.enable_chunking() + headers[hdrs.TRANSFER_ENCODING] = "chunked" + elif self._length_check: # Disabled for WebSockets + writer.length = self.content_length + if writer.length is None: + if version >= HttpVersion11: + if not self._must_be_empty_body: + writer.enable_chunking() + headers[hdrs.TRANSFER_ENCODING] = "chunked" + elif not self._must_be_empty_body: + keep_alive = False + + # HTTP 1.1: https://tools.ietf.org/html/rfc7230#section-3.3.2 + # HTTP 1.0: https://tools.ietf.org/html/rfc1945#section-10.4 + if self._must_be_empty_body: + if hdrs.CONTENT_LENGTH in headers and should_remove_content_length( + request.method, self.status + ): + del headers[hdrs.CONTENT_LENGTH] + # https://datatracker.ietf.org/doc/html/rfc9112#section-6.1-10 + # https://datatracker.ietf.org/doc/html/rfc9112#section-6.1-13 + if hdrs.TRANSFER_ENCODING in headers: + del headers[hdrs.TRANSFER_ENCODING] + elif (writer.length if self._length_check else self.content_length) != 0: + # https://www.rfc-editor.org/rfc/rfc9110#section-8.3-5 + headers.setdefault(hdrs.CONTENT_TYPE, "application/octet-stream") + headers.setdefault(hdrs.DATE, rfc822_formatted_time()) + headers.setdefault(hdrs.SERVER, SERVER_SOFTWARE) + + # connection header + if hdrs.CONNECTION not in headers: + if keep_alive: + if version == HttpVersion10: + headers[hdrs.CONNECTION] = "keep-alive" + elif version == HttpVersion11: + headers[hdrs.CONNECTION] = "close" + + async def _write_headers(self) -> None: + request = self._req + assert request is not None + writer = self._payload_writer + assert writer is not None + # status line + version = request.version + status_line = f"HTTP/{version[0]}.{version[1]} {self._status} {self._reason}" + await writer.write_headers(status_line, self._headers) + # Send headers immediately if not opted into buffering + if self._send_headers_immediately: + writer.send_headers() + + async def write(self, data: bytes | bytearray | memoryview) -> None: + assert isinstance( + data, (bytes, bytearray, memoryview) + ), "data argument must be byte-ish (%r)" % type(data) + + if self._eof_sent: + raise RuntimeError("Cannot call write() after write_eof()") + if self._payload_writer is None: + raise RuntimeError("Cannot call write() before prepare()") + + await self._payload_writer.write(data) + + async def drain(self) -> None: + assert not self._eof_sent, "EOF has already been sent" + assert self._payload_writer is not None, "Response has not been started" + warnings.warn( + "drain method is deprecated, use await resp.write()", + DeprecationWarning, + stacklevel=2, + ) + await self._payload_writer.drain() + + async def write_eof(self, data: bytes = b"") -> None: + assert isinstance( + data, (bytes, bytearray, memoryview) + ), "data argument must be byte-ish (%r)" % type(data) + + if self._eof_sent: + return + + assert self._payload_writer is not None, "Response has not been started" + + await self._payload_writer.write_eof(data) + self._eof_sent = True + self._req = None + self._body_length = self._payload_writer.output_size + self._payload_writer = None + + def __repr__(self) -> str: + if self._eof_sent: + info = "eof" + elif self.prepared: + assert self._req is not None + info = f"{self._req.method} {self._req.path} " + else: + info = "not prepared" + return f"<{self.__class__.__name__} {self.reason} {info}>" + + @overload # type: ignore[override] + def __getitem__(self, key: ResponseKey[_T]) -> _T: ... + + @overload + def __getitem__(self, key: str) -> Any: ... + + def __getitem__(self, key: str | ResponseKey[_T]) -> Any: + return self._state[key] + + @overload # type: ignore[override] + def __setitem__(self, key: ResponseKey[_T], value: _T) -> None: ... + + @overload + def __setitem__(self, key: str, value: Any) -> None: ... + + def __setitem__(self, key: str | ResponseKey[_T], value: Any) -> None: + if ( + not isinstance(key, ResponseKey) + and key not in StreamResponse._seen_str_keys + ): + # Import here to break circular dependency + from .web_exceptions import NotAppKeyWarning + + StreamResponse._seen_str_keys.add(key) + warnings.warn( + "It is recommended to use web.ResponseKey instances for keys.\n" + + "https://docs.aiohttp.org/en/stable/web_advanced.html" + + "#response-s-storage", + category=NotAppKeyWarning, + stacklevel=2, + ) + self._state[key] = value + + def __delitem__(self, key: str | ResponseKey[_T]) -> None: + del self._state[key] + + def __len__(self) -> int: + return len(self._state) + + def __iter__(self) -> Iterator[str | ResponseKey[Any]]: + return iter(self._state) + + def __hash__(self) -> int: + return hash(id(self)) + + def __eq__(self, other: object) -> bool: + return self is other + + def __bool__(self) -> bool: + return True + + +class Response(StreamResponse): + + _compressed_body: bytes | None = None + _send_headers_immediately = False + + def __init__( + self, + *, + body: Any = None, + status: int = 200, + reason: str | None = None, + text: str | None = None, + headers: LooseHeaders | None = None, + content_type: str | None = None, + charset: str | None = None, + zlib_executor_size: int = MAX_SYNC_CHUNK_SIZE, + zlib_executor: Executor | None = None, + ) -> None: + if body is not None and text is not None: + raise ValueError("body and text are not allowed together") + + if headers is None: + real_headers: CIMultiDict[str] = CIMultiDict() + else: + real_headers = CIMultiDict(headers) + + if content_type is not None and "charset" in content_type: + raise ValueError("charset must not be in content_type argument") + + if text is not None: + if hdrs.CONTENT_TYPE in real_headers: + if content_type or charset: + raise ValueError( + "passing both Content-Type header and " + "content_type or charset params " + "is forbidden" + ) + else: + # fast path for filling headers + if not isinstance(text, str): + raise TypeError("text argument must be str (%r)" % type(text)) + if content_type is None: + content_type = "text/plain" + if charset is None: + charset = "utf-8" + real_headers[hdrs.CONTENT_TYPE] = content_type + "; charset=" + charset + body = text.encode(charset) + text = None + elif hdrs.CONTENT_TYPE in real_headers: + if content_type is not None or charset is not None: + raise ValueError( + "passing both Content-Type header and " + "content_type or charset params " + "is forbidden" + ) + elif content_type is not None: + if charset is not None: + content_type += "; charset=" + charset + real_headers[hdrs.CONTENT_TYPE] = content_type + + super().__init__(status=status, reason=reason, _real_headers=real_headers) + + if text is not None: + self.text = text + else: + self.body = body + + self._zlib_executor_size = zlib_executor_size + self._zlib_executor = zlib_executor + + @property + def body(self) -> bytes | bytearray | Payload | None: + return self._body + + @body.setter + def body(self, body: Any) -> None: + if body is None: + self._body = None + elif isinstance(body, (bytes, bytearray)): + self._body = body + else: + try: + self._body = body = payload.PAYLOAD_REGISTRY.get(body) + except payload.LookupError: + raise ValueError("Unsupported body type %r" % type(body)) + + headers = self._headers + + # set content-type + if hdrs.CONTENT_TYPE not in headers: + headers[hdrs.CONTENT_TYPE] = body.content_type + + # copy payload headers + if body.headers: + for key, value in body.headers.items(): + if key not in headers: + headers[key] = value + + self._compressed_body = None + + @property + def text(self) -> str | None: + if self._body is None: + return None + # Note: When _body is a Payload (e.g. FilePayload), this may do blocking I/O + # This is generally safe as most common payloads (BytesPayload, StringPayload) + # don't do blocking I/O, but be careful with file-based payloads + return self._body.decode(self.charset or "utf-8") + + @text.setter + def text(self, text: str) -> None: + assert text is None or isinstance( + text, str + ), "text argument must be str (%r)" % type(text) + + if self.content_type == "application/octet-stream": + self.content_type = "text/plain" + if self.charset is None: + self.charset = "utf-8" + + self._body = text.encode(self.charset) + self._compressed_body = None + + @property + def content_length(self) -> int | None: + if self._chunked: + return None + + if hdrs.CONTENT_LENGTH in self._headers: + return int(self._headers[hdrs.CONTENT_LENGTH]) + + if self._compressed_body is not None: + # Return length of the compressed body + return len(self._compressed_body) + elif isinstance(self._body, Payload): + # A payload without content length, or a compressed payload + return None + elif self._body is not None: + return len(self._body) + else: + return 0 + + @content_length.setter + def content_length(self, value: int | None) -> None: + raise RuntimeError("Content length is set automatically") + + async def write_eof(self, data: bytes = b"") -> None: + if self._eof_sent: + return + if self._compressed_body is None: + body = self._body + else: + body = self._compressed_body + assert not data, f"data arg is not supported, got {data!r}" + assert self._req is not None + assert self._payload_writer is not None + if body is None or self._must_be_empty_body: + await super().write_eof() + elif isinstance(self._body, Payload): + try: + await self._body.write(self._payload_writer) + finally: + await self._body.close() + await super().write_eof() + else: + await super().write_eof(cast(bytes, body)) + + async def _start(self, request: "BaseRequest") -> AbstractStreamWriter: + if hdrs.CONTENT_LENGTH in self._headers: + if should_remove_content_length(request.method, self.status): + del self._headers[hdrs.CONTENT_LENGTH] + elif not self._chunked: + if isinstance(self._body, Payload): + if (size := self._body.size) is not None: + self._headers[hdrs.CONTENT_LENGTH] = str(size) + else: + body_len = len(self._body) if self._body else "0" + # https://www.rfc-editor.org/rfc/rfc9110.html#section-8.6-7 + if body_len != "0" or ( + self.status != 304 and request.method != hdrs.METH_HEAD + ): + self._headers[hdrs.CONTENT_LENGTH] = str(body_len) + + return await super()._start(request) + + async def _do_start_compression(self, coding: ContentCoding) -> None: + if self._chunked or isinstance(self._body, Payload): + return await super()._do_start_compression(coding) + if coding is ContentCoding.identity: + return + # Instead of using _payload_writer.enable_compression, + # compress the whole body + compressor = ZLibCompressor( + encoding=coding.value, + max_sync_chunk_size=self._zlib_executor_size, + executor=self._zlib_executor, + ) + assert self._body is not None + self._compressed_body = ( + await compressor.compress(self._body) + compressor.flush() + ) + self._headers[hdrs.CONTENT_ENCODING] = coding.value + self._headers[hdrs.CONTENT_LENGTH] = str(len(self._compressed_body)) + + +def json_response( + data: Any = sentinel, + *, + text: str | None = None, + body: bytes | None = None, + status: int = 200, + reason: str | None = None, + headers: LooseHeaders | None = None, + content_type: str = "application/json", + dumps: JSONEncoder = json.dumps, +) -> Response: + if data is not sentinel: + if text or body: + raise ValueError("only one of data, text, or body should be specified") + else: + text = dumps(data) + return Response( + text=text, + body=body, + status=status, + reason=reason, + headers=headers, + content_type=content_type, + ) + + +def json_bytes_response( + data: Any = sentinel, + *, + dumps: JSONBytesEncoder, + body: bytes | None = None, + status: int = 200, + reason: str | None = None, + headers: LooseHeaders | None = None, + content_type: str = "application/json", +) -> Response: + """Create a JSON response using a bytes-returning encoder. + + Use this when your JSON encoder (like orjson) returns bytes + instead of str, avoiding the encode/decode overhead. + """ + if data is not sentinel: + if body is not None: + raise ValueError("only one of data or body should be specified") + else: + body = dumps(data) + return Response( + body=body, + status=status, + reason=reason, + headers=headers, + content_type=content_type, + ) diff --git a/py_modules/aiohttp/web_routedef.py b/py_modules/aiohttp/web_routedef.py new file mode 100644 index 0000000..4c40272 --- /dev/null +++ b/py_modules/aiohttp/web_routedef.py @@ -0,0 +1,203 @@ +import abc +import os # noqa +from collections.abc import Callable, Iterator, Sequence +from typing import TYPE_CHECKING, Any, Union, overload + +import attr + +from . import hdrs +from .abc import AbstractView +from .typedefs import Handler, PathLike + +if TYPE_CHECKING: + from .web_request import Request + from .web_response import StreamResponse + from .web_urldispatcher import AbstractRoute, UrlDispatcher +else: + Request = StreamResponse = UrlDispatcher = AbstractRoute = None + + +__all__ = ( + "AbstractRouteDef", + "RouteDef", + "StaticDef", + "RouteTableDef", + "head", + "options", + "get", + "post", + "patch", + "put", + "delete", + "route", + "view", + "static", +) + + +class AbstractRouteDef(abc.ABC): + @abc.abstractmethod + def register(self, router: UrlDispatcher) -> list[AbstractRoute]: + pass # pragma: no cover + + +_HandlerType = Union[type[AbstractView], Handler] + + +@attr.s(auto_attribs=True, frozen=True, repr=False, slots=True) +class RouteDef(AbstractRouteDef): + method: str + path: str + handler: _HandlerType + kwargs: dict[str, Any] + + def __repr__(self) -> str: + info = [] + for name, value in sorted(self.kwargs.items()): + info.append(f", {name}={value!r}") + return " {handler.__name__!r}{info}>".format( + method=self.method, path=self.path, handler=self.handler, info="".join(info) + ) + + def register(self, router: UrlDispatcher) -> list[AbstractRoute]: + if self.method in hdrs.METH_ALL: + reg = getattr(router, "add_" + self.method.lower()) + return [reg(self.path, self.handler, **self.kwargs)] + else: + return [ + router.add_route(self.method, self.path, self.handler, **self.kwargs) + ] + + +@attr.s(auto_attribs=True, frozen=True, repr=False, slots=True) +class StaticDef(AbstractRouteDef): + prefix: str + path: PathLike + kwargs: dict[str, Any] + + def __repr__(self) -> str: + info = [] + for name, value in sorted(self.kwargs.items()): + info.append(f", {name}={value!r}") + return " {path}{info}>".format( + prefix=self.prefix, path=self.path, info="".join(info) + ) + + def register(self, router: UrlDispatcher) -> list[AbstractRoute]: + resource = router.add_static(self.prefix, self.path, **self.kwargs) + routes = resource.get_info().get("routes", {}) + return list(routes.values()) + + +def route(method: str, path: str, handler: _HandlerType, **kwargs: Any) -> RouteDef: + return RouteDef(method, path, handler, kwargs) + + +def head(path: str, handler: _HandlerType, **kwargs: Any) -> RouteDef: + return route(hdrs.METH_HEAD, path, handler, **kwargs) + + +def options(path: str, handler: _HandlerType, **kwargs: Any) -> RouteDef: + return route(hdrs.METH_OPTIONS, path, handler, **kwargs) + + +def get( + path: str, + handler: _HandlerType, + *, + name: str | None = None, + allow_head: bool = True, + **kwargs: Any, +) -> RouteDef: + return route( + hdrs.METH_GET, path, handler, name=name, allow_head=allow_head, **kwargs + ) + + +def post(path: str, handler: _HandlerType, **kwargs: Any) -> RouteDef: + return route(hdrs.METH_POST, path, handler, **kwargs) + + +def put(path: str, handler: _HandlerType, **kwargs: Any) -> RouteDef: + return route(hdrs.METH_PUT, path, handler, **kwargs) + + +def patch(path: str, handler: _HandlerType, **kwargs: Any) -> RouteDef: + return route(hdrs.METH_PATCH, path, handler, **kwargs) + + +def delete(path: str, handler: _HandlerType, **kwargs: Any) -> RouteDef: + return route(hdrs.METH_DELETE, path, handler, **kwargs) + + +def view(path: str, handler: type[AbstractView], **kwargs: Any) -> RouteDef: + return route(hdrs.METH_ANY, path, handler, **kwargs) + + +def static(prefix: str, path: PathLike, **kwargs: Any) -> StaticDef: + return StaticDef(prefix, path, kwargs) + + +_Deco = Callable[[_HandlerType], _HandlerType] + + +class RouteTableDef(Sequence[AbstractRouteDef]): + """Route definition table""" + + def __init__(self) -> None: + self._items: list[AbstractRouteDef] = [] + + def __repr__(self) -> str: + return f"" + + @overload + def __getitem__(self, index: int) -> AbstractRouteDef: ... + + @overload + def __getitem__(self, index: slice) -> list[AbstractRouteDef]: ... + + def __getitem__(self, index): # type: ignore[no-untyped-def] + return self._items[index] + + def __iter__(self) -> Iterator[AbstractRouteDef]: + return iter(self._items) + + def __len__(self) -> int: + return len(self._items) + + def __contains__(self, item: object) -> bool: + return item in self._items + + def route(self, method: str, path: str, **kwargs: Any) -> _Deco: + def inner(handler: _HandlerType) -> _HandlerType: + self._items.append(RouteDef(method, path, handler, kwargs)) + return handler + + return inner + + def head(self, path: str, **kwargs: Any) -> _Deco: + return self.route(hdrs.METH_HEAD, path, **kwargs) + + def get(self, path: str, **kwargs: Any) -> _Deco: + return self.route(hdrs.METH_GET, path, **kwargs) + + def post(self, path: str, **kwargs: Any) -> _Deco: + return self.route(hdrs.METH_POST, path, **kwargs) + + def put(self, path: str, **kwargs: Any) -> _Deco: + return self.route(hdrs.METH_PUT, path, **kwargs) + + def patch(self, path: str, **kwargs: Any) -> _Deco: + return self.route(hdrs.METH_PATCH, path, **kwargs) + + def delete(self, path: str, **kwargs: Any) -> _Deco: + return self.route(hdrs.METH_DELETE, path, **kwargs) + + def options(self, path: str, **kwargs: Any) -> _Deco: + return self.route(hdrs.METH_OPTIONS, path, **kwargs) + + def view(self, path: str, **kwargs: Any) -> _Deco: + return self.route(hdrs.METH_ANY, path, **kwargs) + + def static(self, prefix: str, path: PathLike, **kwargs: Any) -> None: + self._items.append(StaticDef(prefix, path, kwargs)) diff --git a/py_modules/aiohttp/web_runner.py b/py_modules/aiohttp/web_runner.py new file mode 100644 index 0000000..b2128fa --- /dev/null +++ b/py_modules/aiohttp/web_runner.py @@ -0,0 +1,425 @@ +import asyncio +import signal +import socket +import warnings +from abc import ABC, abstractmethod +from typing import TYPE_CHECKING, Any + +from yarl import URL + +from .abc import AbstractAccessLogger +from .typedefs import PathLike +from .web_app import Application +from .web_log import AccessLogger +from .web_server import Server + +if TYPE_CHECKING: + from ssl import SSLContext +else: + try: + from ssl import SSLContext + except ImportError: # pragma: no cover + SSLContext = object # type: ignore[misc,assignment] + +__all__ = ( + "BaseSite", + "TCPSite", + "UnixSite", + "NamedPipeSite", + "SockSite", + "BaseRunner", + "AppRunner", + "ServerRunner", + "GracefulExit", +) + + +class GracefulExit(SystemExit): + code = 1 + + +def _raise_graceful_exit() -> None: + raise GracefulExit() + + +class BaseSite(ABC): + __slots__ = ("_runner", "_ssl_context", "_backlog", "_server") + + def __init__( + self, + runner: "BaseRunner", + *, + shutdown_timeout: float = 60.0, + ssl_context: SSLContext | None = None, + backlog: int = 128, + ) -> None: + if runner.server is None: + raise RuntimeError("Call runner.setup() before making a site") + if shutdown_timeout != 60.0: + msg = "shutdown_timeout should be set on BaseRunner" + warnings.warn(msg, DeprecationWarning, stacklevel=2) + runner._shutdown_timeout = shutdown_timeout + self._runner = runner + self._ssl_context = ssl_context + self._backlog = backlog + self._server: asyncio.AbstractServer | None = None + + @property + @abstractmethod + def name(self) -> str: + pass # pragma: no cover + + @abstractmethod + async def start(self) -> None: + self._runner._reg_site(self) + + async def stop(self) -> None: + self._runner._check_site(self) + if self._server is not None: # Maybe not started yet + self._server.close() + + self._runner._unreg_site(self) + + +class TCPSite(BaseSite): + __slots__ = ("_host", "_port", "_bound_port", "_reuse_address", "_reuse_port") + + def __init__( + self, + runner: "BaseRunner", + host: str | None = None, + port: int | None = None, + *, + shutdown_timeout: float = 60.0, + ssl_context: SSLContext | None = None, + backlog: int = 128, + reuse_address: bool | None = None, + reuse_port: bool | None = None, + ) -> None: + super().__init__( + runner, + shutdown_timeout=shutdown_timeout, + ssl_context=ssl_context, + backlog=backlog, + ) + self._host = host + if port is None: + port = 8443 if self._ssl_context else 8080 + self._port = port + self._bound_port: int | None = None + self._reuse_address = reuse_address + self._reuse_port = reuse_port + + @property + def port(self) -> int: + """The port the server is listening on. + + If the server hasn't been started yet, this returns the requested port + (which might be 0 for a dynamic port). + After the server starts, it returns the actual bound port. This is + especially useful when port=0 was requested, as it allows retrieving the + dynamically assigned port after the site has started. + """ + if self._bound_port is not None: + return self._bound_port + return self._port + + @property + def name(self) -> str: + scheme = "https" if self._ssl_context else "http" + host = "0.0.0.0" if not self._host else self._host + return str(URL.build(scheme=scheme, host=host, port=self.port)) + + async def start(self) -> None: + await super().start() + loop = asyncio.get_event_loop() + server = self._runner.server + assert server is not None + self._server = await loop.create_server( + server, + self._host, + self._port, + ssl=self._ssl_context, + backlog=self._backlog, + reuse_address=self._reuse_address, + reuse_port=self._reuse_port, + ) + if self._server.sockets: + self._bound_port = self._server.sockets[0].getsockname()[1] + else: + self._bound_port = self._port + + +class UnixSite(BaseSite): + __slots__ = ("_path",) + + def __init__( + self, + runner: "BaseRunner", + path: PathLike, + *, + shutdown_timeout: float = 60.0, + ssl_context: SSLContext | None = None, + backlog: int = 128, + ) -> None: + super().__init__( + runner, + shutdown_timeout=shutdown_timeout, + ssl_context=ssl_context, + backlog=backlog, + ) + self._path = path + + @property + def name(self) -> str: + scheme = "https" if self._ssl_context else "http" + return f"{scheme}://unix:{self._path}:" + + async def start(self) -> None: + await super().start() + loop = asyncio.get_event_loop() + server = self._runner.server + assert server is not None + self._server = await loop.create_unix_server( + server, + self._path, + ssl=self._ssl_context, + backlog=self._backlog, + ) + + +class NamedPipeSite(BaseSite): + __slots__ = ("_path",) + + def __init__( + self, runner: "BaseRunner", path: str, *, shutdown_timeout: float = 60.0 + ) -> None: + loop = asyncio.get_event_loop() + if not isinstance( + loop, asyncio.ProactorEventLoop # type: ignore[attr-defined] + ): + raise RuntimeError( + "Named Pipes only available in proactor loop under windows" + ) + super().__init__(runner, shutdown_timeout=shutdown_timeout) + self._path = path + + @property + def name(self) -> str: + return self._path + + async def start(self) -> None: + await super().start() + loop = asyncio.get_event_loop() + server = self._runner.server + assert server is not None + _server = await loop.start_serving_pipe( # type: ignore[attr-defined] + server, self._path + ) + self._server = _server[0] + + +class SockSite(BaseSite): + __slots__ = ("_sock", "_name") + + def __init__( + self, + runner: "BaseRunner", + sock: socket.socket, + *, + shutdown_timeout: float = 60.0, + ssl_context: SSLContext | None = None, + backlog: int = 128, + ) -> None: + super().__init__( + runner, + shutdown_timeout=shutdown_timeout, + ssl_context=ssl_context, + backlog=backlog, + ) + self._sock = sock + scheme = "https" if self._ssl_context else "http" + if hasattr(socket, "AF_UNIX") and sock.family == socket.AF_UNIX: + name = f"{scheme}://unix:{sock.getsockname()}:" + else: + host, port = sock.getsockname()[:2] + name = str(URL.build(scheme=scheme, host=host, port=port)) + self._name = name + + @property + def name(self) -> str: + return self._name + + async def start(self) -> None: + await super().start() + loop = asyncio.get_event_loop() + server = self._runner.server + assert server is not None + self._server = await loop.create_server( + server, sock=self._sock, ssl=self._ssl_context, backlog=self._backlog + ) + + +class BaseRunner(ABC): + __slots__ = ("_handle_signals", "_kwargs", "_server", "_sites", "_shutdown_timeout") + + def __init__( + self, + *, + handle_signals: bool = False, + shutdown_timeout: float = 60.0, + **kwargs: Any, + ) -> None: + self._handle_signals = handle_signals + self._kwargs = kwargs + self._server: Server | None = None + self._sites: list[BaseSite] = [] + self._shutdown_timeout = shutdown_timeout + + @property + def server(self) -> Server | None: + return self._server + + @property + def addresses(self) -> list[Any]: + ret: list[Any] = [] + for site in self._sites: + server = site._server + if server is not None: + sockets = server.sockets # type: ignore[attr-defined] + if sockets is not None: + for sock in sockets: + ret.append(sock.getsockname()) + return ret + + @property + def sites(self) -> set[BaseSite]: + return set(self._sites) + + async def setup(self) -> None: + loop = asyncio.get_event_loop() + + if self._handle_signals: + try: + loop.add_signal_handler(signal.SIGINT, _raise_graceful_exit) + loop.add_signal_handler(signal.SIGTERM, _raise_graceful_exit) + except NotImplementedError: # pragma: no cover + # add_signal_handler is not implemented on Windows + pass + + self._server = await self._make_server() + + @abstractmethod + async def shutdown(self) -> None: + """Call any shutdown hooks to help server close gracefully.""" + + async def cleanup(self) -> None: + # The loop over sites is intentional, an exception on gather() + # leaves self._sites in unpredictable state. + # The loop guaranties that a site is either deleted on success or + # still present on failure + for site in list(self._sites): + await site.stop() + + if self._server: # If setup succeeded + # Yield to event loop to ensure incoming requests prior to stopping the sites + # have all started to be handled before we proceed to close idle connections. + await asyncio.sleep(0) + self._server.pre_shutdown() + await self.shutdown() + await self._server.shutdown(self._shutdown_timeout) + await self._cleanup_server() + + self._server = None + if self._handle_signals: + loop = asyncio.get_running_loop() + try: + loop.remove_signal_handler(signal.SIGINT) + loop.remove_signal_handler(signal.SIGTERM) + except NotImplementedError: # pragma: no cover + # remove_signal_handler is not implemented on Windows + pass + + @abstractmethod + async def _make_server(self) -> Server: + pass # pragma: no cover + + @abstractmethod + async def _cleanup_server(self) -> None: + pass # pragma: no cover + + def _reg_site(self, site: BaseSite) -> None: + if site in self._sites: + raise RuntimeError(f"Site {site} is already registered in runner {self}") + self._sites.append(site) + + def _check_site(self, site: BaseSite) -> None: + if site not in self._sites: + raise RuntimeError(f"Site {site} is not registered in runner {self}") + + def _unreg_site(self, site: BaseSite) -> None: + if site not in self._sites: + raise RuntimeError(f"Site {site} is not registered in runner {self}") + self._sites.remove(site) + + +class ServerRunner(BaseRunner): + """Low-level web server runner""" + + __slots__ = ("_web_server",) + + def __init__( + self, web_server: Server, *, handle_signals: bool = False, **kwargs: Any + ) -> None: + super().__init__(handle_signals=handle_signals, **kwargs) + self._web_server = web_server + + async def shutdown(self) -> None: + pass + + async def _make_server(self) -> Server: + return self._web_server + + async def _cleanup_server(self) -> None: + pass + + +class AppRunner(BaseRunner): + """Web Application runner""" + + __slots__ = ("_app",) + + def __init__( + self, + app: Application, + *, + handle_signals: bool = False, + access_log_class: type[AbstractAccessLogger] = AccessLogger, + **kwargs: Any, + ) -> None: + super().__init__(handle_signals=handle_signals, **kwargs) + if not isinstance(app, Application): + raise TypeError( + f"The first argument should be web.Application instance, got {app!r}" + ) + self._kwargs["access_log_class"] = access_log_class + self._app = app + + @property + def app(self) -> Application: + return self._app + + async def shutdown(self) -> None: + await self._app.shutdown() + + async def _make_server(self) -> Server: + loop = asyncio.get_event_loop() + self._app._set_loop(loop) + self._app.on_startup.freeze() + await self._app.startup() + self._app.freeze() + + return self._app._make_handler(loop=loop, **self._kwargs) + + async def _cleanup_server(self) -> None: + await self._app.cleanup() diff --git a/py_modules/aiohttp/web_server.py b/py_modules/aiohttp/web_server.py new file mode 100644 index 0000000..97cf2a6 --- /dev/null +++ b/py_modules/aiohttp/web_server.py @@ -0,0 +1,91 @@ +"""Low level HTTP server.""" + +import asyncio +from typing import Any, Awaitable, Callable, Dict, List, Optional # noqa + +from .abc import AbstractStreamWriter +from .http_parser import RawRequestMessage +from .streams import StreamReader +from .web_protocol import RequestHandler, _RequestFactory, _RequestHandler +from .web_request import BaseRequest + +__all__ = ("Server",) + + +class Server: + def __init__( + self, + handler: _RequestHandler, + *, + request_factory: _RequestFactory | None = None, + handler_cancellation: bool = False, + loop: asyncio.AbstractEventLoop | None = None, + **kwargs: Any, + ) -> None: + self._loop = loop or asyncio.get_running_loop() + self._connections: dict[RequestHandler, asyncio.Transport] = {} + self._kwargs = kwargs + # requests_count is the number of requests being processed by the server + # for the lifetime of the server. + self.requests_count = 0 + self.request_handler = handler + self.request_factory = request_factory or self._make_request + self.handler_cancellation = handler_cancellation + + @property + def connections(self) -> list[RequestHandler]: + return list(self._connections.keys()) + + def connection_made( + self, handler: RequestHandler, transport: asyncio.Transport + ) -> None: + self._connections[handler] = transport + + def connection_lost( + self, handler: RequestHandler, exc: BaseException | None = None + ) -> None: + if handler in self._connections: + if handler._task_handler: + handler._task_handler.add_done_callback( + lambda f: self._connections.pop(handler, None) + ) + else: + del self._connections[handler] + + def _make_request( + self, + message: RawRequestMessage, + payload: StreamReader, + protocol: RequestHandler, + writer: AbstractStreamWriter, + task: "asyncio.Task[None]", + ) -> BaseRequest: + return BaseRequest(message, payload, protocol, writer, task, self._loop) + + def pre_shutdown(self) -> None: + for conn in self._connections: + conn.close() + + async def shutdown(self, timeout: float | None = None) -> None: + coros = (conn.shutdown(timeout) for conn in self._connections) + await asyncio.gather(*coros) + self._connections.clear() + + def __call__(self) -> RequestHandler: + try: + return RequestHandler(self, loop=self._loop, **self._kwargs) + except TypeError: + # Failsafe creation: remove all custom handler_args + kwargs = { + k: v + for k, v in self._kwargs.items() + if k in ["debug", "access_log_class"] + } + handler = RequestHandler(self, loop=self._loop, **kwargs) + handler.logger.warning( + "Failed to create request handler with custom kwargs %r, " + "falling back to filtered kwargs. This may indicate a " + "misconfiguration.", + self._kwargs, + ) + return handler diff --git a/py_modules/aiohttp/web_urldispatcher.py b/py_modules/aiohttp/web_urldispatcher.py new file mode 100644 index 0000000..74351ee --- /dev/null +++ b/py_modules/aiohttp/web_urldispatcher.py @@ -0,0 +1,1281 @@ +import abc +import asyncio +import base64 +import functools +import hashlib +import html +import inspect +import keyword +import os +import platform +import re +import sys +import warnings +from collections.abc import ( + Awaitable, + Callable, + Container, + Generator, + Iterable, + Iterator, + Mapping, + Sized, +) +from functools import wraps +from pathlib import Path +from re import Pattern +from types import MappingProxyType +from typing import TYPE_CHECKING, Any, Final, NoReturn, Optional, TypedDict, cast + +from yarl import URL, __version__ as yarl_version + +from . import hdrs +from .abc import AbstractMatchInfo, AbstractRouter, AbstractView +from .helpers import DEBUG, DEFAULT_CHUNK_SIZE +from .http import HttpVersion11 +from .typedefs import Handler, PathLike +from .web_exceptions import ( + HTTPException, + HTTPExpectationFailed, + HTTPForbidden, + HTTPMethodNotAllowed, + HTTPNotFound, +) +from .web_fileresponse import FileResponse +from .web_request import Request +from .web_response import Response, StreamResponse +from .web_routedef import AbstractRouteDef + +__all__ = ( + "UrlDispatcher", + "UrlMappingMatchInfo", + "AbstractResource", + "Resource", + "PlainResource", + "DynamicResource", + "AbstractRoute", + "ResourceRoute", + "StaticResource", + "View", +) + + +if TYPE_CHECKING: + from .web_app import Application + + BaseDict = dict[str, str] +else: + BaseDict = dict + +CIRCULAR_SYMLINK_ERROR = (RuntimeError,) if sys.version_info < (3, 13) else () + +YARL_VERSION: Final[tuple[int, ...]] = tuple(map(int, yarl_version.split(".")[:2])) + +HTTP_METHOD_RE: Final[Pattern[str]] = re.compile( + r"^[0-9A-Za-z!#\$%&'\*\+\-\.\^_`\|~]+$" +) +ROUTE_RE: Final[Pattern[str]] = re.compile( + r"(\{[_a-zA-Z][^{}]*(?:\{[^{}]*\}[^{}]*)*\})" +) +PATH_SEP: Final[str] = re.escape("/") + +IS_WINDOWS: Final[bool] = platform.system() == "Windows" + +_ExpectHandler = Callable[[Request], Awaitable[StreamResponse | None]] +_Resolve = tuple[Optional["UrlMappingMatchInfo"], set[str]] + +html_escape = functools.partial(html.escape, quote=True) + + +class _InfoDict(TypedDict, total=False): + path: str + + formatter: str + pattern: Pattern[str] + + directory: Path + prefix: str + routes: Mapping[str, "AbstractRoute"] + + app: "Application" + + domain: str + + rule: "AbstractRuleMatching" + + http_exception: HTTPException + + +class AbstractResource(Sized, Iterable["AbstractRoute"]): + def __init__(self, *, name: str | None = None) -> None: + self._name = name + + @property + def name(self) -> str | None: + return self._name + + @property + @abc.abstractmethod + def canonical(self) -> str: + """Exposes the resource's canonical path. + + For example '/foo/bar/{name}' + + """ + + @abc.abstractmethod # pragma: no branch + def url_for(self, **kwargs: str) -> URL: + """Construct url for resource with additional params.""" + + @abc.abstractmethod # pragma: no branch + async def resolve(self, request: Request) -> _Resolve: + """Resolve resource. + + Return (UrlMappingMatchInfo, allowed_methods) pair. + """ + + @abc.abstractmethod + def add_prefix(self, prefix: str) -> None: + """Add a prefix to processed URLs. + + Required for subapplications support. + """ + + @abc.abstractmethod + def get_info(self) -> _InfoDict: + """Return a dict with additional info useful for introspection""" + + def freeze(self) -> None: + pass + + @abc.abstractmethod + def raw_match(self, path: str) -> bool: + """Perform a raw match against path""" + + +class AbstractRoute(abc.ABC): + def __init__( + self, + method: str, + handler: Handler | type[AbstractView], + *, + expect_handler: _ExpectHandler | None = None, + resource: AbstractResource | None = None, + ) -> None: + + if expect_handler is None: + expect_handler = _default_expect_handler + + assert inspect.iscoroutinefunction(expect_handler) or ( + sys.version_info < (3, 14) and asyncio.iscoroutinefunction(expect_handler) + ), f"Coroutine is expected, got {expect_handler!r}" + + method = method.upper() + if not HTTP_METHOD_RE.match(method): + raise ValueError(f"{method} is not allowed HTTP method") + + assert callable(handler), handler + if inspect.iscoroutinefunction(handler) or ( + sys.version_info < (3, 14) and asyncio.iscoroutinefunction(handler) + ): + pass + elif inspect.isgeneratorfunction(handler): + if TYPE_CHECKING: + assert False + warnings.warn( + "Bare generators are deprecated, use @coroutine wrapper", + DeprecationWarning, + ) + elif isinstance(handler, type) and issubclass(handler, AbstractView): + pass + else: + warnings.warn( + "Bare functions are deprecated, use async ones", DeprecationWarning + ) + + @wraps(handler) + async def handler_wrapper(request: Request) -> StreamResponse: + result = old_handler(request) # type: ignore[call-arg] + if asyncio.iscoroutine(result): + result = await result + assert isinstance(result, StreamResponse) + return result + + old_handler = handler + handler = handler_wrapper + + self._method = method + self._handler = handler + self._expect_handler = expect_handler + self._resource = resource + + @property + def method(self) -> str: + return self._method + + @property + def handler(self) -> Handler: + return self._handler + + @property + @abc.abstractmethod + def name(self) -> str | None: + """Optional route's name, always equals to resource's name.""" + + @property + def resource(self) -> AbstractResource | None: + return self._resource + + @abc.abstractmethod + def get_info(self) -> _InfoDict: + """Return a dict with additional info useful for introspection""" + + @abc.abstractmethod # pragma: no branch + def url_for(self, *args: str, **kwargs: str) -> URL: + """Construct url for route with additional params.""" + + async def handle_expect_header(self, request: Request) -> StreamResponse | None: + return await self._expect_handler(request) + + +class UrlMappingMatchInfo(BaseDict, AbstractMatchInfo): + + __slots__ = ("_route", "_apps", "_current_app", "_frozen") + + def __init__(self, match_dict: dict[str, str], route: AbstractRoute) -> None: + super().__init__(match_dict) + self._route = route + self._apps: list[Application] = [] + self._current_app: Application | None = None + self._frozen = False + + @property + def handler(self) -> Handler: + return self._route.handler + + @property + def route(self) -> AbstractRoute: + return self._route + + @property + def expect_handler(self) -> _ExpectHandler: + return self._route.handle_expect_header + + @property + def http_exception(self) -> HTTPException | None: + return None + + def get_info(self) -> _InfoDict: # type: ignore[override] + return self._route.get_info() + + @property + def apps(self) -> tuple["Application", ...]: + return tuple(self._apps) + + def add_app(self, app: "Application") -> None: + if self._frozen: + raise RuntimeError("Cannot change apps stack after .freeze() call") + if self._current_app is None: + self._current_app = app + self._apps.insert(0, app) + + @property + def current_app(self) -> "Application": + app = self._current_app + assert app is not None + return app + + @current_app.setter + def current_app(self, app: "Application") -> None: + if DEBUG: # pragma: no cover + if app not in self._apps: + raise RuntimeError( + f"Expected one of the following apps {self._apps!r}, got {app!r}" + ) + self._current_app = app + + def freeze(self) -> None: + self._frozen = True + + def __repr__(self) -> str: + return f"" + + +class MatchInfoError(UrlMappingMatchInfo): + + __slots__ = ("_exception",) + + def __init__(self, http_exception: HTTPException) -> None: + self._exception = http_exception + super().__init__({}, SystemRoute(self._exception)) + + @property + def http_exception(self) -> HTTPException: + return self._exception + + def __repr__(self) -> str: + return f"" + + +async def _default_expect_handler(request: Request) -> None: + """Default handler for Expect header. + + Just send "100 Continue" to client. + raise HTTPExpectationFailed if value of header is not "100-continue" + """ + expect = request.headers.get(hdrs.EXPECT, "") + if request.version == HttpVersion11: + if expect.lower() == "100-continue": + await request.writer.write(b"HTTP/1.1 100 Continue\r\n\r\n") + # Reset output_size as we haven't started the main body yet. + request.writer.output_size = 0 + else: + raise HTTPExpectationFailed(text="Unknown Expect: %s" % expect) + + +class Resource(AbstractResource): + def __init__(self, *, name: str | None = None) -> None: + super().__init__(name=name) + self._routes: dict[str, ResourceRoute] = {} + self._any_route: ResourceRoute | None = None + self._allowed_methods: set[str] = set() + + def add_route( + self, + method: str, + handler: type[AbstractView] | Handler, + *, + expect_handler: _ExpectHandler | None = None, + ) -> "ResourceRoute": + if route := self._routes.get(method, self._any_route): + raise RuntimeError( + "Added route will never be executed, " + f"method {route.method} is already " + "registered" + ) + + route_obj = ResourceRoute(method, handler, self, expect_handler=expect_handler) + self.register_route(route_obj) + return route_obj + + def register_route(self, route: "ResourceRoute") -> None: + assert isinstance( + route, ResourceRoute + ), f"Instance of Route class is required, got {route!r}" + if route.method == hdrs.METH_ANY: + self._any_route = route + self._allowed_methods.add(route.method) + self._routes[route.method] = route + + async def resolve(self, request: Request) -> _Resolve: + if (match_dict := self._match(request.rel_url.path_safe)) is None: + return None, set() + if route := self._routes.get(request.method, self._any_route): + return UrlMappingMatchInfo(match_dict, route), self._allowed_methods + return None, self._allowed_methods + + @abc.abstractmethod + def _match(self, path: str) -> dict[str, str] | None: + pass # pragma: no cover + + def __len__(self) -> int: + return len(self._routes) + + def __iter__(self) -> Iterator["ResourceRoute"]: + return iter(self._routes.values()) + + # TODO: implement all abstract methods + + +class PlainResource(Resource): + def __init__(self, path: str, *, name: str | None = None) -> None: + super().__init__(name=name) + assert not path or path.startswith("/") + self._path = path + + @property + def canonical(self) -> str: + return self._path + + def freeze(self) -> None: + if not self._path: + self._path = "/" + + def add_prefix(self, prefix: str) -> None: + assert prefix.startswith("/") + assert not prefix.endswith("/") + assert len(prefix) > 1 + self._path = prefix + self._path + + def _match(self, path: str) -> dict[str, str] | None: + # string comparison is about 10 times faster than regexp matching + if self._path == path: + return {} + return None + + def raw_match(self, path: str) -> bool: + return self._path == path + + def get_info(self) -> _InfoDict: + return {"path": self._path} + + def url_for(self) -> URL: # type: ignore[override] + return URL.build(path=self._path, encoded=True) + + def __repr__(self) -> str: + name = "'" + self.name + "' " if self.name is not None else "" + return f"" + + +class DynamicResource(Resource): + + DYN = re.compile(r"\{(?P[_a-zA-Z][_a-zA-Z0-9]*)\}") + DYN_WITH_RE = re.compile(r"\{(?P[_a-zA-Z][_a-zA-Z0-9]*):(?P.+)\}") + GOOD = r"[^{}/]+" + + def __init__(self, path: str, *, name: str | None = None) -> None: + super().__init__(name=name) + self._orig_path = path + pattern = "" + formatter = "" + for part in ROUTE_RE.split(path): + match = self.DYN.fullmatch(part) + if match: + pattern += "(?P<{}>{})".format(match.group("var"), self.GOOD) + formatter += "{" + match.group("var") + "}" + continue + + match = self.DYN_WITH_RE.fullmatch(part) + if match: + pattern += "(?P<{var}>{re})".format(**match.groupdict()) + formatter += "{" + match.group("var") + "}" + continue + + if "{" in part or "}" in part: + raise ValueError(f"Invalid path '{path}'['{part}']") + + part = _requote_path(part) + formatter += part + pattern += re.escape(part) + + try: + compiled = re.compile(pattern) + except re.error as exc: + raise ValueError(f"Bad pattern '{pattern}': {exc}") from None + assert compiled.pattern.startswith(PATH_SEP) + assert formatter.startswith("/") + self._pattern = compiled + self._formatter = formatter + + @property + def canonical(self) -> str: + return self._formatter + + def add_prefix(self, prefix: str) -> None: + assert prefix.startswith("/") + assert not prefix.endswith("/") + assert len(prefix) > 1 + self._pattern = re.compile(re.escape(prefix) + self._pattern.pattern) + self._formatter = prefix + self._formatter + + def _match(self, path: str) -> dict[str, str] | None: + match = self._pattern.fullmatch(path) + if match is None: + return None + return { + key: _unquote_path_safe(value) for key, value in match.groupdict().items() + } + + def raw_match(self, path: str) -> bool: + return self._orig_path == path + + def get_info(self) -> _InfoDict: + return {"formatter": self._formatter, "pattern": self._pattern} + + def url_for(self, **parts: str) -> URL: + url = self._formatter.format_map({k: _quote_path(v) for k, v in parts.items()}) + return URL.build(path=url, encoded=True) + + def __repr__(self) -> str: + name = "'" + self.name + "' " if self.name is not None else "" + return f"" + + +class PrefixResource(AbstractResource): + def __init__(self, prefix: str, *, name: str | None = None) -> None: + assert not prefix or prefix.startswith("/"), prefix + assert prefix in ("", "/") or not prefix.endswith("/"), prefix + super().__init__(name=name) + self._prefix = _requote_path(prefix) + self._prefix2 = self._prefix + "/" + + @property + def canonical(self) -> str: + return self._prefix + + def add_prefix(self, prefix: str) -> None: + assert prefix.startswith("/") + assert not prefix.endswith("/") + assert len(prefix) > 1 + self._prefix = prefix + self._prefix + self._prefix2 = self._prefix + "/" + + def raw_match(self, prefix: str) -> bool: + return False + + # TODO: impl missing abstract methods + + +class StaticResource(PrefixResource): + VERSION_KEY = "v" + + def __init__( + self, + prefix: str, + directory: PathLike, + *, + name: str | None = None, + expect_handler: _ExpectHandler | None = None, + chunk_size: int = DEFAULT_CHUNK_SIZE, + show_index: bool = False, + follow_symlinks: bool = False, + append_version: bool = False, + ) -> None: + super().__init__(prefix, name=name) + try: + directory = Path(directory).expanduser().resolve(strict=True) + except FileNotFoundError as error: + raise ValueError(f"'{directory}' does not exist") from error + if not directory.is_dir(): + raise ValueError(f"'{directory}' is not a directory") + self._directory = directory + self._show_index = show_index + self._chunk_size = chunk_size + self._follow_symlinks = follow_symlinks + self._expect_handler = expect_handler + self._append_version = append_version + + self._routes = { + "GET": ResourceRoute( + "GET", self._handle, self, expect_handler=expect_handler + ), + "HEAD": ResourceRoute( + "HEAD", self._handle, self, expect_handler=expect_handler + ), + } + self._allowed_methods = set(self._routes) + + def url_for( # type: ignore[override] + self, + *, + filename: PathLike, + append_version: bool | None = None, + ) -> URL: + if append_version is None: + append_version = self._append_version + filename = str(filename).lstrip("/") + + url = URL.build(path=self._prefix, encoded=True) + # filename is not encoded + if YARL_VERSION < (1, 6): + url = url / filename.replace("%", "%25") + else: + url = url / filename + + if append_version: + unresolved_path = self._directory.joinpath(filename) + try: + if self._follow_symlinks: + normalized_path = Path(os.path.normpath(unresolved_path)) + normalized_path.relative_to(self._directory) + filepath = normalized_path.resolve() + else: + filepath = unresolved_path.resolve() + filepath.relative_to(self._directory) + except (ValueError, FileNotFoundError): + # ValueError for case when path point to symlink + # with follow_symlinks is False + return url # relatively safe + if filepath.is_file(): + # TODO cache file content + # with file watcher for cache invalidation + with filepath.open("rb") as f: + file_bytes = f.read() + h = self._get_file_hash(file_bytes) + url = url.with_query({self.VERSION_KEY: h}) + return url + return url + + @staticmethod + def _get_file_hash(byte_array: bytes) -> str: + m = hashlib.sha256() # todo sha256 can be configurable param + m.update(byte_array) + b64 = base64.urlsafe_b64encode(m.digest()) + return b64.decode("ascii") + + def get_info(self) -> _InfoDict: + return { + "directory": self._directory, + "prefix": self._prefix, + "routes": self._routes, + } + + def set_options_route(self, handler: Handler) -> None: + if "OPTIONS" in self._routes: + raise RuntimeError("OPTIONS route was set already") + self._routes["OPTIONS"] = ResourceRoute( + "OPTIONS", handler, self, expect_handler=self._expect_handler + ) + self._allowed_methods.add("OPTIONS") + + async def resolve(self, request: Request) -> _Resolve: + path = request.rel_url.path_safe + method = request.method + # We normalise here to avoid matches that traverse below the static root. + # e.g. /static/../../../../home/user/webapp/static/ + norm_path = os.path.normpath(path) + if IS_WINDOWS: + norm_path = norm_path.replace("\\", "/") + if not norm_path.startswith(self._prefix2) and norm_path != self._prefix: + return None, set() + + allowed_methods = self._allowed_methods + if method not in allowed_methods: + return None, allowed_methods + + match_dict = {"filename": _unquote_path_safe(path[len(self._prefix) + 1 :])} + return (UrlMappingMatchInfo(match_dict, self._routes[method]), allowed_methods) + + def __len__(self) -> int: + return len(self._routes) + + def __iter__(self) -> Iterator[AbstractRoute]: + return iter(self._routes.values()) + + async def _handle(self, request: Request) -> StreamResponse: + filename = request.match_info["filename"] + if Path(filename).is_absolute(): + # filename is an absolute path e.g. //network/share or D:\path + # which could be a UNC path leading to NTLM credential theft + raise HTTPNotFound() + unresolved_path = self._directory.joinpath(filename) + loop = asyncio.get_running_loop() + return await loop.run_in_executor( + None, self._resolve_path_to_response, unresolved_path + ) + + def _resolve_path_to_response(self, unresolved_path: Path) -> StreamResponse: + """Take the unresolved path and query the file system to form a response.""" + # Check for access outside the root directory. For follow symlinks, URI + # cannot traverse out, but symlinks can. Otherwise, no access outside + # root is permitted. + try: + if self._follow_symlinks: + normalized_path = Path(os.path.normpath(unresolved_path)) + normalized_path.relative_to(self._directory) + file_path = normalized_path.resolve() + else: + file_path = unresolved_path.resolve() + file_path.relative_to(self._directory) + except (ValueError, *CIRCULAR_SYMLINK_ERROR) as error: + # ValueError is raised for the relative check. Circular symlinks + # raise here on resolving for python < 3.13. + raise HTTPNotFound() from error + + # if path is a directory, return the contents if permitted. Note the + # directory check will raise if a segment is not readable. + try: + if file_path.is_dir(): + if self._show_index: + return Response( + text=self._directory_as_html(file_path), + content_type="text/html", + ) + else: + raise HTTPForbidden() + except PermissionError as error: + raise HTTPForbidden() from error + + # Return the file response, which handles all other checks. + return FileResponse(file_path, chunk_size=self._chunk_size) + + def _directory_as_html(self, dir_path: Path) -> str: + """returns directory's index as html.""" + assert dir_path.is_dir() + + relative_path_to_dir = dir_path.relative_to(self._directory).as_posix() + index_of = f"Index of /{html_escape(relative_path_to_dir)}" + h1 = f"

{index_of}

" + + index_list = [] + dir_index = dir_path.iterdir() + for _file in sorted(dir_index): + # show file url as relative to static path + rel_path = _file.relative_to(self._directory).as_posix() + quoted_file_url = _quote_path(f"{self._prefix}/{rel_path}") + + # if file is a directory, add '/' to the end of the name + if _file.is_dir(): + file_name = f"{_file.name}/" + else: + file_name = _file.name + + index_list.append( + f'
  • {html_escape(file_name)}
    • ' + ) + ul = "
      \n{}\n
    ".format("\n".join(index_list)) + body = f"\n{h1}\n{ul}\n" + + head_str = f"\n{index_of}\n" + html = f"\n{head_str}\n{body}\n" + + return html + + def __repr__(self) -> str: + name = "'" + self.name + "'" if self.name is not None else "" + return f" {self._directory!r}>" + + +class PrefixedSubAppResource(PrefixResource): + def __init__(self, prefix: str, app: "Application") -> None: + super().__init__(prefix) + self._app = app + self._add_prefix_to_resources(prefix) + + def add_prefix(self, prefix: str) -> None: + super().add_prefix(prefix) + self._add_prefix_to_resources(prefix) + + def _add_prefix_to_resources(self, prefix: str) -> None: + router = self._app.router + for resource in router.resources(): + # Since the canonical path of a resource is about + # to change, we need to unindex it and then reindex + router.unindex_resource(resource) + resource.add_prefix(prefix) + router.index_resource(resource) + + def url_for(self, *args: str, **kwargs: str) -> URL: + raise RuntimeError(".url_for() is not supported by sub-application root") + + def get_info(self) -> _InfoDict: + return {"app": self._app, "prefix": self._prefix} + + async def resolve(self, request: Request) -> _Resolve: + match_info = await self._app.router.resolve(request) + match_info.add_app(self._app) + if isinstance(match_info.http_exception, HTTPMethodNotAllowed): + methods = match_info.http_exception.allowed_methods + else: + methods = set() + return match_info, methods + + def __len__(self) -> int: + return len(self._app.router.routes()) + + def __iter__(self) -> Iterator[AbstractRoute]: + return iter(self._app.router.routes()) + + def __repr__(self) -> str: + return f" {self._app!r}>" + + +class AbstractRuleMatching(abc.ABC): + @abc.abstractmethod # pragma: no branch + async def match(self, request: Request) -> bool: + """Return bool if the request satisfies the criteria""" + + @abc.abstractmethod # pragma: no branch + def get_info(self) -> _InfoDict: + """Return a dict with additional info useful for introspection""" + + @property + @abc.abstractmethod # pragma: no branch + def canonical(self) -> str: + """Return a str""" + + +class Domain(AbstractRuleMatching): + re_part = re.compile(r"(?!-)[a-z\d-]{1,63}(? None: + super().__init__() + self._domain = self.validation(domain) + + @property + def canonical(self) -> str: + return self._domain + + def validation(self, domain: str) -> str: + if not isinstance(domain, str): + raise TypeError("Domain must be str") + domain = domain.rstrip(".").lower() + if not domain: + raise ValueError("Domain cannot be empty") + elif "://" in domain: + raise ValueError("Scheme not supported") + url = URL("http://" + domain) + assert url.raw_host is not None + if not all(self.re_part.fullmatch(x) for x in url.raw_host.split(".")): + raise ValueError("Domain not valid") + if url.port == 80: + return url.raw_host + return f"{url.raw_host}:{url.port}" + + async def match(self, request: Request) -> bool: + host = request.headers.get(hdrs.HOST) + if not host: + return False + return self.match_domain(host) + + def match_domain(self, host: str) -> bool: + return host.lower() == self._domain + + def get_info(self) -> _InfoDict: + return {"domain": self._domain} + + +class MaskDomain(Domain): + re_part = re.compile(r"(?!-)[a-z\d\*-]{1,63}(? None: + super().__init__(domain) + mask = self._domain.replace(".", r"\.").replace("*", ".*") + self._mask = re.compile(mask) + + @property + def canonical(self) -> str: + return self._mask.pattern + + def match_domain(self, host: str) -> bool: + return self._mask.fullmatch(host) is not None + + +class MatchedSubAppResource(PrefixedSubAppResource): + def __init__(self, rule: AbstractRuleMatching, app: "Application") -> None: + AbstractResource.__init__(self) + self._prefix = "" + self._app = app + self._rule = rule + + @property + def canonical(self) -> str: + return self._rule.canonical + + def get_info(self) -> _InfoDict: + return {"app": self._app, "rule": self._rule} + + async def resolve(self, request: Request) -> _Resolve: + if not await self._rule.match(request): + return None, set() + match_info = await self._app.router.resolve(request) + match_info.add_app(self._app) + if isinstance(match_info.http_exception, HTTPMethodNotAllowed): + methods = match_info.http_exception.allowed_methods + else: + methods = set() + return match_info, methods + + def __repr__(self) -> str: + return f" {self._app!r}>" + + +class ResourceRoute(AbstractRoute): + """A route with resource""" + + def __init__( + self, + method: str, + handler: Handler | type[AbstractView], + resource: AbstractResource, + *, + expect_handler: _ExpectHandler | None = None, + ) -> None: + super().__init__( + method, handler, expect_handler=expect_handler, resource=resource + ) + + def __repr__(self) -> str: + return f" {self.handler!r}" + + @property + def name(self) -> str | None: + if self._resource is None: + return None + return self._resource.name + + def url_for(self, *args: str, **kwargs: str) -> URL: + """Construct url for route with additional params.""" + assert self._resource is not None + return self._resource.url_for(*args, **kwargs) + + def get_info(self) -> _InfoDict: + assert self._resource is not None + return self._resource.get_info() + + +class SystemRoute(AbstractRoute): + def __init__(self, http_exception: HTTPException) -> None: + super().__init__(hdrs.METH_ANY, self._handle) + self._http_exception = http_exception + + def url_for(self, *args: str, **kwargs: str) -> URL: + raise RuntimeError(".url_for() is not allowed for SystemRoute") + + @property + def name(self) -> str | None: + return None + + def get_info(self) -> _InfoDict: + return {"http_exception": self._http_exception} + + async def _handle(self, request: Request) -> StreamResponse: + raise self._http_exception + + @property + def status(self) -> int: + return self._http_exception.status + + @property + def reason(self) -> str: + return self._http_exception.reason + + def __repr__(self) -> str: + return f"" + + +class View(AbstractView): + async def _iter(self) -> StreamResponse: + if self.request.method not in hdrs.METH_ALL: + self._raise_allowed_methods() + method: Callable[[], Awaitable[StreamResponse]] | None + method = getattr(self, self.request.method.lower(), None) + if method is None: + self._raise_allowed_methods() + ret = await method() + assert isinstance(ret, StreamResponse) + return ret + + def __await__(self) -> Generator[None, None, StreamResponse]: + return self._iter().__await__() + + def _raise_allowed_methods(self) -> NoReturn: + allowed_methods = {m for m in hdrs.METH_ALL if hasattr(self, m.lower())} + raise HTTPMethodNotAllowed(self.request.method, allowed_methods) + + +class ResourcesView(Sized, Iterable[AbstractResource], Container[AbstractResource]): + def __init__(self, resources: list[AbstractResource]) -> None: + self._resources = resources + + def __len__(self) -> int: + return len(self._resources) + + def __iter__(self) -> Iterator[AbstractResource]: + yield from self._resources + + def __contains__(self, resource: object) -> bool: + return resource in self._resources + + +class RoutesView(Sized, Iterable[AbstractRoute], Container[AbstractRoute]): + def __init__(self, resources: list[AbstractResource]): + self._routes: list[AbstractRoute] = [] + for resource in resources: + for route in resource: + self._routes.append(route) + + def __len__(self) -> int: + return len(self._routes) + + def __iter__(self) -> Iterator[AbstractRoute]: + yield from self._routes + + def __contains__(self, route: object) -> bool: + return route in self._routes + + +class UrlDispatcher(AbstractRouter, Mapping[str, AbstractResource]): + + NAME_SPLIT_RE = re.compile(r"[.:-]") + + def __init__(self) -> None: + super().__init__() + self._resources: list[AbstractResource] = [] + self._named_resources: dict[str, AbstractResource] = {} + self._resource_index: dict[str, list[AbstractResource]] = {} + self._matched_sub_app_resources: list[MatchedSubAppResource] = [] + + async def resolve(self, request: Request) -> UrlMappingMatchInfo: + resource_index = self._resource_index + allowed_methods: set[str] = set() + + # MatchedSubAppResource is primarily used to match on domain names + # (though custom rules could match on other things). This means that + # the traversal algorithm below can't be applied, and that we likely + # need to check these first so a sub app that defines the same path + # as a parent app will get priority if there's a domain match. + # + # For most cases we do not expect there to be many of these since + # currently they are only added by `.add_domain()`. + for resource in self._matched_sub_app_resources: + match_dict, allowed = await resource.resolve(request) + if match_dict is not None: + return match_dict + else: + allowed_methods |= allowed + + # Walk the url parts looking for candidates. We walk the url backwards + # to ensure the most explicit match is found first. If there are multiple + # candidates for a given url part because there are multiple resources + # registered for the same canonical path, we resolve them in a linear + # fashion to ensure registration order is respected. + url_part = request.rel_url.path_safe + while url_part: + for candidate in resource_index.get(url_part, ()): + match_dict, allowed = await candidate.resolve(request) + if match_dict is not None: + return match_dict + else: + allowed_methods |= allowed + if url_part == "/": + break + url_part = url_part.rpartition("/")[0] or "/" + + if allowed_methods: + return MatchInfoError(HTTPMethodNotAllowed(request.method, allowed_methods)) + + return MatchInfoError(HTTPNotFound()) + + def __iter__(self) -> Iterator[str]: + return iter(self._named_resources) + + def __len__(self) -> int: + return len(self._named_resources) + + def __contains__(self, resource: object) -> bool: + return resource in self._named_resources + + def __getitem__(self, name: str) -> AbstractResource: + return self._named_resources[name] + + def resources(self) -> ResourcesView: + return ResourcesView(self._resources) + + def routes(self) -> RoutesView: + return RoutesView(self._resources) + + def named_resources(self) -> Mapping[str, AbstractResource]: + return MappingProxyType(self._named_resources) + + def register_resource(self, resource: AbstractResource) -> None: + assert isinstance( + resource, AbstractResource + ), f"Instance of AbstractResource class is required, got {resource!r}" + if self.frozen: + raise RuntimeError("Cannot register a resource into frozen router.") + + name = resource.name + + if name is not None: + parts = self.NAME_SPLIT_RE.split(name) + for part in parts: + if keyword.iskeyword(part): + raise ValueError( + f"Incorrect route name {name!r}, " + "python keywords cannot be used " + "for route name" + ) + if not part.isidentifier(): + raise ValueError( + f"Incorrect route name {name!r}, " + "the name should be a sequence of " + "python identifiers separated " + "by dash, dot or column" + ) + if name in self._named_resources: + raise ValueError( + f"Duplicate {name!r}, " + f"already handled by {self._named_resources[name]!r}" + ) + self._named_resources[name] = resource + self._resources.append(resource) + + if isinstance(resource, MatchedSubAppResource): + # We cannot index match sub-app resources because they have match rules + self._matched_sub_app_resources.append(resource) + else: + self.index_resource(resource) + + def _get_resource_index_key(self, resource: AbstractResource) -> str: + """Return a key to index the resource in the resource index.""" + if "{" in (index_key := resource.canonical): + # strip at the first { to allow for variables, and than + # rpartition at / to allow for variable parts in the path + # For example if the canonical path is `/core/locations{tail:.*}` + # the index key will be `/core` since index is based on the + # url parts split by `/` + index_key = index_key.partition("{")[0].rpartition("/")[0] + return index_key.rstrip("/") or "/" + + def index_resource(self, resource: AbstractResource) -> None: + """Add a resource to the resource index.""" + resource_key = self._get_resource_index_key(resource) + # There may be multiple resources for a canonical path + # so we keep them in a list to ensure that registration + # order is respected. + self._resource_index.setdefault(resource_key, []).append(resource) + + def unindex_resource(self, resource: AbstractResource) -> None: + """Remove a resource from the resource index.""" + resource_key = self._get_resource_index_key(resource) + self._resource_index[resource_key].remove(resource) + + def add_resource(self, path: str, *, name: str | None = None) -> Resource: + if path and not path.startswith("/"): + raise ValueError("path should be started with / or be empty") + # Reuse last added resource if path and name are the same + if self._resources: + resource = self._resources[-1] + if resource.name == name and resource.raw_match(path): + return cast(Resource, resource) + if not ("{" in path or "}" in path or ROUTE_RE.search(path)): + resource = PlainResource(path, name=name) + self.register_resource(resource) + return resource + resource = DynamicResource(path, name=name) + self.register_resource(resource) + return resource + + def add_route( + self, + method: str, + path: str, + handler: Handler | type[AbstractView], + *, + name: str | None = None, + expect_handler: _ExpectHandler | None = None, + ) -> AbstractRoute: + resource = self.add_resource(path, name=name) + return resource.add_route(method, handler, expect_handler=expect_handler) + + def add_static( + self, + prefix: str, + path: PathLike, + *, + name: str | None = None, + expect_handler: _ExpectHandler | None = None, + chunk_size: int = DEFAULT_CHUNK_SIZE, + show_index: bool = False, + follow_symlinks: bool = False, + append_version: bool = False, + ) -> AbstractResource: + """Add static files view. + + prefix - url prefix + path - folder with files + + """ + assert prefix.startswith("/") + if prefix.endswith("/"): + prefix = prefix[:-1] + resource = StaticResource( + prefix, + path, + name=name, + expect_handler=expect_handler, + chunk_size=chunk_size, + show_index=show_index, + follow_symlinks=follow_symlinks, + append_version=append_version, + ) + self.register_resource(resource) + return resource + + def add_head(self, path: str, handler: Handler, **kwargs: Any) -> AbstractRoute: + """Shortcut for add_route with method HEAD.""" + return self.add_route(hdrs.METH_HEAD, path, handler, **kwargs) + + def add_options(self, path: str, handler: Handler, **kwargs: Any) -> AbstractRoute: + """Shortcut for add_route with method OPTIONS.""" + return self.add_route(hdrs.METH_OPTIONS, path, handler, **kwargs) + + def add_get( + self, + path: str, + handler: Handler, + *, + name: str | None = None, + allow_head: bool = True, + **kwargs: Any, + ) -> AbstractRoute: + """Shortcut for add_route with method GET. + + If allow_head is true, another + route is added allowing head requests to the same endpoint. + """ + resource = self.add_resource(path, name=name) + if allow_head: + resource.add_route(hdrs.METH_HEAD, handler, **kwargs) + return resource.add_route(hdrs.METH_GET, handler, **kwargs) + + def add_post(self, path: str, handler: Handler, **kwargs: Any) -> AbstractRoute: + """Shortcut for add_route with method POST.""" + return self.add_route(hdrs.METH_POST, path, handler, **kwargs) + + def add_put(self, path: str, handler: Handler, **kwargs: Any) -> AbstractRoute: + """Shortcut for add_route with method PUT.""" + return self.add_route(hdrs.METH_PUT, path, handler, **kwargs) + + def add_patch(self, path: str, handler: Handler, **kwargs: Any) -> AbstractRoute: + """Shortcut for add_route with method PATCH.""" + return self.add_route(hdrs.METH_PATCH, path, handler, **kwargs) + + def add_delete(self, path: str, handler: Handler, **kwargs: Any) -> AbstractRoute: + """Shortcut for add_route with method DELETE.""" + return self.add_route(hdrs.METH_DELETE, path, handler, **kwargs) + + def add_view( + self, path: str, handler: type[AbstractView], **kwargs: Any + ) -> AbstractRoute: + """Shortcut for add_route with ANY methods for a class-based view.""" + return self.add_route(hdrs.METH_ANY, path, handler, **kwargs) + + def freeze(self) -> None: + super().freeze() + for resource in self._resources: + resource.freeze() + + def add_routes(self, routes: Iterable[AbstractRouteDef]) -> list[AbstractRoute]: + """Append routes to route table. + + Parameter should be a sequence of RouteDef objects. + + Returns a list of registered AbstractRoute instances. + """ + registered_routes = [] + for route_def in routes: + registered_routes.extend(route_def.register(self)) + return registered_routes + + +def _quote_path(value: str) -> str: + if YARL_VERSION < (1, 6): + value = value.replace("%", "%25") + return URL.build(path=value, encoded=False).raw_path + + +def _unquote_path_safe(value: str) -> str: + if "%" not in value: + return value + return value.replace("%2F", "/").replace("%25", "%") + + +def _requote_path(value: str) -> str: + # Quote non-ascii characters and other characters which must be quoted, + # but preserve existing %-sequences. + result = _quote_path(value) + if "%" in value: + result = result.replace("%25", "%") + return result diff --git a/py_modules/aiohttp/web_ws.py b/py_modules/aiohttp/web_ws.py new file mode 100644 index 0000000..f4cef84 --- /dev/null +++ b/py_modules/aiohttp/web_ws.py @@ -0,0 +1,783 @@ +import asyncio +import base64 +import binascii +import hashlib +import json +import sys +from collections.abc import Callable, Iterable +from typing import Any, Final, Generic, Literal, cast, overload + +import attr +from multidict import CIMultiDict + +from . import hdrs +from ._websocket.reader import WebSocketDataQueue +from .abc import AbstractStreamWriter +from .client_exceptions import WSMessageTypeError +from .helpers import ( + DEFAULT_CHUNK_SIZE, + calculate_timeout_when, + set_exception, + set_result, +) +from .http import ( + WS_CLOSED_MESSAGE, + WS_CLOSING_MESSAGE, + WS_KEY, + WebSocketError, + WebSocketReader, + WebSocketWriter, + WSCloseCode, + WSMessage, + WSMessageDecodeText, + WSMessageNoDecodeText, + WSMsgType as WSMsgType, + ws_ext_gen, + ws_ext_parse, +) +from .http_websocket import _INTERNAL_RECEIVE_TYPES +from .log import ws_logger +from .streams import EofStream +from .typedefs import JSONBytesEncoder, JSONDecoder, JSONEncoder +from .web_exceptions import HTTPBadRequest, HTTPException +from .web_request import BaseRequest +from .web_response import StreamResponse + +if sys.version_info >= (3, 13): + from typing import TypeVar +else: + from typing_extensions import TypeVar + +if sys.version_info >= (3, 12): + from collections.abc import Buffer +else: + from typing import Union + + Buffer = Union[bytes, bytearray, "memoryview[int]", "memoryview[bytes]"] + +if sys.version_info >= (3, 11): + import asyncio as async_timeout + from typing import Self +else: + import async_timeout + from typing_extensions import Self + +__all__ = ( + "WebSocketResponse", + "WebSocketReady", + "WSMsgType", +) + +THRESHOLD_CONNLOST_ACCESS: Final[int] = 5 + +# TypeVar for whether text messages are decoded to str (True) or kept as bytes (False) +_DecodeText = TypeVar("_DecodeText", bound=bool, covariant=True, default=Literal[True]) + + +@attr.s(auto_attribs=True, frozen=True, slots=True) +class WebSocketReady: + ok: bool + protocol: str | None + + def __bool__(self) -> bool: + return self.ok + + +class WebSocketResponse(StreamResponse, Generic[_DecodeText]): + + _length_check: bool = False + _ws_protocol: str | None = None + _writer: WebSocketWriter | None = None + _reader: WebSocketDataQueue | None = None + _closed: bool = False + _closing: bool = False + _conn_lost: int = 0 + _close_code: int | None = None + _loop: asyncio.AbstractEventLoop | None = None + _waiting: bool = False + _close_wait: asyncio.Future[None] | None = None + _exception: BaseException | None = None + _heartbeat_when: float = 0.0 + _heartbeat_cb: asyncio.TimerHandle | None = None + _pong_response_cb: asyncio.TimerHandle | None = None + _ping_task: asyncio.Task[None] | None = None + _need_heartbeat_reset: bool = False + _heartbeat_reset_handle: asyncio.Handle | None = None + + def __init__( + self, + *, + timeout: float = 10.0, + receive_timeout: float | None = None, + autoclose: bool = True, + autoping: bool = True, + heartbeat: float | None = None, + protocols: Iterable[str] = (), + compress: bool = True, + max_msg_size: int = 4 * 1024 * 1024, + writer_limit: int = DEFAULT_CHUNK_SIZE, + decode_text: bool = True, + ) -> None: + super().__init__(status=101) + self._protocols = protocols + self._timeout = timeout + self._receive_timeout = receive_timeout + self._autoclose = autoclose + self._autoping = autoping + self._heartbeat = heartbeat + if heartbeat is not None: + self._pong_heartbeat = heartbeat / 2.0 + self._compress: bool | int = compress + self._max_msg_size = max_msg_size + self._writer_limit = writer_limit + self._decode_text = decode_text + self._need_heartbeat_reset = False + self._heartbeat_reset_handle = None + + def _cancel_heartbeat(self) -> None: + self._cancel_pong_response_cb() + if self._heartbeat_reset_handle is not None: + self._heartbeat_reset_handle.cancel() + self._heartbeat_reset_handle = None + self._need_heartbeat_reset = False + if self._heartbeat_cb is not None: + self._heartbeat_cb.cancel() + self._heartbeat_cb = None + if self._ping_task is not None: + self._ping_task.cancel() + self._ping_task = None + + def _cancel_pong_response_cb(self) -> None: + if self._pong_response_cb is not None: + self._pong_response_cb.cancel() + self._pong_response_cb = None + + def _on_data_received(self) -> None: + if self._heartbeat is None or self._need_heartbeat_reset: + return + loop = self._loop + assert loop is not None + # Coalesce multiple chunks received in the same loop tick into a single + # heartbeat reset. Resetting immediately per chunk increases timer churn. + self._need_heartbeat_reset = True + self._heartbeat_reset_handle = loop.call_soon(self._flush_heartbeat_reset) + + def _flush_heartbeat_reset(self) -> None: + self._heartbeat_reset_handle = None + if not self._need_heartbeat_reset: + return + self._reset_heartbeat() + self._need_heartbeat_reset = False + + def _reset_heartbeat(self) -> None: + if self._heartbeat is None: + return + self._cancel_pong_response_cb() + req = self._req + timeout_ceil_threshold = ( + req._protocol._timeout_ceil_threshold if req is not None else 5 + ) + loop = self._loop + assert loop is not None + now = loop.time() + when = calculate_timeout_when(now, self._heartbeat, timeout_ceil_threshold) + self._heartbeat_when = when + if self._heartbeat_cb is None: + # We do not cancel the previous heartbeat_cb here because + # it generates a significant amount of TimerHandle churn + # which causes asyncio to rebuild the heap frequently. + # Instead _send_heartbeat() will reschedule the next + # heartbeat if it fires too early. + self._heartbeat_cb = loop.call_at(when, self._send_heartbeat) + + def _send_heartbeat(self) -> None: + self._heartbeat_cb = None + + # If heartbeat reset is pending (data is being received), skip sending + # the ping and let the reset callback handle rescheduling the heartbeat. + if self._need_heartbeat_reset: + return + + loop = self._loop + assert loop is not None and self._writer is not None + now = loop.time() + if now < self._heartbeat_when: + # Heartbeat fired too early, reschedule + self._heartbeat_cb = loop.call_at( + self._heartbeat_when, self._send_heartbeat + ) + return + + req = self._req + timeout_ceil_threshold = ( + req._protocol._timeout_ceil_threshold if req is not None else 5 + ) + when = calculate_timeout_when(now, self._pong_heartbeat, timeout_ceil_threshold) + self._cancel_pong_response_cb() + self._pong_response_cb = loop.call_at(when, self._pong_not_received) + + coro = self._writer.send_frame(b"", WSMsgType.PING) + if sys.version_info >= (3, 12): + # Optimization for Python 3.12, try to send the ping + # immediately to avoid having to schedule + # the task on the event loop. + ping_task = asyncio.Task(coro, loop=loop, eager_start=True) + else: + ping_task = loop.create_task(coro) + + if not ping_task.done(): + self._ping_task = ping_task + ping_task.add_done_callback(self._ping_task_done) + else: + self._ping_task_done(ping_task) + + def _ping_task_done(self, task: "asyncio.Task[None]") -> None: + """Callback for when the ping task completes.""" + if not task.cancelled() and (exc := task.exception()): + self._handle_ping_pong_exception(exc) + self._ping_task = None + + def _pong_not_received(self) -> None: + if self._req is not None and self._req.transport is not None: + self._handle_ping_pong_exception( + asyncio.TimeoutError( + f"No PONG received after {self._pong_heartbeat} seconds" + ) + ) + + def _handle_ping_pong_exception(self, exc: BaseException) -> None: + """Handle exceptions raised during ping/pong processing.""" + if self._closed: + return + self._set_closed() + self._set_code_close_transport(WSCloseCode.ABNORMAL_CLOSURE) + self._exception = exc + if self._waiting and not self._closing and self._reader is not None: + self._reader.feed_data(WSMessage(WSMsgType.ERROR, exc, None), 0) + + def _set_closed(self) -> None: + """Set the connection to closed. + + Cancel any heartbeat timers and set the closed flag. + """ + self._closed = True + self._cancel_heartbeat() + + async def prepare(self, request: BaseRequest) -> AbstractStreamWriter: + # make pre-check to don't hide it by do_handshake() exceptions + if self._payload_writer is not None: + return self._payload_writer + + protocol, writer = self._pre_start(request) + payload_writer = await super().prepare(request) + assert payload_writer is not None + self._post_start(request, protocol, writer) + await payload_writer.drain() + return payload_writer + + def _handshake( + self, request: BaseRequest + ) -> tuple["CIMultiDict[str]", str | None, int, bool]: + headers = request.headers + if "websocket" != headers.get(hdrs.UPGRADE, "").lower().strip(): + raise HTTPBadRequest( + text=( + f"No WebSocket UPGRADE hdr: {headers.get(hdrs.UPGRADE)}\n Can " + '"Upgrade" only to "WebSocket".' + ) + ) + + if not request._message.upgrade: + raise HTTPBadRequest( + text=f"No CONNECTION upgrade hdr: {headers.get(hdrs.CONNECTION)}" + ) + + # find common sub-protocol between client and server + protocol: str | None = None + if hdrs.SEC_WEBSOCKET_PROTOCOL in headers: + req_protocols = [ + str(proto.strip()) + for proto in headers[hdrs.SEC_WEBSOCKET_PROTOCOL].split(",") + ] + + for proto in req_protocols: + if proto in self._protocols: + protocol = proto + break + else: + # No overlap found: Return no protocol as per spec + ws_logger.warning( + "%s: Client protocols %r don’t overlap server-known ones %r", + request.remote, + req_protocols, + self._protocols, + ) + + # check supported version + version = headers.get(hdrs.SEC_WEBSOCKET_VERSION, "") + if version not in ("13", "8", "7"): + raise HTTPBadRequest(text=f"Unsupported version: {version}") + + # check client handshake for validity + key = headers.get(hdrs.SEC_WEBSOCKET_KEY) + try: + if not key or len(base64.b64decode(key)) != 16: + raise HTTPBadRequest(text=f"Handshake error: {key!r}") + except binascii.Error: + raise HTTPBadRequest(text=f"Handshake error: {key!r}") from None + + accept_val = base64.b64encode( + hashlib.sha1(key.encode() + WS_KEY).digest() + ).decode() + response_headers = CIMultiDict( + { + hdrs.UPGRADE: "websocket", + hdrs.CONNECTION: "upgrade", + hdrs.SEC_WEBSOCKET_ACCEPT: accept_val, + } + ) + + notakeover = False + compress = 0 + if self._compress: + extensions = headers.get(hdrs.SEC_WEBSOCKET_EXTENSIONS) + # Server side always get return with no exception. + # If something happened, just drop compress extension + compress, notakeover = ws_ext_parse(extensions, isserver=True) + if compress: + enabledext = ws_ext_gen( + compress=compress, isserver=True, server_notakeover=notakeover + ) + response_headers[hdrs.SEC_WEBSOCKET_EXTENSIONS] = enabledext + + if protocol: + response_headers[hdrs.SEC_WEBSOCKET_PROTOCOL] = protocol + return ( + response_headers, + protocol, + compress, + notakeover, + ) + + def _pre_start(self, request: BaseRequest) -> tuple[str | None, WebSocketWriter]: + self._loop = request._loop + + headers, protocol, compress, notakeover = self._handshake(request) + + self.set_status(101) + self.headers.update(headers) + self.force_close() + self._compress = compress + transport = request._protocol.transport + if transport is None: + raise ConnectionResetError("Connection lost") + writer = WebSocketWriter( + request._protocol, + transport, + compress=compress, + notakeover=notakeover, + limit=self._writer_limit, + ) + + return protocol, writer + + def _post_start( + self, request: BaseRequest, protocol: str | None, writer: WebSocketWriter + ) -> None: + self._ws_protocol = protocol + self._writer = writer + + self._reset_heartbeat() + + loop = self._loop + assert loop is not None + self._reader = WebSocketDataQueue( + request._protocol, DEFAULT_CHUNK_SIZE, loop=loop + ) + parser = WebSocketReader( + self._reader, + self._max_msg_size, + compress=bool(self._compress), + decode_text=self._decode_text, + ) + cb = None if self._heartbeat is None else self._on_data_received + request.protocol.set_parser(parser, data_received_cb=cb) + # disable HTTP keepalive for WebSocket + request.protocol.keep_alive(False) + + def can_prepare(self, request: BaseRequest) -> WebSocketReady: + if self._writer is not None: + raise RuntimeError("Already started") + try: + _, protocol, _, _ = self._handshake(request) + except HTTPException: + return WebSocketReady(False, None) + else: + return WebSocketReady(True, protocol) + + @property + def prepared(self) -> bool: + return self._writer is not None + + @property + def closed(self) -> bool: + return self._closed + + @property + def close_code(self) -> int | None: + return self._close_code + + @property + def ws_protocol(self) -> str | None: + return self._ws_protocol + + @property + def compress(self) -> int | bool: + return self._compress + + def get_extra_info(self, name: str, default: Any = None) -> Any: + """Get optional transport information. + + If no value associated with ``name`` is found, ``default`` is returned. + """ + writer = self._writer + if writer is None: + return default + transport = writer.transport + if transport is None: + return default + return transport.get_extra_info(name, default) + + def exception(self) -> BaseException | None: + return self._exception + + async def ping(self, message: bytes = b"") -> None: + if self._writer is None: + raise RuntimeError("Call .prepare() first") + await self._writer.send_frame(message, WSMsgType.PING) + + async def pong(self, message: bytes = b"") -> None: + # unsolicited pong + if self._writer is None: + raise RuntimeError("Call .prepare() first") + await self._writer.send_frame(message, WSMsgType.PONG) + + async def send_frame( + self, message: bytes, opcode: WSMsgType, compress: int | None = None + ) -> None: + """Send a frame over the websocket.""" + if self._writer is None: + raise RuntimeError("Call .prepare() first") + await self._writer.send_frame(message, opcode, compress) + + async def send_str(self, data: str, compress: int | None = None) -> None: + if self._writer is None: + raise RuntimeError("Call .prepare() first") + if not isinstance(data, str): + raise TypeError("data argument must be str (%r)" % type(data)) + await self._writer.send_frame( + data.encode("utf-8"), WSMsgType.TEXT, compress=compress + ) + + async def send_bytes(self, data: bytes, compress: int | None = None) -> None: + if self._writer is None: + raise RuntimeError("Call .prepare() first") + if not isinstance(data, (bytes, bytearray, memoryview)): + raise TypeError("data argument must be byte-ish (%r)" % type(data)) + await self._writer.send_frame(data, WSMsgType.BINARY, compress=compress) + + async def send_json( + self, + data: Any, + compress: int | None = None, + *, + dumps: JSONEncoder = json.dumps, + ) -> None: + await self.send_str(dumps(data), compress=compress) + + async def send_json_bytes( + self, + data: Any, + compress: int | None = None, + *, + dumps: JSONBytesEncoder, + ) -> None: + """Send JSON data using a bytes-returning encoder as a binary frame. + + Use this when your JSON encoder (like orjson) returns bytes + instead of str, avoiding the encode/decode overhead. + """ + await self.send_bytes(dumps(data), compress=compress) + + async def write_eof(self) -> None: # type: ignore[override] + if self._eof_sent: + return + if self._payload_writer is None: + raise RuntimeError("Response has not been started") + + await self.close() + self._eof_sent = True + + async def close( + self, *, code: int = WSCloseCode.OK, message: bytes = b"", drain: bool = True + ) -> bool: + """Close websocket connection.""" + if self._writer is None: + raise RuntimeError("Call .prepare() first") + + if self._closed: + return False + self._set_closed() + + try: + await self._writer.close(code, message) + writer = self._payload_writer + assert writer is not None + if drain: + await writer.drain() + except (asyncio.CancelledError, asyncio.TimeoutError): + self._set_code_close_transport(WSCloseCode.ABNORMAL_CLOSURE) + raise + except Exception as exc: + self._exception = exc + self._set_code_close_transport(WSCloseCode.ABNORMAL_CLOSURE) + return True + + reader = self._reader + assert reader is not None + # we need to break `receive()` cycle before we can call + # `reader.read()` as `close()` may be called from different task + if self._waiting: + assert self._loop is not None + assert self._close_wait is None + self._close_wait = self._loop.create_future() + reader.feed_data(WS_CLOSING_MESSAGE, 0) + await self._close_wait + + if self._closing: + self._close_transport() + return True + + try: + async with async_timeout.timeout(self._timeout): + while True: + msg = await reader.read() + if msg.type is WSMsgType.CLOSE: + self._set_code_close_transport(msg.data) + return True + except asyncio.CancelledError: + self._set_code_close_transport(WSCloseCode.ABNORMAL_CLOSURE) + raise + except Exception as exc: + self._exception = exc + self._set_code_close_transport(WSCloseCode.ABNORMAL_CLOSURE) + return True + + def _set_closing(self, code: WSCloseCode) -> None: + """Set the close code and mark the connection as closing.""" + self._closing = True + self._close_code = code + self._cancel_heartbeat() + + def _set_code_close_transport(self, code: WSCloseCode) -> None: + """Set the close code and close the transport.""" + self._close_code = code + self._close_transport() + + def _close_transport(self) -> None: + """Close the transport.""" + if self._req is not None and self._req.transport is not None: + self._req.transport.close() + + @overload + async def receive( + self: "WebSocketResponse[Literal[True]]", timeout: float | None = None + ) -> WSMessageDecodeText: ... + + @overload + async def receive( + self: "WebSocketResponse[Literal[False]]", timeout: float | None = None + ) -> WSMessageNoDecodeText: ... + + @overload + async def receive( + self: "WebSocketResponse[_DecodeText]", timeout: float | None = None + ) -> WSMessageDecodeText | WSMessageNoDecodeText: ... + + async def receive( + self, timeout: float | None = None + ) -> WSMessageDecodeText | WSMessageNoDecodeText: + if self._reader is None: + raise RuntimeError("Call .prepare() first") + + receive_timeout = timeout or self._receive_timeout + while True: + if self._waiting: + raise RuntimeError("Concurrent call to receive() is not allowed") + + if self._closed: + self._conn_lost += 1 + if self._conn_lost >= THRESHOLD_CONNLOST_ACCESS: + raise RuntimeError("WebSocket connection is closed.") + return WS_CLOSED_MESSAGE + elif self._closing: + return WS_CLOSING_MESSAGE + + try: + self._waiting = True + try: + if receive_timeout: + # Entering the context manager and creating + # Timeout() object can take almost 50% of the + # run time in this loop so we avoid it if + # there is no read timeout. + async with async_timeout.timeout(receive_timeout): + msg = await self._reader.read() + else: + msg = await self._reader.read() + finally: + self._waiting = False + if self._close_wait: + set_result(self._close_wait, None) + except asyncio.TimeoutError: + raise + except EofStream: + self._close_code = WSCloseCode.OK + await self.close() + return WSMessage(WSMsgType.CLOSED, None, None) + except WebSocketError as exc: + self._close_code = exc.code + await self.close(code=exc.code) + return WSMessage(WSMsgType.ERROR, exc, None) + except Exception as exc: + self._exception = exc + self._set_closing(WSCloseCode.ABNORMAL_CLOSURE) + await self.close() + return WSMessage(WSMsgType.ERROR, exc, None) + + if msg.type not in _INTERNAL_RECEIVE_TYPES: + # If its not a close/closing/ping/pong message + # we can return it immediately + return msg + + if msg.type is WSMsgType.CLOSE: + self._set_closing(msg.data) + # Could be closed while awaiting reader. + if not self._closed and self._autoclose: + # The client is likely going to close the + # connection out from under us so we do not + # want to drain any pending writes as it will + # likely result writing to a broken pipe. + await self.close(drain=False) + elif msg.type is WSMsgType.CLOSING: + self._set_closing(WSCloseCode.OK) + elif msg.type is WSMsgType.PING and self._autoping: + await self.pong(msg.data) + continue + elif msg.type is WSMsgType.PONG and self._autoping: + continue + + return msg + + @overload + async def receive_str( + self: "WebSocketResponse[Literal[True]]", *, timeout: float | None = None + ) -> str: ... + + @overload + async def receive_str( + self: "WebSocketResponse[Literal[False]]", *, timeout: float | None = None + ) -> bytes: ... + + @overload + async def receive_str( + self: "WebSocketResponse[_DecodeText]", *, timeout: float | None = None + ) -> str | bytes: ... + + async def receive_str(self, *, timeout: float | None = None) -> str | bytes: + """Receive TEXT message. + + Returns str when decode_text=True (default), bytes when decode_text=False. + """ + msg = await self.receive(timeout) + if msg.type is not WSMsgType.TEXT: + raise WSMessageTypeError( + f"Received message {msg.type}:{msg.data!r} is not WSMsgType.TEXT" + ) + return cast(str, msg.data) + + async def receive_bytes(self, *, timeout: float | None = None) -> bytes: + msg = await self.receive(timeout) + if msg.type is not WSMsgType.BINARY: + raise WSMessageTypeError( + f"Received message {msg.type}:{msg.data!r} is not WSMsgType.BINARY" + ) + return cast(bytes, msg.data) + + @overload + async def receive_json( + self: "WebSocketResponse[Literal[True]]", + *, + loads: JSONDecoder = ..., + timeout: float | None = None, + ) -> Any: ... + + @overload + async def receive_json( + self: "WebSocketResponse[Literal[False]]", + *, + loads: Callable[[bytes], Any] = ..., + timeout: float | None = None, + ) -> Any: ... + + @overload + async def receive_json( + self: "WebSocketResponse[_DecodeText]", + *, + loads: JSONDecoder | Callable[[bytes], Any] = ..., + timeout: float | None = None, + ) -> Any: ... + + async def receive_json( + self, + *, + loads: JSONDecoder | Callable[[bytes], Any] = json.loads, + timeout: float | None = None, + ) -> Any: + data = await self.receive_str(timeout=timeout) + return loads(data) # type: ignore[arg-type] + + async def write(self, data: Buffer) -> None: + raise RuntimeError("Cannot call .write() for websocket") + + def __aiter__(self) -> Self: + return self + + @overload + async def __anext__( + self: "WebSocketResponse[Literal[True]]", + ) -> WSMessageDecodeText: ... + + @overload + async def __anext__( + self: "WebSocketResponse[Literal[False]]", + ) -> WSMessageNoDecodeText: ... + + @overload + async def __anext__( + self: "WebSocketResponse[_DecodeText]", + ) -> WSMessageDecodeText | WSMessageNoDecodeText: ... + + async def __anext__(self) -> WSMessageDecodeText | WSMessageNoDecodeText: + msg = await self.receive() + if msg.type in (WSMsgType.CLOSE, WSMsgType.CLOSING, WSMsgType.CLOSED): + raise StopAsyncIteration + return msg + + def _cancel(self, exc: BaseException) -> None: + # web_protocol calls this from connection_lost + # or when the server is shutting down. + self._closing = True + self._cancel_heartbeat() + if self._reader is not None: + set_exception(self._reader, exc) diff --git a/py_modules/aiohttp/worker.py b/py_modules/aiohttp/worker.py new file mode 100644 index 0000000..6bbdbdc --- /dev/null +++ b/py_modules/aiohttp/worker.py @@ -0,0 +1,278 @@ +"""Async gunicorn worker for aiohttp.web""" + +import asyncio +import inspect +import os +import re +import signal +import sys +from types import FrameType +from typing import TYPE_CHECKING, Any, Optional + +from gunicorn.config import AccessLogFormat as GunicornAccessLogFormat +from gunicorn.workers import base + +from aiohttp import web + +from .helpers import set_result +from .web_app import Application +from .web_log import AccessLogger + +if TYPE_CHECKING: + import ssl + + SSLContext = ssl.SSLContext +else: + try: + import ssl + + SSLContext = ssl.SSLContext + except ImportError: # pragma: no cover + ssl = None # type: ignore[assignment] + SSLContext = object # type: ignore[misc,assignment] + + +__all__ = ("GunicornWebWorker", "GunicornUVLoopWebWorker") + + +class GunicornWebWorker(base.Worker): # type: ignore[misc,no-any-unimported] + DEFAULT_AIOHTTP_LOG_FORMAT = AccessLogger.LOG_FORMAT + DEFAULT_GUNICORN_LOG_FORMAT = GunicornAccessLogFormat.default + + def __init__(self, *args: Any, **kw: Any) -> None: # pragma: no cover + super().__init__(*args, **kw) + + self._task: asyncio.Task[None] | None = None + self.exit_code = 0 + self._notify_waiter: asyncio.Future[bool] | None = None + + def init_process(self) -> None: + # create new event_loop after fork + try: + asyncio.get_event_loop().close() + except RuntimeError: + # No loop was running + pass + + self.loop = asyncio.new_event_loop() + asyncio.set_event_loop(self.loop) + + super().init_process() + + def run(self) -> None: + # base.Worker.init_process() sets self.booted = True before + # invoking run(), but for the aiohttp worker the real boot work + # (factory call, runner setup, binding sockets) happens here. + # Reset until _run() reaches the serve loop so that the arbiter + # can tell a startup failure from a normal worker exit and + # halt instead of endlessly respawning workers. + self.booted = False + + self._task = self.loop.create_task(self._run()) + try: + self.loop.run_until_complete(self._task) + finally: + self.loop.run_until_complete(self.loop.shutdown_asyncgens()) + self.loop.close() + + sys.exit(self.exit_code) + + async def _run(self) -> None: + runner = None + if isinstance(self.wsgi, Application): + app = self.wsgi + elif inspect.iscoroutinefunction(self.wsgi) or ( + sys.version_info < (3, 14) and asyncio.iscoroutinefunction(self.wsgi) + ): + wsgi = await self.wsgi() + if isinstance(wsgi, web.AppRunner): + runner = wsgi + app = runner.app + else: + app = wsgi + else: + raise RuntimeError( + "wsgi app should be either Application or " + f"async function returning Application, got {self.wsgi}" + ) + + if runner is None: + access_log = self.log.access_log if self.cfg.accesslog else None + runner = web.AppRunner( + app, + logger=self.log, + keepalive_timeout=self.cfg.keepalive, + access_log=access_log, + access_log_format=self._get_valid_log_format( + self.cfg.access_log_format + ), + shutdown_timeout=self.cfg.graceful_timeout / 100 * 95, + ) + await runner.setup() + + ctx = self._create_ssl_context(self.cfg) if self.cfg.is_ssl else None + + runner = runner + assert runner is not None + server = runner.server + assert server is not None + for sock in self.sockets: + site = web.SockSite( + runner, + sock, + ssl_context=ctx, + ) + await site.start() + + # Sockets are bound; tell the arbiter the worker is ready to + # accept requests. Any failure before this point propagates out + # of run() with self.booted=False so the arbiter exits with + # WORKER_BOOT_ERROR instead of treating this as a clean exit. + self.booted = True + + # If our parent changed then we shut down. + pid = os.getpid() + try: + while self.alive: # type: ignore[has-type] + self.notify() + + cnt = server.requests_count + if self.max_requests and cnt > self.max_requests: + self.alive = False + self.log.info("Max requests, shutting down: %s", self) + + elif pid == os.getpid() and self.ppid != os.getppid(): + self.alive = False + self.log.info("Parent changed, shutting down: %s", self) + else: + await self._wait_next_notify() + except Exception: + pass + + await runner.cleanup() + + def _wait_next_notify(self) -> "asyncio.Future[bool]": + self._notify_waiter_done() + + loop = self.loop + assert loop is not None + self._notify_waiter = waiter = loop.create_future() + self.loop.call_later(1.0, self._notify_waiter_done, waiter) + + return waiter + + def _notify_waiter_done( + self, waiter: Optional["asyncio.Future[bool]"] = None + ) -> None: + if waiter is None: + waiter = self._notify_waiter + if waiter is not None: + set_result(waiter, True) + + if waiter is self._notify_waiter: + self._notify_waiter = None + + def init_signals(self) -> None: + # Set up signals through the event loop API. + + self.loop.add_signal_handler( + signal.SIGQUIT, self.handle_quit, signal.SIGQUIT, None + ) + + self.loop.add_signal_handler( + signal.SIGTERM, self.handle_exit, signal.SIGTERM, None + ) + + self.loop.add_signal_handler( + signal.SIGINT, self.handle_quit, signal.SIGINT, None + ) + + self.loop.add_signal_handler( + signal.SIGWINCH, self.handle_winch, signal.SIGWINCH, None + ) + + self.loop.add_signal_handler( + signal.SIGUSR1, self.handle_usr1, signal.SIGUSR1, None + ) + + self.loop.add_signal_handler( + signal.SIGABRT, self.handle_abort, signal.SIGABRT, None + ) + + # Don't let SIGTERM and SIGUSR1 disturb active requests + # by interrupting system calls + signal.siginterrupt(signal.SIGTERM, False) + signal.siginterrupt(signal.SIGUSR1, False) + + # Reset SIGCHLD to default so Gunicorn doesn't swallow subprocess + # return codes. Without this, workers inherit the master arbiter's + # SIGCHLD handler, causing spurious "Worker exited" errors when + # application code spawns subprocesses. + signal.signal(signal.SIGCHLD, signal.SIG_DFL) + + def handle_quit(self, sig: int, frame: FrameType | None) -> None: + self.alive = False + + # worker_int callback + self.cfg.worker_int(self) + + # wakeup closing process + self._notify_waiter_done() + + def handle_abort(self, sig: int, frame: FrameType | None) -> None: + self.alive = False + self.exit_code = 1 + self.cfg.worker_abort(self) + sys.exit(1) + + @staticmethod + def _create_ssl_context(cfg: Any) -> "SSLContext": + """Creates SSLContext instance for usage in asyncio.create_server. + + See ssl.SSLSocket.__init__ for more details. + """ + if ssl is None: # pragma: no cover + raise RuntimeError("SSL is not supported.") + + ctx = ssl.SSLContext(cfg.ssl_version) + ctx.load_cert_chain(cfg.certfile, cfg.keyfile) + ctx.verify_mode = cfg.cert_reqs + if cfg.ca_certs: + ctx.load_verify_locations(cfg.ca_certs) + if cfg.ciphers: + ctx.set_ciphers(cfg.ciphers) + return ctx + + def _get_valid_log_format(self, source_format: str) -> str: + if source_format == self.DEFAULT_GUNICORN_LOG_FORMAT: + return self.DEFAULT_AIOHTTP_LOG_FORMAT + elif re.search(r"%\([^\)]+\)", source_format): + raise ValueError( + "Gunicorn's style options in form of `%(name)s` are not " + "supported for the log formatting. Please use aiohttp's " + "format specification to configure access log formatting: " + "http://docs.aiohttp.org/en/stable/logging.html" + "#format-specification" + ) + else: + return source_format + + +class GunicornUVLoopWebWorker(GunicornWebWorker): + def init_process(self) -> None: + import uvloop + + # Close any existing event loop before setting a + # new policy. + try: + asyncio.get_event_loop().close() + except RuntimeError: + # No loop was running + pass + + # Setup uvloop policy, so that every + # asyncio.get_event_loop() will create an instance + # of uvloop event loop. + asyncio.set_event_loop_policy(uvloop.EventLoopPolicy()) + + super().init_process() diff --git a/py_modules/aiosignal/__init__.py b/py_modules/aiosignal/__init__.py new file mode 100644 index 0000000..5ede009 --- /dev/null +++ b/py_modules/aiosignal/__init__.py @@ -0,0 +1,59 @@ +import sys +from typing import Any, Awaitable, Callable, TypeVar + +from frozenlist import FrozenList + +if sys.version_info >= (3, 11): + from typing import Unpack +else: + from typing_extensions import Unpack + +if sys.version_info >= (3, 13): + from typing import TypeVarTuple +else: + from typing_extensions import TypeVarTuple + +_T = TypeVar("_T") +_Ts = TypeVarTuple("_Ts", default=Unpack[tuple[()]]) + +__version__ = "1.4.0" + +__all__ = ("Signal",) + + +class Signal(FrozenList[Callable[[Unpack[_Ts]], Awaitable[object]]]): + """Coroutine-based signal implementation. + + To connect a callback to a signal, use any list method. + + Signals are fired using the send() coroutine, which takes named + arguments. + """ + + __slots__ = ("_owner",) + + def __init__(self, owner: object): + super().__init__() + self._owner = owner + + def __repr__(self) -> str: + return "".format( + self._owner, self.frozen, list(self) + ) + + async def send(self, *args: Unpack[_Ts], **kwargs: Any) -> None: + """ + Sends data to all registered receivers. + """ + if not self.frozen: + raise RuntimeError("Cannot send non-frozen signal.") + + for receiver in self: + await receiver(*args, **kwargs) + + def __call__( + self, func: Callable[[Unpack[_Ts]], Awaitable[_T]] + ) -> Callable[[Unpack[_Ts]], Awaitable[_T]]: + """Decorator to add a function to this Signal.""" + self.append(func) + return func diff --git a/py_modules/aiosignal/py.typed b/py_modules/aiosignal/py.typed new file mode 100644 index 0000000..e69de29 diff --git a/py_modules/argon2/__init__.py b/py_modules/argon2/__init__.py new file mode 100644 index 0000000..bea9839 --- /dev/null +++ b/py_modules/argon2/__init__.py @@ -0,0 +1,79 @@ +# SPDX-License-Identifier: MIT + +""" +Argon2 for Python +""" + +from . import exceptions, low_level, profiles +from ._legacy import hash_password, hash_password_raw, verify_password +from ._password_hasher import ( + DEFAULT_HASH_LENGTH, + DEFAULT_MEMORY_COST, + DEFAULT_PARALLELISM, + DEFAULT_RANDOM_SALT_LENGTH, + DEFAULT_TIME_COST, + PasswordHasher, +) +from ._utils import Parameters, extract_parameters +from .low_level import Type + + +__title__ = "argon2-cffi" + +__author__ = "Hynek Schlawack" +__copyright__ = "Copyright (c) 2015 " + __author__ +__license__ = "MIT" + + +__all__ = [ + "DEFAULT_HASH_LENGTH", + "DEFAULT_MEMORY_COST", + "DEFAULT_PARALLELISM", + "DEFAULT_RANDOM_SALT_LENGTH", + "DEFAULT_TIME_COST", + "Parameters", + "PasswordHasher", + "Type", + "exceptions", + "extract_parameters", + "hash_password", + "hash_password_raw", + "low_level", + "profiles", + "verify_password", +] + + +def __getattr__(name: str) -> str: + dunder_to_metadata = { + "__version__": "version", + "__description__": "summary", + "__uri__": "", + "__url__": "", + "__email__": "", + } + if name not in dunder_to_metadata: + msg = f"module {__name__} has no attribute {name}" + raise AttributeError(msg) + + import warnings + + from importlib.metadata import metadata + + warnings.warn( + f"Accessing argon2.{name} is deprecated and will be " + "removed in a future release. Use importlib.metadata directly " + "to query for argon2-cffi's packaging metadata.", + DeprecationWarning, + stacklevel=2, + ) + + meta = metadata("argon2-cffi") + + if name in ("__uri__", "__url__"): + return meta["Project-URL"].split(" ", 1)[-1] + + if name == "__email__": + return meta["Author-email"].split("<", 1)[1].rstrip(">") + + return meta[dunder_to_metadata[name]] diff --git a/py_modules/argon2/__main__.py b/py_modules/argon2/__main__.py new file mode 100644 index 0000000..80ba933 --- /dev/null +++ b/py_modules/argon2/__main__.py @@ -0,0 +1,91 @@ +# SPDX-License-Identifier: MIT + +from __future__ import annotations + +import argparse +import sys +import timeit + +from . import ( + DEFAULT_HASH_LENGTH, + DEFAULT_MEMORY_COST, + DEFAULT_PARALLELISM, + DEFAULT_TIME_COST, + PasswordHasher, + profiles, +) + + +def main(argv: list[str]) -> None: + parser = argparse.ArgumentParser( + description="Benchmark Argon2.", + formatter_class=argparse.ArgumentDefaultsHelpFormatter, + ) + parser.add_argument( + "-n", type=int, default=100, help="Number of iterations to measure." + ) + parser.add_argument( + "-t", type=int, help="`time_cost`", default=DEFAULT_TIME_COST + ) + parser.add_argument( + "-m", type=int, help="`memory_cost`", default=DEFAULT_MEMORY_COST + ) + parser.add_argument( + "-p", type=int, help="`parallelism`", default=DEFAULT_PARALLELISM + ) + parser.add_argument( + "-l", type=int, help="`hash_length`", default=DEFAULT_HASH_LENGTH + ) + parser.add_argument( + "--profile", + type=str, + help="A profile from `argon2.profiles. Takes precedence.", + default=None, + ) + + args = parser.parse_args(argv[1:]) + + password = b"secret" + if args.profile: + ph = PasswordHasher.from_parameters( + getattr(profiles, args.profile.upper()) + ) + else: + ph = PasswordHasher( + time_cost=args.t, + memory_cost=args.m, + parallelism=args.p, + hash_len=args.l, + ) + hash = ph.hash(password) + + print(f"Running Argon2id {args.n} times with:") + + for name, value, units in [ + ("hash_len", ph.hash_len, "bytes"), + ("memory_cost", ph.memory_cost, "KiB"), + ("parallelism", ph.parallelism, "threads"), + ("time_cost", ph.time_cost, "iterations"), + ]: + print(f"{name}: {value} {units}") + + print("\nMeasuring...") + duration = timeit.timeit( + f"ph.verify({hash!r}, {password!r})", + setup=f"""\ +from argon2 import PasswordHasher + +ph = PasswordHasher( + time_cost={args.t!r}, + memory_cost={args.m!r}, + parallelism={args.p!r}, + hash_len={args.l!r}, +) +gc.enable()""", + number=args.n, + ) + print(f"\n{duration / args.n * 1000:.1f}ms per password verification") + + +if __name__ == "__main__": # pragma: no cover + main(sys.argv) diff --git a/py_modules/argon2/_legacy.py b/py_modules/argon2/_legacy.py new file mode 100644 index 0000000..05a1e81 --- /dev/null +++ b/py_modules/argon2/_legacy.py @@ -0,0 +1,92 @@ +# SPDX-License-Identifier: MIT + +""" +Legacy mid-level functions. +""" + +from __future__ import annotations + +import os +import warnings + +from typing import Literal + +from ._password_hasher import ( + DEFAULT_HASH_LENGTH, + DEFAULT_MEMORY_COST, + DEFAULT_PARALLELISM, + DEFAULT_RANDOM_SALT_LENGTH, + DEFAULT_TIME_COST, +) +from .low_level import Type, hash_secret, hash_secret_raw, verify_secret + + +_INSTEAD = " is deprecated, use argon2.PasswordHasher instead" + + +def hash_password( + password: bytes, + salt: bytes | None = None, + time_cost: int = DEFAULT_TIME_COST, + memory_cost: int = DEFAULT_MEMORY_COST, + parallelism: int = DEFAULT_PARALLELISM, + hash_len: int = DEFAULT_HASH_LENGTH, + type: Type = Type.I, +) -> bytes: + """ + Legacy alias for :func:`argon2.low_level.hash_secret` with default + parameters. + + .. deprecated:: 16.0.0 + Use :class:`argon2.PasswordHasher` for passwords. + """ + warnings.warn( + "argon2.hash_password" + _INSTEAD, DeprecationWarning, stacklevel=2 + ) + if salt is None: + salt = os.urandom(DEFAULT_RANDOM_SALT_LENGTH) + return hash_secret( + password, salt, time_cost, memory_cost, parallelism, hash_len, type + ) + + +def hash_password_raw( + password: bytes, + salt: bytes | None = None, + time_cost: int = DEFAULT_TIME_COST, + memory_cost: int = DEFAULT_MEMORY_COST, + parallelism: int = DEFAULT_PARALLELISM, + hash_len: int = DEFAULT_HASH_LENGTH, + type: Type = Type.I, +) -> bytes: + """ + Legacy alias for :func:`argon2.low_level.hash_secret_raw` with default + parameters. + + .. deprecated:: 16.0.0 + Use :class:`argon2.PasswordHasher` for passwords. + """ + warnings.warn( + "argon2.hash_password_raw" + _INSTEAD, DeprecationWarning, stacklevel=2 + ) + if salt is None: + salt = os.urandom(DEFAULT_RANDOM_SALT_LENGTH) + return hash_secret_raw( + password, salt, time_cost, memory_cost, parallelism, hash_len, type + ) + + +def verify_password( + hash: bytes, password: bytes, type: Type = Type.I +) -> Literal[True]: + """ + Legacy alias for :func:`argon2.low_level.verify_secret` with default + parameters. + + .. deprecated:: 16.0.0 + Use :class:`argon2.PasswordHasher` for passwords. + """ + warnings.warn( + "argon2.verify_password" + _INSTEAD, DeprecationWarning, stacklevel=2 + ) + return verify_secret(hash, password, type) diff --git a/py_modules/argon2/_password_hasher.py b/py_modules/argon2/_password_hasher.py new file mode 100644 index 0000000..ddf1ce5 --- /dev/null +++ b/py_modules/argon2/_password_hasher.py @@ -0,0 +1,287 @@ +# SPDX-License-Identifier: MIT + +from __future__ import annotations + +import os + +from typing import ClassVar, Literal + +from ._utils import ( + Parameters, + _check_types, + extract_parameters, + validate_params_for_platform, +) +from .exceptions import InvalidHashError +from .low_level import Type, hash_secret, verify_secret +from .profiles import get_default_parameters + + +default_params = get_default_parameters() + +DEFAULT_RANDOM_SALT_LENGTH = default_params.salt_len +DEFAULT_HASH_LENGTH = default_params.hash_len +DEFAULT_TIME_COST = default_params.time_cost +DEFAULT_MEMORY_COST = default_params.memory_cost +DEFAULT_PARALLELISM = default_params.parallelism + + +def _ensure_bytes(s: bytes | str, encoding: str) -> bytes: + """ + Ensure *s* is a bytes string. Encode using *encoding* if it isn't. + """ + if isinstance(s, bytes): + return s + return s.encode(encoding) + + +class PasswordHasher: + r""" + High level class to hash passwords with sensible defaults. + + Uses Argon2\ **id** by default and uses a random salt_ for hashing. But it + can verify any type of Argon2 as long as the hash is correctly encoded. + + The reason for this being a class is both for convenience to carry + parameters and to verify the parameters only *once*. Any unnecessary + slowdown when hashing is a tangible advantage for a brute-force attacker. + + Args: + time_cost: + Defines the amount of computation realized and therefore the + execution time, given in number of iterations. + + memory_cost: Defines the memory usage, given in kibibytes_. + + parallelism: + Defines the number of parallel threads (*changes* the resulting + hash value). + + hash_len: Length of the hash in bytes. + + salt_len: Length of random salt to be generated for each password. + + encoding: + The Argon2 C library expects bytes. So if :meth:`hash` or + :meth:`verify` are passed a ``str``, it will be encoded using this + encoding. + + type: + Argon2 type to use. Only change for interoperability with legacy + systems. + + .. versionadded:: 16.0.0 + .. versionchanged:: 18.2.0 + Switch from Argon2i to Argon2id based on the recommendation by the + current RFC draft. See also :doc:`parameters`. + .. versionchanged:: 18.2.0 + Changed default *memory_cost* to 100 MiB and default *parallelism* to 8. + .. versionchanged:: 18.2.0 ``verify`` now will determine the type of hash. + .. versionchanged:: 18.3.0 The Argon2 type is configurable now. + .. versionadded:: 21.2.0 :meth:`from_parameters` + .. versionchanged:: 21.2.0 + Changed defaults to :data:`argon2.profiles.RFC_9106_LOW_MEMORY`. + + .. _salt: https://en.wikipedia.org/wiki/Salt_(cryptography) + .. _kibibytes: https://en.wikipedia.org/wiki/Binary_prefix#kibi + """ + + __slots__ = ["_parameters", "encoding"] + + _parameters: Parameters + encoding: str + + def __init__( + self, + time_cost: int = DEFAULT_TIME_COST, + memory_cost: int = DEFAULT_MEMORY_COST, + parallelism: int = DEFAULT_PARALLELISM, + hash_len: int = DEFAULT_HASH_LENGTH, + salt_len: int = DEFAULT_RANDOM_SALT_LENGTH, + encoding: str = "utf-8", + type: Type = Type.ID, + ): + e = _check_types( + time_cost=(time_cost, int), + memory_cost=(memory_cost, int), + parallelism=(parallelism, int), + hash_len=(hash_len, int), + salt_len=(salt_len, int), + encoding=(encoding, str), + type=(type, Type), + ) + if e: + raise TypeError(e) + + params = Parameters( + type=type, + version=19, + salt_len=salt_len, + hash_len=hash_len, + time_cost=time_cost, + memory_cost=memory_cost, + parallelism=parallelism, + ) + + validate_params_for_platform(params) + + # Cache a Parameters object for check_needs_rehash. + self._parameters = params + self.encoding = encoding + + @classmethod + def from_parameters(cls, params: Parameters) -> PasswordHasher: + """ + Construct a `PasswordHasher` from *params*. + + Returns: + A `PasswordHasher` instance with the parameters from *params*. + + .. versionadded:: 21.2.0 + """ + + return cls( + time_cost=params.time_cost, + memory_cost=params.memory_cost, + parallelism=params.parallelism, + hash_len=params.hash_len, + salt_len=params.salt_len, + type=params.type, + ) + + @property + def time_cost(self) -> int: + return self._parameters.time_cost + + @property + def memory_cost(self) -> int: + return self._parameters.memory_cost + + @property + def parallelism(self) -> int: + return self._parameters.parallelism + + @property + def hash_len(self) -> int: + return self._parameters.hash_len + + @property + def salt_len(self) -> int: + return self._parameters.salt_len + + @property + def type(self) -> Type: + return self._parameters.type + + def hash(self, password: str | bytes, *, salt: bytes | None = None) -> str: + """ + Hash *password* and return an encoded hash. + + Args: + password: Password to hash. + + salt: + If None, a random salt is securely created. + + .. danger:: + + You should **not** pass a salt unless you really know what + you are doing. + + Raises: + argon2.exceptions.HashingError: If hashing fails. + + Returns: + Hashed *password*. + + .. versionadded:: 23.1.0 *salt* parameter + """ + return hash_secret( + secret=_ensure_bytes(password, self.encoding), + salt=salt or os.urandom(self.salt_len), + time_cost=self.time_cost, + memory_cost=self.memory_cost, + parallelism=self.parallelism, + hash_len=self.hash_len, + type=self.type, + ).decode("ascii") + + _header_to_type: ClassVar[dict[bytes, Type]] = { + b"$argon2i$": Type.I, + b"$argon2d$": Type.D, + b"$argon2id": Type.ID, + } + + def verify( + self, hash: str | bytes, password: str | bytes + ) -> Literal[True]: + """ + Verify that *password* matches *hash*. + + .. warning:: + + It is assumed that the caller is in full control of the hash. No + other parsing than the determination of the hash type is done by + *argon2-cffi*. + + Args: + hash: An encoded hash as returned from :meth:`PasswordHasher.hash`. + + password: The password to verify. + + Raises: + argon2.exceptions.VerifyMismatchError: + If verification fails because *hash* is not valid for + *password*. + + argon2.exceptions.VerificationError: + If verification fails for other reasons. + + argon2.exceptions.InvalidHashError: + If *hash* is so clearly invalid, that it couldn't be passed to + Argon2. + + Returns: + ``True`` on success, otherwise an exception is raised. + + .. versionchanged:: 16.1.0 + Raise :exc:`~argon2.exceptions.VerifyMismatchError` on mismatches + instead of its more generic superclass. + .. versionadded:: 18.2.0 Hash type agility. + """ + hash = _ensure_bytes(hash, "ascii") + try: + hash_type = self._header_to_type[hash[:9]] + except LookupError: + raise InvalidHashError from None + + return verify_secret( + hash, _ensure_bytes(password, self.encoding), hash_type + ) + + def check_needs_rehash(self, hash: str | bytes) -> bool: + """ + Check whether *hash* was created using the instance's parameters. + + Whenever your Argon2 parameters -- or *argon2-cffi*'s defaults! -- + change, you should rehash your passwords at the next opportunity. The + common approach is to do that whenever a user logs in, since that + should be the only time when you have access to the cleartext + password. + + Therefore it's best practice to check -- and if necessary rehash -- + passwords after each successful authentication. + + Args: + hash: An encoded Argon2 password hash. + + Returns: + Whether *hash* was created using the instance's parameters. + + .. versionadded:: 18.2.0 + .. versionchanged:: 24.1.0 Accepts bytes for *hash*. + """ + if isinstance(hash, bytes): + hash = hash.decode("ascii") + + return self._parameters != extract_parameters(hash) diff --git a/py_modules/argon2/_utils.py b/py_modules/argon2/_utils.py new file mode 100644 index 0000000..21a6362 --- /dev/null +++ b/py_modules/argon2/_utils.py @@ -0,0 +1,174 @@ +# SPDX-License-Identifier: MIT + +from __future__ import annotations + +import platform +import sys + +from dataclasses import dataclass +from typing import Any + +from .exceptions import InvalidHashError, UnsupportedParametersError +from .low_level import Type + + +NoneType = type(None) + + +def _check_types(**kw: Any) -> str | None: + """ + Check each ``name: (value, types)`` in *kw*. + + Returns a human-readable string of all violations or `None``. + """ + errors = [] + for name, (value, types) in kw.items(): + if not isinstance(value, types): + if isinstance(types, tuple): + types = ", or ".join(t.__name__ for t in types) + else: + types = types.__name__ + errors.append( + f"'{name}' must be a {types} (got {type(value).__name__})" + ) + + if errors != []: + return ", ".join(errors) + "." + + return None + + +def _is_wasm() -> bool: + return sys.platform == "emscripten" or platform.machine() in [ + "wasm32", + "wasm64", + ] + + +def _decoded_str_len(length: int) -> int: + """ + Compute how long an encoded string of length *l* becomes. + """ + rem = length % 4 + + if rem == 3: + last_group_len = 2 + elif rem == 2: + last_group_len = 1 + else: + last_group_len = 0 + + return length // 4 * 3 + last_group_len + + +@dataclass +class Parameters: + """ + Argon2 hash parameters. + + See :doc:`parameters` on how to pick them. + + Attributes: + type: Hash type. + + version: Argon2 version. + + salt_len: Length of the salt in bytes. + + hash_len: Length of the hash in bytes. + + time_cost: Time cost in iterations. + + memory_cost: Memory cost in kibibytes. + + parallelism: Number of parallel threads. + + .. versionadded:: 18.2.0 + """ + + type: Type + version: int + salt_len: int + hash_len: int + time_cost: int + memory_cost: int + parallelism: int + + __slots__ = ( + "hash_len", + "memory_cost", + "parallelism", + "salt_len", + "time_cost", + "type", + "version", + ) + + +_NAME_TO_TYPE = {"argon2id": Type.ID, "argon2i": Type.I, "argon2d": Type.D} +_REQUIRED_KEYS = sorted(("v", "m", "t", "p")) + + +def extract_parameters(hash: str) -> Parameters: + """ + Extract parameters from an encoded *hash*. + + Args: + hash: An encoded Argon2 hash string. + + Returns: + The parameters used to create the hash. + + .. versionadded:: 18.2.0 + """ + parts = hash.split("$") + + # Backwards compatibility for Argon v1.2 hashes + if len(parts) == 5: + parts.insert(2, "v=18") + + if len(parts) != 6: + raise InvalidHashError + + if parts[0]: + raise InvalidHashError + + try: + type = _NAME_TO_TYPE[parts[1]] + + kvs = { + k: int(v) + for k, v in ( + s.split("=") for s in [parts[2], *parts[3].split(",")] + ) + } + except Exception: # noqa: BLE001 + raise InvalidHashError from None + + if sorted(kvs.keys()) != _REQUIRED_KEYS: + raise InvalidHashError + + return Parameters( + type=type, + salt_len=_decoded_str_len(len(parts[4])), + hash_len=_decoded_str_len(len(parts[5])), + version=kvs["v"], + time_cost=kvs["t"], + memory_cost=kvs["m"], + parallelism=kvs["p"], + ) + + +def validate_params_for_platform(params: Parameters) -> None: + """ + Validate *params* against current platform. + + Args: + params: Parameters to be validated + + Returns: + None + """ + if _is_wasm() and params.parallelism != 1: + msg = "In WebAssembly environments `parallelism` must be 1." + raise UnsupportedParametersError(msg) diff --git a/py_modules/argon2/exceptions.py b/py_modules/argon2/exceptions.py new file mode 100644 index 0000000..a0dade5 --- /dev/null +++ b/py_modules/argon2/exceptions.py @@ -0,0 +1,66 @@ +# SPDX-License-Identifier: MIT + +from __future__ import annotations + + +class Argon2Error(Exception): + """ + Superclass of all argon2 exceptions. + + Never thrown directly. + """ + + +class VerificationError(Argon2Error): + """ + Verification failed. + + You can find the original error message from Argon2 in ``args[0]``. + """ + + +class VerifyMismatchError(VerificationError): + """ + The secret does not match the hash. + + Subclass of :exc:`argon2.exceptions.VerificationError`. + + .. versionadded:: 16.1.0 + """ + + +class HashingError(Argon2Error): + """ + Raised if hashing failed. + + You can find the original error message from Argon2 in ``args[0]``. + """ + + +class InvalidHashError(ValueError): + """ + Raised if the hash is invalid before passing it to Argon2. + + .. versionadded:: 23.1.0 + As a replacement for :exc:`argon2.exceptions.InvalidHash`. + """ + + +class UnsupportedParametersError(ValueError): + """ + Raised if the current platform does not support the parameters. + + For example, in WebAssembly parallelism must be set to 1. + + .. versionadded:: 25.1.0 + """ + + +InvalidHash = InvalidHashError +""" +Deprecated alias for :class:`InvalidHashError`. + +.. versionadded:: 18.2.0 +.. deprecated:: 23.1.0 + Use :exc:`argon2.exceptions.InvalidHashError` instead. +""" diff --git a/py_modules/argon2/low_level.py b/py_modules/argon2/low_level.py new file mode 100644 index 0000000..0598466 --- /dev/null +++ b/py_modules/argon2/low_level.py @@ -0,0 +1,253 @@ +# SPDX-License-Identifier: MIT + +""" +Low-level functions if you want to build your own higher level abstractions. + +.. warning:: + This is a "Hazardous Materials" module. You should **ONLY** use it if + you're 100% absolutely sure that you know what you're doing because this + module is full of land mines, dragons, and dinosaurs with laser guns. +""" + +from __future__ import annotations + +from enum import Enum +from typing import Any, Literal + +from _argon2_cffi_bindings import ffi, lib + +from .exceptions import HashingError, VerificationError, VerifyMismatchError + + +__all__ = [ + "ARGON2_VERSION", + "Type", + "ffi", + "hash_secret", + "hash_secret_raw", + "verify_secret", +] + +ARGON2_VERSION = lib.ARGON2_VERSION_NUMBER +""" +The latest version of the Argon2 algorithm that is supported (and used by +default). + +.. versionadded:: 16.1.0 +""" + + +class Type(Enum): + """ + Enum of Argon2 variants. + + Please see :doc:`parameters` on how to pick one. + """ + + D = lib.Argon2_d + I = lib.Argon2_i # noqa: E741 + ID = lib.Argon2_id + + +def hash_secret( + secret: bytes, + salt: bytes, + time_cost: int, + memory_cost: int, + parallelism: int, + hash_len: int, + type: Type, + version: int = ARGON2_VERSION, +) -> bytes: + """ + Hash *secret* and return an **encoded** hash. + + An encoded hash can be directly passed into :func:`verify_secret` as it + contains all parameters and the salt. + + Args: + secret: Secret to hash. + + salt: A salt_. Should be random and different for each secret. + + type: Which Argon2 variant to use. + + version: Which Argon2 version to use. + + For an explanation of the Argon2 parameters see + :class:`argon2.PasswordHasher`. + + Returns: + An encoded Argon2 hash. + + Raises: + argon2.exceptions.HashingError: If hashing fails. + + .. versionadded:: 16.0.0 + + .. _salt: https://en.wikipedia.org/wiki/Salt_(cryptography) + """ + size = ( + lib.argon2_encodedlen( + time_cost, + memory_cost, + parallelism, + len(salt), + hash_len, + type.value, + ) + + 1 + ) + buf = ffi.new("char[]", size) + rv = lib.argon2_hash( + time_cost, + memory_cost, + parallelism, + ffi.new("uint8_t[]", secret), + len(secret), + ffi.new("uint8_t[]", salt), + len(salt), + ffi.NULL, + hash_len, + buf, + size, + type.value, + version, + ) + if rv != lib.ARGON2_OK: + raise HashingError(error_to_str(rv)) + + return ffi.string(buf) # type: ignore[no-any-return] + + +def hash_secret_raw( + secret: bytes, + salt: bytes, + time_cost: int, + memory_cost: int, + parallelism: int, + hash_len: int, + type: Type, + version: int = ARGON2_VERSION, +) -> bytes: + """ + Hash *password* and return a **raw** hash. + + This function takes the same parameters as :func:`hash_secret`. + + .. versionadded:: 16.0.0 + """ + buf = ffi.new("uint8_t[]", hash_len) + + rv = lib.argon2_hash( + time_cost, + memory_cost, + parallelism, + ffi.new("uint8_t[]", secret), + len(secret), + ffi.new("uint8_t[]", salt), + len(salt), + buf, + hash_len, + ffi.NULL, + 0, + type.value, + version, + ) + if rv != lib.ARGON2_OK: + raise HashingError(error_to_str(rv)) + + return bytes(ffi.buffer(buf, hash_len)) + + +def verify_secret(hash: bytes, secret: bytes, type: Type) -> Literal[True]: + """ + Verify whether *secret* is correct for *hash* of *type*. + + Args: + hash: + An encoded Argon2 hash as returned by :func:`hash_secret`. + + secret: + The secret to verify whether it matches the one in *hash*. + + type: Type for *hash*. + + Raises: + argon2.exceptions.VerifyMismatchError: + If verification fails because *hash* is not valid for *secret* of + *type*. + + argon2.exceptions.VerificationError: + If verification fails for other reasons. + + Returns: + ``True`` on success, raise :exc:`~argon2.exceptions.VerificationError` + otherwise. + + .. versionadded:: 16.0.0 + .. versionchanged:: 16.1.0 + Raise :exc:`~argon2.exceptions.VerifyMismatchError` on mismatches + instead of its more generic superclass. + """ + rv = lib.argon2_verify( + ffi.new("char[]", hash), + ffi.new("uint8_t[]", secret), + len(secret), + type.value, + ) + + if rv == lib.ARGON2_OK: + return True + + if rv == lib.ARGON2_VERIFY_MISMATCH: + raise VerifyMismatchError(error_to_str(rv)) + + raise VerificationError(error_to_str(rv)) + + +def core(context: Any, type: int) -> int: + """ + Direct binding to the ``argon2_ctx`` function. + + .. warning:: + This is a strictly advanced function working on raw C data structures. + Both Argon2's and *argon2-cffi*'s higher-level bindings do a lot of + sanity checks and housekeeping work that *you* are now responsible for + (e.g. clearing buffers). The structure of the *context* object can, + has, and will change with *any* release! + + Use at your own peril; *argon2-cffi* does *not* use this binding + itself. + + Args: + context: + A CFFI Argon2 context object (i.e. an ``struct Argon2_Context`` / + ``argon2_context``). + + type: + Which Argon2 variant to use. You can use the ``value`` field of + :class:`Type`'s fields. + + Returns: + An Argon2 error code. Can be transformed into a string using + :func:`error_to_str`. + + .. versionadded:: 16.0.0 + """ + return lib.argon2_ctx(context, type) # type: ignore[no-any-return] + + +def error_to_str(error: int) -> str: + """ + Convert an Argon2 error code into a native string. + + Args: + error: An Argon2 error code as returned by :func:`core`. + + Returns: + A human-readable string describing the error. + + .. versionadded:: 16.0.0 + """ + return ffi.string(lib.argon2_error_message(error)).decode("ascii") # type: ignore[no-any-return] diff --git a/py_modules/argon2/profiles.py b/py_modules/argon2/profiles.py new file mode 100644 index 0000000..e6fd69b --- /dev/null +++ b/py_modules/argon2/profiles.py @@ -0,0 +1,79 @@ +# SPDX-License-Identifier: MIT + +""" +This module offers access to standardized parameters that you can load using +:meth:`argon2.PasswordHasher.from_parameters()`. See the `source code +`_ for +concrete values and :doc:`parameters` for more information. + +.. versionadded:: 21.2.0 +""" + +from __future__ import annotations + +import dataclasses + +from ._utils import Parameters, _is_wasm +from .low_level import Type + + +def get_default_parameters() -> Parameters: + """ + Create default parameters for current platform. + + Returns: + Default, compatible, parameters for current platform. + + .. versionadded:: 25.1.0 + """ + params = RFC_9106_LOW_MEMORY + + if _is_wasm(): + params = dataclasses.replace(params, parallelism=1) + + return params + + +# FIRST RECOMMENDED option per RFC 9106. +RFC_9106_HIGH_MEMORY = Parameters( + type=Type.ID, + version=19, + salt_len=16, + hash_len=32, + time_cost=1, + memory_cost=2097152, # 2 GiB + parallelism=4, +) + +# SECOND RECOMMENDED option per RFC 9106. +RFC_9106_LOW_MEMORY = Parameters( + type=Type.ID, + version=19, + salt_len=16, + hash_len=32, + time_cost=3, + memory_cost=65536, # 64 MiB + parallelism=4, +) + +# The pre-RFC defaults in argon2-cffi 18.2.0 - 21.1.0. +PRE_21_2 = Parameters( + type=Type.ID, + version=19, + salt_len=16, + hash_len=16, + time_cost=2, + memory_cost=102400, # 100 MiB + parallelism=8, +) + +# Only for testing! +CHEAPEST = Parameters( + type=Type.ID, + version=19, + salt_len=8, + hash_len=4, + time_cost=1, + memory_cost=8, + parallelism=1, +) diff --git a/py_modules/argon2/py.typed b/py_modules/argon2/py.typed new file mode 100644 index 0000000..e69de29 diff --git a/py_modules/async_timeout/__init__.py b/py_modules/async_timeout/__init__.py new file mode 100644 index 0000000..fe4aa58 --- /dev/null +++ b/py_modules/async_timeout/__init__.py @@ -0,0 +1,276 @@ +import asyncio +import enum +import sys +from types import TracebackType +from typing import Optional, Type, final + + +__version__ = "5.0.1" + + +__all__ = ("timeout", "timeout_at", "Timeout") + + +def timeout(delay: Optional[float]) -> "Timeout": + """timeout context manager. + + Useful in cases when you want to apply timeout logic around block + of code or in cases when asyncio.wait_for is not suitable. For example: + + >>> async with timeout(0.001): + ... async with aiohttp.get('https://github.com') as r: + ... await r.text() + + + delay - value in seconds or None to disable timeout logic + """ + loop = asyncio.get_running_loop() + if delay is not None: + deadline = loop.time() + delay # type: Optional[float] + else: + deadline = None + return Timeout(deadline, loop) + + +def timeout_at(deadline: Optional[float]) -> "Timeout": + """Schedule the timeout at absolute time. + + deadline argument points on the time in the same clock system + as loop.time(). + + Please note: it is not POSIX time but a time with + undefined starting base, e.g. the time of the system power on. + + >>> async with timeout_at(loop.time() + 10): + ... async with aiohttp.get('https://github.com') as r: + ... await r.text() + + + """ + loop = asyncio.get_running_loop() + return Timeout(deadline, loop) + + +class _State(enum.Enum): + INIT = "INIT" + ENTER = "ENTER" + TIMEOUT = "TIMEOUT" + EXIT = "EXIT" + + +if sys.version_info >= (3, 11): + + class _Expired: + __slots__ = ("_val",) + + def __init__(self, val: bool) -> None: + self._val = val + + def __call__(self) -> bool: + return self._val + + def __bool__(self) -> bool: + return self._val + + def __repr__(self) -> str: + return repr(self._val) + + def __str__(self) -> str: + return str(self._val) + + @final + class Timeout(asyncio.Timeout): # type: ignore[misc] + # Supports full asyncio.Timeout API. + # Also provides several asyncio_timeout specific methods + # for backward compatibility. + def __init__( + self, deadline: Optional[float], loop: asyncio.AbstractEventLoop + ) -> None: + super().__init__(deadline) + + @property + def expired(self) -> _Expired: + # a hacky property hat can provide both roles: + # timeout.expired() from asyncio + # timeout.expired from asyncio_timeout + return _Expired(super().expired()) + + @property + def deadline(self) -> Optional[float]: + return self.when() + + def reject(self) -> None: + """Reject scheduled timeout if any.""" + # cancel is maybe better name but + # task.cancel() raises CancelledError in asyncio world. + self.reschedule(None) + + def shift(self, delay: float) -> None: + """Advance timeout on delay seconds. + + The delay can be negative. + + Raise RuntimeError if shift is called when deadline is not scheduled + """ + deadline = self.when() + if deadline is None: + raise RuntimeError("cannot shift timeout if deadline is not scheduled") + self.reschedule(deadline + delay) + + def update(self, deadline: float) -> None: + """Set deadline to absolute value. + + deadline argument points on the time in the same clock system + as loop.time(). + + If new deadline is in the past the timeout is raised immediately. + + Please note: it is not POSIX time but a time with + undefined starting base, e.g. the time of the system power on. + """ + self.reschedule(deadline) + +else: + + @final + class Timeout: + # Internal class, please don't instantiate it directly + # Use timeout() and timeout_at() public factories instead. + # + # Implementation note: `async with timeout()` is preferred + # over `with timeout()`. + # While technically the Timeout class implementation + # doesn't need to be async at all, + # the `async with` statement explicitly points that + # the context manager should be used from async function context. + # + # This design allows to avoid many silly misusages. + # + # TimeoutError is raised immediately when scheduled + # if the deadline is passed. + # The purpose is to time out as soon as possible + # without waiting for the next await expression. + + __slots__ = ("_deadline", "_loop", "_state", "_timeout_handler", "_task") + + def __init__( + self, deadline: Optional[float], loop: asyncio.AbstractEventLoop + ) -> None: + self._loop = loop + self._state = _State.INIT + + self._task: Optional["asyncio.Task[object]"] = None + self._timeout_handler = None # type: Optional[asyncio.Handle] + if deadline is None: + self._deadline = None # type: Optional[float] + else: + self.update(deadline) + + async def __aenter__(self) -> "Timeout": + self._do_enter() + return self + + async def __aexit__( + self, + exc_type: Optional[Type[BaseException]], + exc_val: Optional[BaseException], + exc_tb: Optional[TracebackType], + ) -> Optional[bool]: + self._do_exit(exc_type) + return None + + @property + def expired(self) -> bool: + """Is timeout expired during execution?""" + return self._state == _State.TIMEOUT + + @property + def deadline(self) -> Optional[float]: + return self._deadline + + def reject(self) -> None: + """Reject scheduled timeout if any.""" + # cancel is maybe better name but + # task.cancel() raises CancelledError in asyncio world. + if self._state not in (_State.INIT, _State.ENTER): + raise RuntimeError(f"invalid state {self._state.value}") + self._reject() + + def _reject(self) -> None: + self._task = None + if self._timeout_handler is not None: + self._timeout_handler.cancel() + self._timeout_handler = None + + def shift(self, delay: float) -> None: + """Advance timeout on delay seconds. + + The delay can be negative. + + Raise RuntimeError if shift is called when deadline is not scheduled + """ + deadline = self._deadline + if deadline is None: + raise RuntimeError("cannot shift timeout if deadline is not scheduled") + self.update(deadline + delay) + + def update(self, deadline: float) -> None: + """Set deadline to absolute value. + + deadline argument points on the time in the same clock system + as loop.time(). + + If new deadline is in the past the timeout is raised immediately. + + Please note: it is not POSIX time but a time with + undefined starting base, e.g. the time of the system power on. + """ + if self._state == _State.EXIT: + raise RuntimeError("cannot reschedule after exit from context manager") + if self._state == _State.TIMEOUT: + raise RuntimeError("cannot reschedule expired timeout") + if self._timeout_handler is not None: + self._timeout_handler.cancel() + self._deadline = deadline + if self._state != _State.INIT: + self._reschedule() + + def _reschedule(self) -> None: + assert self._state == _State.ENTER + deadline = self._deadline + if deadline is None: + return + + now = self._loop.time() + if self._timeout_handler is not None: + self._timeout_handler.cancel() + + self._task = asyncio.current_task() + if deadline <= now: + self._timeout_handler = self._loop.call_soon(self._on_timeout) + else: + self._timeout_handler = self._loop.call_at(deadline, self._on_timeout) + + def _do_enter(self) -> None: + if self._state != _State.INIT: + raise RuntimeError(f"invalid state {self._state.value}") + self._state = _State.ENTER + self._reschedule() + + def _do_exit(self, exc_type: Optional[Type[BaseException]]) -> None: + if exc_type is asyncio.CancelledError and self._state == _State.TIMEOUT: + assert self._task is not None + self._timeout_handler = None + self._task = None + raise asyncio.TimeoutError + # timeout has not expired + self._state = _State.EXIT + self._reject() + return None + + def _on_timeout(self) -> None: + assert self._task is not None + self._task.cancel() + self._state = _State.TIMEOUT + # drop the reference early + self._timeout_handler = None diff --git a/py_modules/async_timeout/py.typed b/py_modules/async_timeout/py.typed new file mode 100644 index 0000000..3b94f91 --- /dev/null +++ b/py_modules/async_timeout/py.typed @@ -0,0 +1 @@ +Placeholder diff --git a/py_modules/attr/__init__.py b/py_modules/attr/__init__.py new file mode 100644 index 0000000..5c6e065 --- /dev/null +++ b/py_modules/attr/__init__.py @@ -0,0 +1,104 @@ +# SPDX-License-Identifier: MIT + +""" +Classes Without Boilerplate +""" + +from functools import partial +from typing import Callable, Literal, Protocol + +from . import converters, exceptions, filters, setters, validators +from ._cmp import cmp_using +from ._config import get_run_validators, set_run_validators +from ._funcs import asdict, assoc, astuple, has, resolve_types +from ._make import ( + NOTHING, + Attribute, + Converter, + Factory, + _Nothing, + attrib, + attrs, + evolve, + fields, + fields_dict, + make_class, + validate, +) +from ._next_gen import define, field, frozen, mutable +from ._version_info import VersionInfo + + +s = attributes = attrs +ib = attr = attrib +dataclass = partial(attrs, auto_attribs=True) # happy Easter ;) + + +class AttrsInstance(Protocol): + pass + + +NothingType = Literal[_Nothing.NOTHING] + +__all__ = [ + "NOTHING", + "Attribute", + "AttrsInstance", + "Converter", + "Factory", + "NothingType", + "asdict", + "assoc", + "astuple", + "attr", + "attrib", + "attributes", + "attrs", + "cmp_using", + "converters", + "define", + "evolve", + "exceptions", + "field", + "fields", + "fields_dict", + "filters", + "frozen", + "get_run_validators", + "has", + "ib", + "make_class", + "mutable", + "resolve_types", + "s", + "set_run_validators", + "setters", + "validate", + "validators", +] + + +def _make_getattr(mod_name: str) -> Callable: + """ + Create a metadata proxy for packaging information that uses *mod_name* in + its warnings and errors. + """ + + def __getattr__(name: str) -> str: + if name not in ("__version__", "__version_info__"): + msg = f"module {mod_name} has no attribute {name}" + raise AttributeError(msg) + + from importlib.metadata import metadata + + meta = metadata("attrs") + + if name == "__version_info__": + return VersionInfo._from_version_string(meta["version"]) + + return meta["version"] + + return __getattr__ + + +__getattr__ = _make_getattr(__name__) diff --git a/py_modules/attr/__init__.pyi b/py_modules/attr/__init__.pyi new file mode 100644 index 0000000..758decf --- /dev/null +++ b/py_modules/attr/__init__.pyi @@ -0,0 +1,389 @@ +import enum +import sys + +from typing import ( + Any, + Callable, + Generic, + Literal, + Mapping, + Protocol, + Sequence, + TypeVar, + overload, +) + +# `import X as X` is required to make these public +from . import converters as converters +from . import exceptions as exceptions +from . import filters as filters +from . import setters as setters +from . import validators as validators +from ._cmp import cmp_using as cmp_using +from ._typing_compat import AttrsInstance_ +from ._version_info import VersionInfo +from attrs import ( + define as define, + field as field, + mutable as mutable, + frozen as frozen, + _EqOrderType, + _ValidatorType, + _ConverterType, + _ReprArgType, + _OnSetAttrType, + _OnSetAttrArgType, + _FieldTransformer, + _ValidatorArgType, +) + +if sys.version_info >= (3, 10): + from typing import TypeGuard, TypeAlias +else: + from typing_extensions import TypeGuard, TypeAlias + +if sys.version_info >= (3, 11): + from typing import dataclass_transform +else: + from typing_extensions import dataclass_transform + +__version__: str +__version_info__: VersionInfo +__title__: str +__description__: str +__url__: str +__uri__: str +__author__: str +__email__: str +__license__: str +__copyright__: str + +_T = TypeVar("_T") +_C = TypeVar("_C", bound=type) + +_FilterType = Callable[["Attribute[_T]", _T], bool] + +# We subclass this here to keep the protocol's qualified name clean. +class AttrsInstance(AttrsInstance_, Protocol): + pass + +_A = TypeVar("_A", bound=type[AttrsInstance]) + +class _Nothing(enum.Enum): + NOTHING = enum.auto() + +NOTHING = _Nothing.NOTHING +NothingType: TypeAlias = Literal[_Nothing.NOTHING] + +# NOTE: Factory lies about its return type to make this possible: +# `x: List[int] # = Factory(list)` +# Work around mypy issue #4554 in the common case by using an overload. + +@overload +def Factory(factory: Callable[[], _T]) -> _T: ... +@overload +def Factory( + factory: Callable[[Any], _T], + takes_self: Literal[True], +) -> _T: ... +@overload +def Factory( + factory: Callable[[], _T], + takes_self: Literal[False], +) -> _T: ... + +In = TypeVar("In") +Out = TypeVar("Out") + +class Converter(Generic[In, Out]): + @overload + def __init__(self, converter: Callable[[In], Out]) -> None: ... + @overload + def __init__( + self, + converter: Callable[[In, AttrsInstance, Attribute], Out], + *, + takes_self: Literal[True], + takes_field: Literal[True], + ) -> None: ... + @overload + def __init__( + self, + converter: Callable[[In, Attribute], Out], + *, + takes_field: Literal[True], + ) -> None: ... + @overload + def __init__( + self, + converter: Callable[[In, AttrsInstance], Out], + *, + takes_self: Literal[True], + ) -> None: ... + +class Attribute(Generic[_T]): + name: str + default: _T | None + validator: _ValidatorType[_T] | None + repr: _ReprArgType + cmp: _EqOrderType + eq: _EqOrderType + order: _EqOrderType + hash: bool | None + init: bool + converter: Converter | None + metadata: dict[Any, Any] + type: type[_T] | None + kw_only: bool + on_setattr: _OnSetAttrType + alias: str | None + + def evolve(self, **changes: Any) -> "Attribute[Any]": ... + +# NOTE: We had several choices for the annotation to use for type arg: +# 1) Type[_T] +# - Pros: Handles simple cases correctly +# - Cons: Might produce less informative errors in the case of conflicting +# TypeVars e.g. `attr.ib(default='bad', type=int)` +# 2) Callable[..., _T] +# - Pros: Better error messages than #1 for conflicting TypeVars +# - Cons: Terrible error messages for validator checks. +# e.g. attr.ib(type=int, validator=validate_str) +# -> error: Cannot infer function type argument +# 3) type (and do all of the work in the mypy plugin) +# - Pros: Simple here, and we could customize the plugin with our own errors. +# - Cons: Would need to write mypy plugin code to handle all the cases. +# We chose option #1. + +# `attr` lies about its return type to make the following possible: +# attr() -> Any +# attr(8) -> int +# attr(validator=) -> Whatever the callable expects. +# This makes this type of assignments possible: +# x: int = attr(8) +# +# This form catches explicit None or no default but with no other arguments +# returns Any. +@overload +def attrib( + default: None = ..., + validator: None = ..., + repr: _ReprArgType = ..., + cmp: _EqOrderType | None = ..., + hash: bool | None = ..., + init: bool = ..., + metadata: Mapping[Any, Any] | None = ..., + type: None = ..., + converter: None = ..., + factory: None = ..., + kw_only: bool | None = ..., + eq: _EqOrderType | None = ..., + order: _EqOrderType | None = ..., + on_setattr: _OnSetAttrArgType | None = ..., + alias: str | None = ..., +) -> Any: ... + +# This form catches an explicit None or no default and infers the type from the +# other arguments. +@overload +def attrib( + default: None = ..., + validator: _ValidatorArgType[_T] | None = ..., + repr: _ReprArgType = ..., + cmp: _EqOrderType | None = ..., + hash: bool | None = ..., + init: bool = ..., + metadata: Mapping[Any, Any] | None = ..., + type: type[_T] | None = ..., + converter: _ConverterType + | list[_ConverterType] + | tuple[_ConverterType] + | None = ..., + factory: Callable[[], _T] | None = ..., + kw_only: bool | None = ..., + eq: _EqOrderType | None = ..., + order: _EqOrderType | None = ..., + on_setattr: _OnSetAttrArgType | None = ..., + alias: str | None = ..., +) -> _T: ... + +# This form catches an explicit default argument. +@overload +def attrib( + default: _T, + validator: _ValidatorArgType[_T] | None = ..., + repr: _ReprArgType = ..., + cmp: _EqOrderType | None = ..., + hash: bool | None = ..., + init: bool = ..., + metadata: Mapping[Any, Any] | None = ..., + type: type[_T] | None = ..., + converter: _ConverterType + | list[_ConverterType] + | tuple[_ConverterType] + | None = ..., + factory: Callable[[], _T] | None = ..., + kw_only: bool | None = ..., + eq: _EqOrderType | None = ..., + order: _EqOrderType | None = ..., + on_setattr: _OnSetAttrArgType | None = ..., + alias: str | None = ..., +) -> _T: ... + +# This form covers type=non-Type: e.g. forward references (str), Any +@overload +def attrib( + default: _T | None = ..., + validator: _ValidatorArgType[_T] | None = ..., + repr: _ReprArgType = ..., + cmp: _EqOrderType | None = ..., + hash: bool | None = ..., + init: bool = ..., + metadata: Mapping[Any, Any] | None = ..., + type: object = ..., + converter: _ConverterType + | list[_ConverterType] + | tuple[_ConverterType] + | None = ..., + factory: Callable[[], _T] | None = ..., + kw_only: bool | None = ..., + eq: _EqOrderType | None = ..., + order: _EqOrderType | None = ..., + on_setattr: _OnSetAttrArgType | None = ..., + alias: str | None = ..., +) -> Any: ... +@overload +@dataclass_transform(order_default=True, field_specifiers=(attrib, field)) +def attrs( + maybe_cls: _C, + these: dict[str, Any] | None = ..., + repr_ns: str | None = ..., + repr: bool = ..., + cmp: _EqOrderType | None = ..., + hash: bool | None = ..., + init: bool = ..., + slots: bool = ..., + frozen: bool = ..., + weakref_slot: bool = ..., + str: bool = ..., + auto_attribs: bool = ..., + kw_only: bool = ..., + cache_hash: bool = ..., + auto_exc: bool = ..., + eq: _EqOrderType | None = ..., + order: _EqOrderType | None = ..., + auto_detect: bool = ..., + collect_by_mro: bool = ..., + getstate_setstate: bool | None = ..., + on_setattr: _OnSetAttrArgType | None = ..., + field_transformer: _FieldTransformer | None = ..., + match_args: bool = ..., + unsafe_hash: bool | None = ..., +) -> _C: ... +@overload +@dataclass_transform(order_default=True, field_specifiers=(attrib, field)) +def attrs( + maybe_cls: None = ..., + these: dict[str, Any] | None = ..., + repr_ns: str | None = ..., + repr: bool = ..., + cmp: _EqOrderType | None = ..., + hash: bool | None = ..., + init: bool = ..., + slots: bool = ..., + frozen: bool = ..., + weakref_slot: bool = ..., + str: bool = ..., + auto_attribs: bool = ..., + kw_only: bool = ..., + cache_hash: bool = ..., + auto_exc: bool = ..., + eq: _EqOrderType | None = ..., + order: _EqOrderType | None = ..., + auto_detect: bool = ..., + collect_by_mro: bool = ..., + getstate_setstate: bool | None = ..., + on_setattr: _OnSetAttrArgType | None = ..., + field_transformer: _FieldTransformer | None = ..., + match_args: bool = ..., + unsafe_hash: bool | None = ..., +) -> Callable[[_C], _C]: ... +def fields(cls: type[AttrsInstance] | AttrsInstance) -> Any: ... +def fields_dict(cls: type[AttrsInstance]) -> dict[str, Attribute[Any]]: ... +def validate(inst: AttrsInstance) -> None: ... +def resolve_types( + cls: _A, + globalns: dict[str, Any] | None = ..., + localns: dict[str, Any] | None = ..., + attribs: list[Attribute[Any]] | None = ..., + include_extras: bool = ..., +) -> _A: ... + +# TODO: add support for returning a proper attrs class from the mypy plugin +# we use Any instead of _CountingAttr so that e.g. `make_class('Foo', +# [attr.ib()])` is valid +def make_class( + name: str, + attrs: list[str] | tuple[str, ...] | dict[str, Any], + bases: tuple[type, ...] = ..., + class_body: dict[str, Any] | None = ..., + repr_ns: str | None = ..., + repr: bool = ..., + cmp: _EqOrderType | None = ..., + hash: bool | None = ..., + init: bool = ..., + slots: bool = ..., + frozen: bool = ..., + weakref_slot: bool = ..., + str: bool = ..., + auto_attribs: bool = ..., + kw_only: bool = ..., + cache_hash: bool = ..., + auto_exc: bool = ..., + eq: _EqOrderType | None = ..., + order: _EqOrderType | None = ..., + collect_by_mro: bool = ..., + on_setattr: _OnSetAttrArgType | None = ..., + field_transformer: _FieldTransformer | None = ..., +) -> type: ... + +# _funcs -- + +# TODO: add support for returning TypedDict from the mypy plugin +# FIXME: asdict/astuple do not honor their factory args. Waiting on one of +# these: +# https://github.com/python/mypy/issues/4236 +# https://github.com/python/typing/issues/253 +# XXX: remember to fix attrs.asdict/astuple too! +def asdict( + inst: AttrsInstance, + recurse: bool = ..., + filter: _FilterType[Any] | None = ..., + dict_factory: type[Mapping[Any, Any]] = ..., + retain_collection_types: bool = ..., + value_serializer: Callable[[type, Attribute[Any], Any], Any] | None = ..., + tuple_keys: bool | None = ..., +) -> dict[str, Any]: ... + +# TODO: add support for returning NamedTuple from the mypy plugin +def astuple( + inst: AttrsInstance, + recurse: bool = ..., + filter: _FilterType[Any] | None = ..., + tuple_factory: type[Sequence[Any]] = ..., + retain_collection_types: bool = ..., +) -> tuple[Any, ...]: ... +def has(cls: type) -> TypeGuard[type[AttrsInstance]]: ... +def assoc(inst: _T, **changes: Any) -> _T: ... +def evolve(inst: _T, **changes: Any) -> _T: ... + +# _config -- + +def set_run_validators(run: bool) -> None: ... +def get_run_validators() -> bool: ... + +# aliases -- + +s = attributes = attrs +ib = attr = attrib +dataclass = attrs # Technically, partial(attrs, auto_attribs=True) ;) diff --git a/py_modules/attr/_cmp.py b/py_modules/attr/_cmp.py new file mode 100644 index 0000000..09bab49 --- /dev/null +++ b/py_modules/attr/_cmp.py @@ -0,0 +1,160 @@ +# SPDX-License-Identifier: MIT + + +import functools +import types + +from ._make import __ne__ + + +_operation_names = {"eq": "==", "lt": "<", "le": "<=", "gt": ">", "ge": ">="} + + +def cmp_using( + eq=None, + lt=None, + le=None, + gt=None, + ge=None, + require_same_type=True, + class_name="Comparable", +): + """ + Create a class that can be passed into `attrs.field`'s ``eq``, ``order``, + and ``cmp`` arguments to customize field comparison. + + The resulting class will have a full set of ordering methods if at least + one of ``{lt, le, gt, ge}`` and ``eq`` are provided. + + Args: + eq (typing.Callable | None): + Callable used to evaluate equality of two objects. + + lt (typing.Callable | None): + Callable used to evaluate whether one object is less than another + object. + + le (typing.Callable | None): + Callable used to evaluate whether one object is less than or equal + to another object. + + gt (typing.Callable | None): + Callable used to evaluate whether one object is greater than + another object. + + ge (typing.Callable | None): + Callable used to evaluate whether one object is greater than or + equal to another object. + + require_same_type (bool): + When `True`, equality and ordering methods will return + `NotImplemented` if objects are not of the same type. + + class_name (str | None): Name of class. Defaults to "Comparable". + + See `comparison` for more details. + + .. versionadded:: 21.1.0 + """ + + body = { + "__slots__": ["value"], + "__init__": _make_init(), + "_requirements": [], + "_is_comparable_to": _is_comparable_to, + } + + # Add operations. + num_order_functions = 0 + has_eq_function = False + + if eq is not None: + has_eq_function = True + body["__eq__"] = _make_operator("eq", eq) + body["__ne__"] = __ne__ + + if lt is not None: + num_order_functions += 1 + body["__lt__"] = _make_operator("lt", lt) + + if le is not None: + num_order_functions += 1 + body["__le__"] = _make_operator("le", le) + + if gt is not None: + num_order_functions += 1 + body["__gt__"] = _make_operator("gt", gt) + + if ge is not None: + num_order_functions += 1 + body["__ge__"] = _make_operator("ge", ge) + + type_ = types.new_class( + class_name, (object,), {}, lambda ns: ns.update(body) + ) + + # Add same type requirement. + if require_same_type: + type_._requirements.append(_check_same_type) + + # Add total ordering if at least one operation was defined. + if 0 < num_order_functions < 4: + if not has_eq_function: + # functools.total_ordering requires __eq__ to be defined, + # so raise early error here to keep a nice stack. + msg = "eq must be define is order to complete ordering from lt, le, gt, ge." + raise ValueError(msg) + type_ = functools.total_ordering(type_) + + return type_ + + +def _make_init(): + """ + Create __init__ method. + """ + + def __init__(self, value): + """ + Initialize object with *value*. + """ + self.value = value + + return __init__ + + +def _make_operator(name, func): + """ + Create operator method. + """ + + def method(self, other): + if not self._is_comparable_to(other): + return NotImplemented + + result = func(self.value, other.value) + if result is NotImplemented: + return NotImplemented + + return result + + method.__name__ = f"__{name}__" + method.__doc__ = ( + f"Return a {_operation_names[name]} b. Computed by attrs." + ) + + return method + + +def _is_comparable_to(self, other): + """ + Check whether `other` is comparable to `self`. + """ + return all(func(self, other) for func in self._requirements) + + +def _check_same_type(self, other): + """ + Return True if *self* and *other* are of the same type, False otherwise. + """ + return other.value.__class__ is self.value.__class__ diff --git a/py_modules/attr/_cmp.pyi b/py_modules/attr/_cmp.pyi new file mode 100644 index 0000000..cc7893b --- /dev/null +++ b/py_modules/attr/_cmp.pyi @@ -0,0 +1,13 @@ +from typing import Any, Callable + +_CompareWithType = Callable[[Any, Any], bool] + +def cmp_using( + eq: _CompareWithType | None = ..., + lt: _CompareWithType | None = ..., + le: _CompareWithType | None = ..., + gt: _CompareWithType | None = ..., + ge: _CompareWithType | None = ..., + require_same_type: bool = ..., + class_name: str = ..., +) -> type: ... diff --git a/py_modules/attr/_compat.py b/py_modules/attr/_compat.py new file mode 100644 index 0000000..bc68ed9 --- /dev/null +++ b/py_modules/attr/_compat.py @@ -0,0 +1,99 @@ +# SPDX-License-Identifier: MIT + +import inspect +import platform +import sys +import threading + +from collections.abc import Mapping, Sequence # noqa: F401 +from typing import _GenericAlias + + +PYPY = platform.python_implementation() == "PyPy" +PY_3_10_PLUS = sys.version_info[:2] >= (3, 10) +PY_3_11_PLUS = sys.version_info[:2] >= (3, 11) +PY_3_12_PLUS = sys.version_info[:2] >= (3, 12) +PY_3_13_PLUS = sys.version_info[:2] >= (3, 13) +PY_3_14_PLUS = sys.version_info[:2] >= (3, 14) + + +if PY_3_14_PLUS: + import annotationlib + + # We request forward-ref annotations to not break in the presence of + # forward references. + + def _get_annotations(cls): + return annotationlib.get_annotations( + cls, format=annotationlib.Format.FORWARDREF + ) + +else: + + def _get_annotations(cls): + """ + Get annotations for *cls*. + """ + return cls.__dict__.get("__annotations__", {}) + + +class _AnnotationExtractor: + """ + Extract type annotations from a callable, returning None whenever there + is none. + """ + + __slots__ = ["sig"] + + def __init__(self, callable): + try: + self.sig = inspect.signature(callable) + except (ValueError, TypeError): # inspect failed + self.sig = None + + def get_first_param_type(self): + """ + Return the type annotation of the first argument if it's not empty. + """ + if not self.sig: + return None + + params = list(self.sig.parameters.values()) + if params and params[0].annotation is not inspect.Parameter.empty: + return params[0].annotation + + return None + + def get_return_type(self): + """ + Return the return type if it's not empty. + """ + if ( + self.sig + and self.sig.return_annotation is not inspect.Signature.empty + ): + return self.sig.return_annotation + + return None + + +# Thread-local global to track attrs instances which are already being repr'd. +# This is needed because there is no other (thread-safe) way to pass info +# about the instances that are already being repr'd through the call stack +# in order to ensure we don't perform infinite recursion. +# +# For instance, if an instance contains a dict which contains that instance, +# we need to know that we're already repr'ing the outside instance from within +# the dict's repr() call. +# +# This lives here rather than in _make.py so that the functions in _make.py +# don't have a direct reference to the thread-local in their globals dict. +# If they have such a reference, it breaks cloudpickle. +repr_context = threading.local() + + +def get_generic_base(cl): + """If this is a generic class (A[str]), return the generic base for it.""" + if cl.__class__ is _GenericAlias: + return cl.__origin__ + return None diff --git a/py_modules/attr/_config.py b/py_modules/attr/_config.py new file mode 100644 index 0000000..4b25772 --- /dev/null +++ b/py_modules/attr/_config.py @@ -0,0 +1,31 @@ +# SPDX-License-Identifier: MIT + +__all__ = ["get_run_validators", "set_run_validators"] + +_run_validators = True + + +def set_run_validators(run): + """ + Set whether or not validators are run. By default, they are run. + + .. deprecated:: 21.3.0 It will not be removed, but it also will not be + moved to new ``attrs`` namespace. Use `attrs.validators.set_disabled()` + instead. + """ + if not isinstance(run, bool): + msg = "'run' must be bool." + raise TypeError(msg) + global _run_validators + _run_validators = run + + +def get_run_validators(): + """ + Return whether or not validators are run. + + .. deprecated:: 21.3.0 It will not be removed, but it also will not be + moved to new ``attrs`` namespace. Use `attrs.validators.get_disabled()` + instead. + """ + return _run_validators diff --git a/py_modules/attr/_funcs.py b/py_modules/attr/_funcs.py new file mode 100644 index 0000000..1adb500 --- /dev/null +++ b/py_modules/attr/_funcs.py @@ -0,0 +1,497 @@ +# SPDX-License-Identifier: MIT + + +import copy + +from ._compat import get_generic_base +from ._make import _OBJ_SETATTR, NOTHING, fields +from .exceptions import AttrsAttributeNotFoundError + + +_ATOMIC_TYPES = frozenset( + { + type(None), + bool, + int, + float, + str, + complex, + bytes, + type(...), + type, + range, + property, + } +) + + +def asdict( + inst, + recurse=True, + filter=None, + dict_factory=dict, + retain_collection_types=False, + value_serializer=None, +): + """ + Return the *attrs* attribute values of *inst* as a dict. + + Optionally recurse into other *attrs*-decorated classes. + + Args: + inst: Instance of an *attrs*-decorated class. + + recurse (bool): Recurse into classes that are also *attrs*-decorated. + + filter (~typing.Callable): + A callable whose return code determines whether an attribute or + element is included (`True`) or dropped (`False`). Is called with + the `attrs.Attribute` as the first argument and the value as the + second argument. + + dict_factory (~typing.Callable): + A callable to produce dictionaries from. For example, to produce + ordered dictionaries instead of normal Python dictionaries, pass in + ``collections.OrderedDict``. + + retain_collection_types (bool): + Do not convert to `list` when encountering an attribute whose type + is `tuple` or `set`. Only meaningful if *recurse* is `True`. + + value_serializer (typing.Callable | None): + A hook that is called for every attribute or dict key/value. It + receives the current instance, field and value and must return the + (updated) value. The hook is run *after* the optional *filter* has + been applied. + + Returns: + Return type of *dict_factory*. + + Raises: + attrs.exceptions.NotAnAttrsClassError: + If *cls* is not an *attrs* class. + + .. versionadded:: 16.0.0 *dict_factory* + .. versionadded:: 16.1.0 *retain_collection_types* + .. versionadded:: 20.3.0 *value_serializer* + .. versionadded:: 21.3.0 + If a dict has a collection for a key, it is serialized as a tuple. + """ + attrs = fields(inst.__class__) + rv = dict_factory() + for a in attrs: + v = getattr(inst, a.name) + if filter is not None and not filter(a, v): + continue + + if value_serializer is not None: + v = value_serializer(inst, a, v) + + if recurse is True: + value_type = type(v) + if value_type in _ATOMIC_TYPES: + rv[a.name] = v + elif has(value_type): + rv[a.name] = asdict( + v, + recurse=True, + filter=filter, + dict_factory=dict_factory, + retain_collection_types=retain_collection_types, + value_serializer=value_serializer, + ) + elif issubclass(value_type, (tuple, list, set, frozenset)): + cf = value_type if retain_collection_types is True else list + items = [ + _asdict_anything( + i, + is_key=False, + filter=filter, + dict_factory=dict_factory, + retain_collection_types=retain_collection_types, + value_serializer=value_serializer, + ) + for i in v + ] + try: + rv[a.name] = cf(items) + except TypeError: + if not issubclass(cf, tuple): + raise + # Workaround for TypeError: cf.__new__() missing 1 required + # positional argument (which appears, for a namedturle) + rv[a.name] = cf(*items) + elif issubclass(value_type, dict): + df = dict_factory + rv[a.name] = df( + ( + _asdict_anything( + kk, + is_key=True, + filter=filter, + dict_factory=df, + retain_collection_types=retain_collection_types, + value_serializer=value_serializer, + ), + _asdict_anything( + vv, + is_key=False, + filter=filter, + dict_factory=df, + retain_collection_types=retain_collection_types, + value_serializer=value_serializer, + ), + ) + for kk, vv in v.items() + ) + else: + rv[a.name] = v + else: + rv[a.name] = v + return rv + + +def _asdict_anything( + val, + is_key, + filter, + dict_factory, + retain_collection_types, + value_serializer, +): + """ + ``asdict`` only works on attrs instances, this works on anything. + """ + val_type = type(val) + if val_type in _ATOMIC_TYPES: + rv = val + if value_serializer is not None: + rv = value_serializer(None, None, rv) + elif getattr(val_type, "__attrs_attrs__", None) is not None: + # Attrs class. + rv = asdict( + val, + recurse=True, + filter=filter, + dict_factory=dict_factory, + retain_collection_types=retain_collection_types, + value_serializer=value_serializer, + ) + elif issubclass(val_type, (tuple, list, set, frozenset)): + if retain_collection_types is True: + cf = val.__class__ + elif is_key: + cf = tuple + else: + cf = list + + rv = cf( + [ + _asdict_anything( + i, + is_key=False, + filter=filter, + dict_factory=dict_factory, + retain_collection_types=retain_collection_types, + value_serializer=value_serializer, + ) + for i in val + ] + ) + elif issubclass(val_type, dict): + df = dict_factory + rv = df( + ( + _asdict_anything( + kk, + is_key=True, + filter=filter, + dict_factory=df, + retain_collection_types=retain_collection_types, + value_serializer=value_serializer, + ), + _asdict_anything( + vv, + is_key=False, + filter=filter, + dict_factory=df, + retain_collection_types=retain_collection_types, + value_serializer=value_serializer, + ), + ) + for kk, vv in val.items() + ) + else: + rv = val + if value_serializer is not None: + rv = value_serializer(None, None, rv) + + return rv + + +def astuple( + inst, + recurse=True, + filter=None, + tuple_factory=tuple, + retain_collection_types=False, +): + """ + Return the *attrs* attribute values of *inst* as a tuple. + + Optionally recurse into other *attrs*-decorated classes. + + Args: + inst: Instance of an *attrs*-decorated class. + + recurse (bool): + Recurse into classes that are also *attrs*-decorated. + + filter (~typing.Callable): + A callable whose return code determines whether an attribute or + element is included (`True`) or dropped (`False`). Is called with + the `attrs.Attribute` as the first argument and the value as the + second argument. + + tuple_factory (~typing.Callable): + A callable to produce tuples from. For example, to produce lists + instead of tuples. + + retain_collection_types (bool): + Do not convert to `list` or `dict` when encountering an attribute + which type is `tuple`, `dict` or `set`. Only meaningful if + *recurse* is `True`. + + Returns: + Return type of *tuple_factory* + + Raises: + attrs.exceptions.NotAnAttrsClassError: + If *cls* is not an *attrs* class. + + .. versionadded:: 16.2.0 + """ + attrs = fields(inst.__class__) + rv = [] + retain = retain_collection_types # Very long. :/ + for a in attrs: + v = getattr(inst, a.name) + if filter is not None and not filter(a, v): + continue + value_type = type(v) + if recurse is True: + if value_type in _ATOMIC_TYPES: + rv.append(v) + elif has(value_type): + rv.append( + astuple( + v, + recurse=True, + filter=filter, + tuple_factory=tuple_factory, + retain_collection_types=retain, + ) + ) + elif issubclass(value_type, (tuple, list, set, frozenset)): + cf = v.__class__ if retain is True else list + items = [ + ( + astuple( + j, + recurse=True, + filter=filter, + tuple_factory=tuple_factory, + retain_collection_types=retain, + ) + if has(j.__class__) + else j + ) + for j in v + ] + try: + rv.append(cf(items)) + except TypeError: + if not issubclass(cf, tuple): + raise + # Workaround for TypeError: cf.__new__() missing 1 required + # positional argument (which appears, for a namedturle) + rv.append(cf(*items)) + elif issubclass(value_type, dict): + df = value_type if retain is True else dict + rv.append( + df( + ( + ( + astuple( + kk, + tuple_factory=tuple_factory, + retain_collection_types=retain, + ) + if has(kk.__class__) + else kk + ), + ( + astuple( + vv, + tuple_factory=tuple_factory, + retain_collection_types=retain, + ) + if has(vv.__class__) + else vv + ), + ) + for kk, vv in v.items() + ) + ) + else: + rv.append(v) + else: + rv.append(v) + + return rv if tuple_factory is list else tuple_factory(rv) + + +def has(cls): + """ + Check whether *cls* is a class with *attrs* attributes. + + Args: + cls (type): Class to introspect. + + Raises: + TypeError: If *cls* is not a class. + + Returns: + bool: + """ + attrs = getattr(cls, "__attrs_attrs__", None) + if attrs is not None: + return True + + # No attrs, maybe it's a specialized generic (A[str])? + generic_base = get_generic_base(cls) + if generic_base is not None: + generic_attrs = getattr(generic_base, "__attrs_attrs__", None) + if generic_attrs is not None: + # Stick it on here for speed next time. + cls.__attrs_attrs__ = generic_attrs + return generic_attrs is not None + return False + + +def assoc(inst, **changes): + """ + Copy *inst* and apply *changes*. + + This is different from `evolve` that applies the changes to the arguments + that create the new instance. + + `evolve`'s behavior is preferable, but there are `edge cases`_ where it + doesn't work. Therefore `assoc` is deprecated, but will not be removed. + + .. _`edge cases`: https://github.com/python-attrs/attrs/issues/251 + + Args: + inst: Instance of a class with *attrs* attributes. + + changes: Keyword changes in the new copy. + + Returns: + A copy of inst with *changes* incorporated. + + Raises: + attrs.exceptions.AttrsAttributeNotFoundError: + If *attr_name* couldn't be found on *cls*. + + attrs.exceptions.NotAnAttrsClassError: + If *cls* is not an *attrs* class. + + .. deprecated:: 17.1.0 + Use `attrs.evolve` instead if you can. This function will not be + removed du to the slightly different approach compared to + `attrs.evolve`, though. + """ + new = copy.copy(inst) + attrs = fields(inst.__class__) + for k, v in changes.items(): + a = getattr(attrs, k, NOTHING) + if a is NOTHING: + msg = f"{k} is not an attrs attribute on {new.__class__}." + raise AttrsAttributeNotFoundError(msg) + _OBJ_SETATTR(new, k, v) + return new + + +def resolve_types( + cls, globalns=None, localns=None, attribs=None, include_extras=True +): + """ + Resolve any strings and forward annotations in type annotations. + + This is only required if you need concrete types in :class:`Attribute`'s + *type* field. In other words, you don't need to resolve your types if you + only use them for static type checking. + + With no arguments, names will be looked up in the module in which the class + was created. If this is not what you want, for example, if the name only + exists inside a method, you may pass *globalns* or *localns* to specify + other dictionaries in which to look up these names. See the docs of + `typing.get_type_hints` for more details. + + Args: + cls (type): Class to resolve. + + globalns (dict | None): Dictionary containing global variables. + + localns (dict | None): Dictionary containing local variables. + + attribs (list | None): + List of attribs for the given class. This is necessary when calling + from inside a ``field_transformer`` since *cls* is not an *attrs* + class yet. + + include_extras (bool): + Resolve more accurately, if possible. Pass ``include_extras`` to + ``typing.get_hints``, if supported by the typing module. On + supported Python versions (3.9+), this resolves the types more + accurately. + + Raises: + TypeError: If *cls* is not a class. + + attrs.exceptions.NotAnAttrsClassError: + If *cls* is not an *attrs* class and you didn't pass any attribs. + + NameError: If types cannot be resolved because of missing variables. + + Returns: + *cls* so you can use this function also as a class decorator. Please + note that you have to apply it **after** `attrs.define`. That means the + decorator has to come in the line **before** `attrs.define`. + + .. versionadded:: 20.1.0 + .. versionadded:: 21.1.0 *attribs* + .. versionadded:: 23.1.0 *include_extras* + """ + # Since calling get_type_hints is expensive we cache whether we've + # done it already. + if getattr(cls, "__attrs_types_resolved__", None) != cls: + import typing + + kwargs = { + "globalns": globalns, + "localns": localns, + "include_extras": include_extras, + } + + hints = typing.get_type_hints(cls, **kwargs) + for field in fields(cls) if attribs is None else attribs: + if field.name in hints: + # Since fields have been frozen we must work around it. + _OBJ_SETATTR(field, "type", hints[field.name]) + # We store the class we resolved so that subclasses know they haven't + # been resolved. + cls.__attrs_types_resolved__ = cls + + # Return the class so you can use it as a decorator too. + return cls diff --git a/py_modules/attr/_make.py b/py_modules/attr/_make.py new file mode 100644 index 0000000..4b32d6a --- /dev/null +++ b/py_modules/attr/_make.py @@ -0,0 +1,3406 @@ +# SPDX-License-Identifier: MIT + +from __future__ import annotations + +import abc +import contextlib +import copy +import enum +import inspect +import itertools +import linecache +import sys +import types +import unicodedata +import weakref + +from collections.abc import Callable, Mapping +from functools import cached_property +from typing import Any, NamedTuple, TypeVar + +# We need to import _compat itself in addition to the _compat members to avoid +# having the thread-local in the globals here. +from . import _compat, _config, setters +from ._compat import ( + PY_3_10_PLUS, + PY_3_11_PLUS, + PY_3_13_PLUS, + _AnnotationExtractor, + _get_annotations, + get_generic_base, +) +from .exceptions import ( + DefaultAlreadySetError, + FrozenInstanceError, + NotAnAttrsClassError, + UnannotatedAttributeError, +) + + +# This is used at least twice, so cache it here. +_OBJ_SETATTR = object.__setattr__ +_INIT_FACTORY_PAT = "__attr_factory_%s" +_CLASSVAR_PREFIXES = ( + "typing.ClassVar", + "t.ClassVar", + "ClassVar", + "typing_extensions.ClassVar", +) +# we don't use a double-underscore prefix because that triggers +# name mangling when trying to create a slot for the field +# (when slots=True) +_HASH_CACHE_FIELD = "_attrs_cached_hash" + +_EMPTY_METADATA_SINGLETON = types.MappingProxyType({}) + +# Unique object for unequivocal getattr() defaults. +_SENTINEL = object() + +_DEFAULT_ON_SETATTR = setters.pipe(setters.convert, setters.validate) + + +class _Nothing(enum.Enum): + """ + Sentinel to indicate the lack of a value when `None` is ambiguous. + + If extending attrs, you can use ``typing.Literal[NOTHING]`` to show + that a value may be ``NOTHING``. + + .. versionchanged:: 21.1.0 ``bool(NOTHING)`` is now False. + .. versionchanged:: 22.2.0 ``NOTHING`` is now an ``enum.Enum`` variant. + """ + + NOTHING = enum.auto() + + def __repr__(self): + return "NOTHING" + + def __bool__(self): + return False + + +NOTHING = _Nothing.NOTHING +""" +Sentinel to indicate the lack of a value when `None` is ambiguous. + +When using in 3rd party code, use `attrs.NothingType` for type annotations. +""" + + +class _CacheHashWrapper(int): + """ + An integer subclass that pickles / copies as None + + This is used for non-slots classes with ``cache_hash=True``, to avoid + serializing a potentially (even likely) invalid hash value. Since `None` + is the default value for uncalculated hashes, whenever this is copied, + the copy's value for the hash should automatically reset. + + See GH #613 for more details. + """ + + def __reduce__(self, _none_constructor=type(None), _args=()): # noqa: B008 + return _none_constructor, _args + + +def attrib( + default=NOTHING, + validator=None, + repr=True, + cmp=None, + hash=None, + init=True, + metadata=None, + type=None, + converter=None, + factory=None, + kw_only=None, + eq=None, + order=None, + on_setattr=None, + alias=None, +): + """ + Create a new field / attribute on a class. + + Identical to `attrs.field`, except it's not keyword-only. + + Consider using `attrs.field` in new code (``attr.ib`` will *never* go away, + though). + + .. warning:: + + Does **nothing** unless the class is also decorated with + `attr.s` (or similar)! + + + .. versionadded:: 15.2.0 *convert* + .. versionadded:: 16.3.0 *metadata* + .. versionchanged:: 17.1.0 *validator* can be a ``list`` now. + .. versionchanged:: 17.1.0 + *hash* is `None` and therefore mirrors *eq* by default. + .. versionadded:: 17.3.0 *type* + .. deprecated:: 17.4.0 *convert* + .. versionadded:: 17.4.0 + *converter* as a replacement for the deprecated *convert* to achieve + consistency with other noun-based arguments. + .. versionadded:: 18.1.0 + ``factory=f`` is syntactic sugar for ``default=attr.Factory(f)``. + .. versionadded:: 18.2.0 *kw_only* + .. versionchanged:: 19.2.0 *convert* keyword argument removed. + .. versionchanged:: 19.2.0 *repr* also accepts a custom callable. + .. deprecated:: 19.2.0 *cmp* Removal on or after 2021-06-01. + .. versionadded:: 19.2.0 *eq* and *order* + .. versionadded:: 20.1.0 *on_setattr* + .. versionchanged:: 20.3.0 *kw_only* backported to Python 2 + .. versionchanged:: 21.1.0 + *eq*, *order*, and *cmp* also accept a custom callable + .. versionchanged:: 21.1.0 *cmp* undeprecated + .. versionadded:: 22.2.0 *alias* + .. versionchanged:: 25.4.0 + *kw_only* can now be None, and its default is also changed from False to + None. + """ + eq, eq_key, order, order_key = _determine_attrib_eq_order( + cmp, eq, order, True + ) + + if hash is not None and hash is not True and hash is not False: + msg = "Invalid value for hash. Must be True, False, or None." + raise TypeError(msg) + + if factory is not None: + if default is not NOTHING: + msg = ( + "The `default` and `factory` arguments are mutually exclusive." + ) + raise ValueError(msg) + if not callable(factory): + msg = "The `factory` argument must be a callable." + raise ValueError(msg) + default = Factory(factory) + + if metadata is None: + metadata = {} + + # Apply syntactic sugar by auto-wrapping. + if isinstance(on_setattr, (list, tuple)): + on_setattr = setters.pipe(*on_setattr) + + if validator and isinstance(validator, (list, tuple)): + validator = and_(*validator) + + if converter and isinstance(converter, (list, tuple)): + converter = pipe(*converter) + + return _CountingAttr( + default=default, + validator=validator, + repr=repr, + cmp=None, + hash=hash, + init=init, + converter=converter, + metadata=metadata, + type=type, + kw_only=kw_only, + eq=eq, + eq_key=eq_key, + order=order, + order_key=order_key, + on_setattr=on_setattr, + alias=alias, + ) + + +def _compile_and_eval( + script: str, + globs: dict[str, Any] | None, + locs: Mapping[str, object] | None = None, + filename: str = "", +) -> None: + """ + Evaluate the script with the given global (globs) and local (locs) + variables. + """ + bytecode = compile(script, filename, "exec") + eval(bytecode, globs, locs) + + +def _linecache_and_compile( + script: str, + filename: str, + globs: dict[str, Any] | None, + locals: Mapping[str, object] | None = None, +) -> dict[str, Any]: + """ + Cache the script with _linecache_, compile it and return the _locals_. + """ + + locs = {} if locals is None else locals + + # In order of debuggers like PDB being able to step through the code, + # we add a fake linecache entry. + count = 1 + base_filename = filename + while True: + linecache_tuple = ( + len(script), + None, + script.splitlines(True), + filename, + ) + old_val = linecache.cache.setdefault(filename, linecache_tuple) + if old_val == linecache_tuple: + break + + filename = f"{base_filename[:-1]}-{count}>" + count += 1 + + _compile_and_eval(script, globs, locs, filename) + + return locs + + +def _make_attr_tuple_class(cls_name: str, attr_names: list[str]) -> type: + """ + Create a tuple subclass to hold `Attribute`s for an `attrs` class. + + The subclass is a bare tuple with properties for names. + + class MyClassAttributes(tuple): + __slots__ = () + x = property(itemgetter(0)) + """ + attr_class_name = f"{cls_name}Attributes" + body = {} + for i, attr_name in enumerate(attr_names): + + def getter(self, i=i): + return self[i] + + body[attr_name] = property(getter) + return type(attr_class_name, (tuple,), body) + + +# Tuple class for extracted attributes from a class definition. +# `base_attrs` is a subset of `attrs`. +class _Attributes(NamedTuple): + attrs: type + base_attrs: list[Attribute] + base_attrs_map: dict[str, type] + + +def _is_class_var(annot): + """ + Check whether *annot* is a typing.ClassVar. + + The string comparison hack is used to avoid evaluating all string + annotations which would put attrs-based classes at a performance + disadvantage compared to plain old classes. + """ + annot = str(annot) + + # Annotation can be quoted. + if annot.startswith(("'", '"')) and annot.endswith(("'", '"')): + annot = annot[1:-1] + + return annot.startswith(_CLASSVAR_PREFIXES) + + +def _has_own_attribute(cls, attrib_name): + """ + Check whether *cls* defines *attrib_name* (and doesn't just inherit it). + """ + return attrib_name in cls.__dict__ + + +def _collect_base_attrs( + cls, taken_attr_names +) -> tuple[list[Attribute], dict[str, type]]: + """ + Collect attr.ibs from base classes of *cls*, except *taken_attr_names*. + """ + base_attrs = [] + base_attr_map = {} # A dictionary of base attrs to their classes. + + # Traverse the MRO and collect attributes. + for base_cls in reversed(cls.__mro__[1:-1]): + for a in getattr(base_cls, "__attrs_attrs__", []): + if a.inherited or a.name in taken_attr_names: + continue + + a = a.evolve(inherited=True) # noqa: PLW2901 + base_attrs.append(a) + base_attr_map[a.name] = base_cls + + # For each name, only keep the freshest definition i.e. the furthest at the + # back. base_attr_map is fine because it gets overwritten with every new + # instance. + filtered = [] + seen = set() + for a in reversed(base_attrs): + if a.name in seen: + continue + filtered.insert(0, a) + seen.add(a.name) + + return filtered, base_attr_map + + +def _collect_base_attrs_broken(cls, taken_attr_names): + """ + Collect attr.ibs from base classes of *cls*, except *taken_attr_names*. + + N.B. *taken_attr_names* will be mutated. + + Adhere to the old incorrect behavior. + + Notably it collects from the front and considers inherited attributes which + leads to the buggy behavior reported in #428. + """ + base_attrs = [] + base_attr_map = {} # A dictionary of base attrs to their classes. + + # Traverse the MRO and collect attributes. + for base_cls in cls.__mro__[1:-1]: + for a in getattr(base_cls, "__attrs_attrs__", []): + if a.name in taken_attr_names: + continue + + a = a.evolve(inherited=True) # noqa: PLW2901 + taken_attr_names.add(a.name) + base_attrs.append(a) + base_attr_map[a.name] = base_cls + + return base_attrs, base_attr_map + + +def _transform_attrs( + cls, + these, + auto_attribs, + kw_only, + collect_by_mro, + field_transformer, +) -> _Attributes: + """ + Transform all `_CountingAttr`s on a class into `Attribute`s. + + If *these* is passed, use that and don't look for them on the class. + + If *collect_by_mro* is True, collect them in the correct MRO order, + otherwise use the old -- incorrect -- order. See #428. + + Return an `_Attributes`. + """ + cd = cls.__dict__ + anns = _get_annotations(cls) + + if these is not None: + ca_list = list(these.items()) + elif auto_attribs is True: + ca_names = { + name + for name, attr in cd.items() + if attr.__class__ is _CountingAttr + } + ca_list = [] + annot_names = set() + for attr_name, type in anns.items(): + if _is_class_var(type): + continue + annot_names.add(attr_name) + a = cd.get(attr_name, NOTHING) + + if a.__class__ is not _CountingAttr: + a = attrib(a) + ca_list.append((attr_name, a)) + + unannotated = ca_names - annot_names + if unannotated: + raise UnannotatedAttributeError( + "The following `attr.ib`s lack a type annotation: " + + ", ".join( + sorted(unannotated, key=lambda n: cd.get(n).counter) + ) + + "." + ) + else: + ca_list = sorted( + ( + (name, attr) + for name, attr in cd.items() + if attr.__class__ is _CountingAttr + ), + key=lambda e: e[1].counter, + ) + + fca = Attribute.from_counting_attr + no = ClassProps.KeywordOnly.NO + own_attrs = [ + fca( + attr_name, + ca, + kw_only is not no, + anns.get(attr_name), + ) + for attr_name, ca in ca_list + ] + + if collect_by_mro: + base_attrs, base_attr_map = _collect_base_attrs( + cls, {a.name for a in own_attrs} + ) + else: + base_attrs, base_attr_map = _collect_base_attrs_broken( + cls, {a.name for a in own_attrs} + ) + + if kw_only is ClassProps.KeywordOnly.FORCE: + own_attrs = [a.evolve(kw_only=True) for a in own_attrs] + base_attrs = [a.evolve(kw_only=True) for a in base_attrs] + + attrs = base_attrs + own_attrs + + # Resolve default field alias before executing field_transformer, so that + # the transformer receives fully populated Attribute objects with usable + # alias values. + for a in attrs: + if not a.alias: + # Evolve is very slow, so we hold our nose and do it dirty. + _OBJ_SETATTR.__get__(a)("alias", _default_init_alias_for(a.name)) + _OBJ_SETATTR.__get__(a)("alias_is_default", True) + + if field_transformer is not None: + attrs = tuple(field_transformer(cls, attrs)) + + # Check attr order after executing the field_transformer. + # Mandatory vs non-mandatory attr order only matters when they are part of + # the __init__ signature and when they aren't kw_only (which are moved to + # the end and can be mandatory or non-mandatory in any order, as they will + # be specified as keyword args anyway). Check the order of those attrs: + had_default = False + for a in (a for a in attrs if a.init is not False and a.kw_only is False): + if had_default is True and a.default is NOTHING: + msg = f"No mandatory attributes allowed after an attribute with a default value or factory. Attribute in question: {a!r}" + raise ValueError(msg) + + if had_default is False and a.default is not NOTHING: + had_default = True + + # Resolve default field alias for any new attributes that the + # field_transformer may have added without setting an alias. + for a in attrs: + if not a.alias: + _OBJ_SETATTR.__get__(a)("alias", _default_init_alias_for(a.name)) + _OBJ_SETATTR.__get__(a)("alias_is_default", True) + + # Create AttrsClass *after* applying the field_transformer since it may + # add or remove attributes! + attr_names = [a.name for a in attrs] + AttrsClass = _make_attr_tuple_class(cls.__name__, attr_names) + + return _Attributes(AttrsClass(attrs), base_attrs, base_attr_map) + + +def _make_cached_property_getattr(cached_properties, original_getattr, cls): + lines = [ + # Wrapped to get `__class__` into closure cell for super() + # (It will be replaced with the newly constructed class after construction). + "def wrapper(_cls):", + " __class__ = _cls", + " def __getattr__(self, item, cached_properties=cached_properties, original_getattr=original_getattr, _cached_setattr_get=_cached_setattr_get):", + " func = cached_properties.get(item)", + " if func is not None:", + " result = func(self)", + " _setter = _cached_setattr_get(self)", + " _setter(item, result)", + " return result", + ] + if original_getattr is not None: + lines.append( + " return original_getattr(self, item)", + ) + else: + lines.extend( + [ + " try:", + " return super().__getattribute__(item)", + " except AttributeError:", + " if not hasattr(super(), '__getattr__'):", + " raise", + " return super().__getattr__(item)", + " original_error = f\"'{self.__class__.__name__}' object has no attribute '{item}'\"", + " raise AttributeError(original_error)", + ] + ) + + lines.extend( + [ + " return __getattr__", + "__getattr__ = wrapper(_cls)", + ] + ) + + unique_filename = _generate_unique_filename(cls, "getattr") + + glob = { + "cached_properties": cached_properties, + "_cached_setattr_get": _OBJ_SETATTR.__get__, + "original_getattr": original_getattr, + } + + return _linecache_and_compile( + "\n".join(lines), unique_filename, glob, locals={"_cls": cls} + )["__getattr__"] + + +def _frozen_setattrs(self, name, value): + """ + Attached to frozen classes as __setattr__. + """ + if isinstance(self, BaseException) and name in ( + "__cause__", + "__context__", + "__traceback__", + "__suppress_context__", + "__notes__", + ): + BaseException.__setattr__(self, name, value) + return + + raise FrozenInstanceError + + +def _frozen_delattrs(self, name): + """ + Attached to frozen classes as __delattr__. + """ + if isinstance(self, BaseException) and name == "__notes__": + BaseException.__delattr__(self, name) + return + + raise FrozenInstanceError + + +def evolve(*args, **changes): + """ + Create a new instance, based on the first positional argument with + *changes* applied. + + .. tip:: + + On Python 3.13 and later, you can also use `copy.replace` instead. + + Args: + + inst: + Instance of a class with *attrs* attributes. *inst* must be passed + as a positional argument. + + changes: + Keyword changes in the new copy. + + Returns: + A copy of inst with *changes* incorporated. + + Raises: + TypeError: + If *attr_name* couldn't be found in the class ``__init__``. + + attrs.exceptions.NotAnAttrsClassError: + If *cls* is not an *attrs* class. + + .. versionadded:: 17.1.0 + .. deprecated:: 23.1.0 + It is now deprecated to pass the instance using the keyword argument + *inst*. It will raise a warning until at least April 2024, after which + it will become an error. Always pass the instance as a positional + argument. + .. versionchanged:: 24.1.0 + *inst* can't be passed as a keyword argument anymore. + """ + try: + (inst,) = args + except ValueError: + msg = ( + f"evolve() takes 1 positional argument, but {len(args)} were given" + ) + raise TypeError(msg) from None + + cls = inst.__class__ + attrs = fields(cls) + for a in attrs: + if not a.init: + continue + attr_name = a.name # To deal with private attributes. + init_name = a.alias + if init_name not in changes: + changes[init_name] = getattr(inst, attr_name) + + return cls(**changes) + + +class _ClassBuilder: + """ + Iteratively build *one* class. + """ + + __slots__ = ( + "_add_method_dunders", + "_attr_names", + "_attrs", + "_base_attr_map", + "_base_names", + "_cache_hash", + "_cls", + "_cls_dict", + "_delete_attribs", + "_frozen", + "_has_custom_setattr", + "_has_post_init", + "_has_pre_init", + "_is_exc", + "_on_setattr", + "_pre_init_has_args", + "_repr_added", + "_script_snippets", + "_slots", + "_weakref_slot", + "_wrote_own_setattr", + ) + + def __init__( + self, + cls: type, + these, + auto_attribs: bool, + props: ClassProps, + has_custom_setattr: bool, + ): + attrs, base_attrs, base_map = _transform_attrs( + cls, + these, + auto_attribs, + props.kw_only, + props.collected_fields_by_mro, + props.field_transformer, + ) + + self._cls = cls + self._cls_dict = dict(cls.__dict__) if props.is_slotted else {} + self._attrs = attrs + self._base_names = {a.name for a in base_attrs} + self._base_attr_map = base_map + self._attr_names = tuple(a.name for a in attrs) + self._slots = props.is_slotted + self._frozen = props.is_frozen + self._weakref_slot = props.has_weakref_slot + self._cache_hash = ( + props.hashability is ClassProps.Hashability.HASHABLE_CACHED + ) + self._has_pre_init = bool(getattr(cls, "__attrs_pre_init__", False)) + self._pre_init_has_args = False + if self._has_pre_init: + # Check if the pre init method has more arguments than just `self` + # We want to pass arguments if pre init expects arguments + pre_init_func = cls.__attrs_pre_init__ + pre_init_signature = inspect.signature(pre_init_func) + self._pre_init_has_args = len(pre_init_signature.parameters) > 1 + self._has_post_init = bool(getattr(cls, "__attrs_post_init__", False)) + self._delete_attribs = not bool(these) + self._is_exc = props.is_exception + self._on_setattr = props.on_setattr_hook + + self._has_custom_setattr = has_custom_setattr + self._wrote_own_setattr = False + + self._cls_dict["__attrs_attrs__"] = self._attrs + self._cls_dict["__attrs_props__"] = props + + if props.is_frozen: + self._cls_dict["__setattr__"] = _frozen_setattrs + self._cls_dict["__delattr__"] = _frozen_delattrs + + self._wrote_own_setattr = True + elif self._on_setattr in ( + _DEFAULT_ON_SETATTR, + setters.validate, + setters.convert, + ): + has_validator = has_converter = False + for a in attrs: + if a.validator is not None: + has_validator = True + if a.converter is not None: + has_converter = True + + if has_validator and has_converter: + break + if ( + ( + self._on_setattr == _DEFAULT_ON_SETATTR + and not (has_validator or has_converter) + ) + or (self._on_setattr == setters.validate and not has_validator) + or (self._on_setattr == setters.convert and not has_converter) + ): + # If class-level on_setattr is set to convert + validate, but + # there's no field to convert or validate, pretend like there's + # no on_setattr. + self._on_setattr = None + + if props.added_pickling: + ( + self._cls_dict["__getstate__"], + self._cls_dict["__setstate__"], + ) = self._make_getstate_setstate() + + # tuples of script, globs, hook + self._script_snippets: list[ + tuple[str, dict, Callable[[dict, dict], Any]] + ] = [] + self._repr_added = False + + # We want to only do this check once; in 99.9% of cases these + # exist. + if not hasattr(self._cls, "__module__") or not hasattr( + self._cls, "__qualname__" + ): + self._add_method_dunders = self._add_method_dunders_safe + else: + self._add_method_dunders = self._add_method_dunders_unsafe + + def __repr__(self): + return f"<_ClassBuilder(cls={self._cls.__name__})>" + + def _eval_snippets(self) -> None: + """ + Evaluate any registered snippets in one go. + """ + script = "\n".join([snippet[0] for snippet in self._script_snippets]) + globs = {} + for _, snippet_globs, _ in self._script_snippets: + globs.update(snippet_globs) + + locs = _linecache_and_compile( + script, + _generate_unique_filename(self._cls, "methods"), + globs, + ) + + for _, _, hook in self._script_snippets: + hook(self._cls_dict, locs) + + def build_class(self): + """ + Finalize class based on the accumulated configuration. + + Builder cannot be used after calling this method. + """ + self._eval_snippets() + if self._slots is True: + cls = self._create_slots_class() + self._cls.__attrs_base_of_slotted__ = weakref.ref(cls) + else: + cls = self._patch_original_class() + if PY_3_10_PLUS: + cls = abc.update_abstractmethods(cls) + + # The method gets only called if it's not inherited from a base class. + # _has_own_attribute does NOT work properly for classmethods. + if ( + getattr(cls, "__attrs_init_subclass__", None) + and "__attrs_init_subclass__" not in cls.__dict__ + ): + cls.__attrs_init_subclass__() + + return cls + + def _patch_original_class(self): + """ + Apply accumulated methods and return the class. + """ + cls = self._cls + base_names = self._base_names + + # Clean class of attribute definitions (`attr.ib()`s). + if self._delete_attribs: + for name in self._attr_names: + if ( + name not in base_names + and getattr(cls, name, _SENTINEL) is not _SENTINEL + ): + # An AttributeError can happen if a base class defines a + # class variable and we want to set an attribute with the + # same name by using only a type annotation. + with contextlib.suppress(AttributeError): + delattr(cls, name) + + # Attach our dunder methods. + for name, value in self._cls_dict.items(): + setattr(cls, name, value) + + # If we've inherited an attrs __setattr__ and don't write our own, + # reset it to object's. + if not self._wrote_own_setattr and getattr( + cls, "__attrs_own_setattr__", False + ): + cls.__attrs_own_setattr__ = False + + if not self._has_custom_setattr: + cls.__setattr__ = _OBJ_SETATTR + + return cls + + def _create_slots_class(self): + """ + Build and return a new class with a `__slots__` attribute. + """ + cd = { + k: v + for k, v in self._cls_dict.items() + if k not in (*tuple(self._attr_names), "__dict__", "__weakref__") + } + + # 3.14.0rc2+ + if hasattr(sys, "_clear_type_descriptors"): + sys._clear_type_descriptors(self._cls) + + # If our class doesn't have its own implementation of __setattr__ + # (either from the user or by us), check the bases, if one of them has + # an attrs-made __setattr__, that needs to be reset. We don't walk the + # MRO because we only care about our immediate base classes. + # XXX: This can be confused by subclassing a slotted attrs class with + # XXX: a non-attrs class and subclass the resulting class with an attrs + # XXX: class. See `test_slotted_confused` for details. For now that's + # XXX: OK with us. + if not self._wrote_own_setattr: + cd["__attrs_own_setattr__"] = False + + if not self._has_custom_setattr: + for base_cls in self._cls.__bases__: + if base_cls.__dict__.get("__attrs_own_setattr__", False): + cd["__setattr__"] = _OBJ_SETATTR + break + + # Traverse the MRO to collect existing slots + # and check for an existing __weakref__. + existing_slots = {} + weakref_inherited = False + for base_cls in self._cls.__mro__[1:-1]: + if base_cls.__dict__.get("__weakref__", None) is not None: + weakref_inherited = True + existing_slots.update( + { + name: getattr(base_cls, name) + for name in getattr(base_cls, "__slots__", []) + } + ) + + base_names = set(self._base_names) + + names = self._attr_names + if ( + self._weakref_slot + and "__weakref__" not in getattr(self._cls, "__slots__", ()) + and "__weakref__" not in names + and not weakref_inherited + ): + names += ("__weakref__",) + + cached_properties = { + name: cached_prop.func + for name, cached_prop in cd.items() + if isinstance(cached_prop, cached_property) + } + + # Collect methods with a `__class__` reference that are shadowed in the new class. + # To know to update them. + additional_closure_functions_to_update = [] + if cached_properties: + class_annotations = _get_annotations(self._cls) + for name, func in cached_properties.items(): + # Add cached properties to names for slotting. + names += (name,) + # Clear out function from class to avoid clashing. + del cd[name] + additional_closure_functions_to_update.append(func) + annotation = inspect.signature(func).return_annotation + if annotation is not inspect.Parameter.empty: + class_annotations[name] = annotation + + original_getattr = cd.get("__getattr__") + if original_getattr is not None: + additional_closure_functions_to_update.append(original_getattr) + + cd["__getattr__"] = _make_cached_property_getattr( + cached_properties, original_getattr, self._cls + ) + + # We only add the names of attributes that aren't inherited. + # Setting __slots__ to inherited attributes wastes memory. + slot_names = [name for name in names if name not in base_names] + + # There are slots for attributes from current class + # that are defined in parent classes. + # As their descriptors may be overridden by a child class, + # we collect them here and update the class dict + reused_slots = { + slot: slot_descriptor + for slot, slot_descriptor in existing_slots.items() + if slot in slot_names + } + slot_names = [name for name in slot_names if name not in reused_slots] + cd.update(reused_slots) + if self._cache_hash: + slot_names.append(_HASH_CACHE_FIELD) + + cd["__slots__"] = tuple(slot_names) + + cd["__qualname__"] = self._cls.__qualname__ + + # Create new class based on old class and our methods. + cls = type(self._cls)(self._cls.__name__, self._cls.__bases__, cd) + + # The following is a fix for + # . + # If a method mentions `__class__` or uses the no-arg super(), the + # compiler will bake a reference to the class in the method itself + # as `method.__closure__`. Since we replace the class with a + # clone, we rewrite these references so it keeps working. + for item in itertools.chain( + cls.__dict__.values(), additional_closure_functions_to_update + ): + if isinstance(item, (classmethod, staticmethod)): + # Class- and staticmethods hide their functions inside. + # These might need to be rewritten as well. + closure_cells = getattr(item.__func__, "__closure__", None) + elif isinstance(item, property): + # Workaround for property `super()` shortcut (PY3-only). + # There is no universal way for other descriptors. + closure_cells = getattr(item.fget, "__closure__", None) + else: + closure_cells = getattr(item, "__closure__", None) + + if not closure_cells: # Catch None or the empty list. + continue + for cell in closure_cells: + try: + match = cell.cell_contents is self._cls + except ValueError: # noqa: PERF203 + # ValueError: Cell is empty + pass + else: + if match: + cell.cell_contents = cls + return cls + + def add_repr(self, ns): + script, globs = _make_repr_script(self._attrs, ns) + + def _attach_repr(cls_dict, globs): + cls_dict["__repr__"] = self._add_method_dunders(globs["__repr__"]) + + self._script_snippets.append((script, globs, _attach_repr)) + self._repr_added = True + return self + + def add_str(self): + if not self._repr_added: + msg = "__str__ can only be generated if a __repr__ exists." + raise ValueError(msg) + + def __str__(self): + return self.__repr__() + + self._cls_dict["__str__"] = self._add_method_dunders(__str__) + return self + + def _make_getstate_setstate(self): + """ + Create custom __setstate__ and __getstate__ methods. + """ + # __weakref__ is not writable. + state_attr_names = tuple( + an for an in self._attr_names if an != "__weakref__" + ) + + def slots_getstate(self): + """ + Automatically created by attrs. + """ + return {name: getattr(self, name) for name in state_attr_names} + + hash_caching_enabled = self._cache_hash + + def slots_setstate(self, state): + """ + Automatically created by attrs. + """ + __bound_setattr = _OBJ_SETATTR.__get__(self) + if isinstance(state, tuple): + # Backward compatibility with attrs instances pickled with + # attrs versions before v22.2.0 which stored tuples. + for name, value in zip(state_attr_names, state): + __bound_setattr(name, value) + else: + for name in state_attr_names: + if name in state: + __bound_setattr(name, state[name]) + + # The hash code cache is not included when the object is + # serialized, but it still needs to be initialized to None to + # indicate that the first call to __hash__ should be a cache + # miss. + if hash_caching_enabled: + __bound_setattr(_HASH_CACHE_FIELD, None) + + return slots_getstate, slots_setstate + + def make_unhashable(self): + self._cls_dict["__hash__"] = None + return self + + def add_hash(self): + script, globs = _make_hash_script( + self._cls, + self._attrs, + frozen=self._frozen, + cache_hash=self._cache_hash, + ) + + def attach_hash(cls_dict: dict, locs: dict) -> None: + cls_dict["__hash__"] = self._add_method_dunders(locs["__hash__"]) + + self._script_snippets.append((script, globs, attach_hash)) + + return self + + def add_init(self): + script, globs, annotations = _make_init_script( + self._cls, + self._attrs, + self._has_pre_init, + self._pre_init_has_args, + self._has_post_init, + self._frozen, + self._slots, + self._cache_hash, + self._base_attr_map, + self._is_exc, + self._on_setattr, + attrs_init=False, + ) + + def _attach_init(cls_dict, globs): + init = globs["__init__"] + init.__annotations__ = annotations + cls_dict["__init__"] = self._add_method_dunders(init) + + self._script_snippets.append((script, globs, _attach_init)) + + return self + + def add_replace(self): + self._cls_dict["__replace__"] = self._add_method_dunders(evolve) + return self + + def add_match_args(self): + self._cls_dict["__match_args__"] = tuple( + field.name + for field in self._attrs + if field.init and not field.kw_only + ) + + def add_attrs_init(self): + script, globs, annotations = _make_init_script( + self._cls, + self._attrs, + self._has_pre_init, + self._pre_init_has_args, + self._has_post_init, + self._frozen, + self._slots, + self._cache_hash, + self._base_attr_map, + self._is_exc, + self._on_setattr, + attrs_init=True, + ) + + def _attach_attrs_init(cls_dict, globs): + init = globs["__attrs_init__"] + init.__annotations__ = annotations + cls_dict["__attrs_init__"] = self._add_method_dunders(init) + + self._script_snippets.append((script, globs, _attach_attrs_init)) + + return self + + def add_eq(self): + cd = self._cls_dict + + script, globs = _make_eq_script(self._attrs) + + def _attach_eq(cls_dict, globs): + cls_dict["__eq__"] = self._add_method_dunders(globs["__eq__"]) + + self._script_snippets.append((script, globs, _attach_eq)) + + cd["__ne__"] = __ne__ + + return self + + def add_order(self): + cd = self._cls_dict + + cd["__lt__"], cd["__le__"], cd["__gt__"], cd["__ge__"] = ( + self._add_method_dunders(meth) + for meth in _make_order(self._cls, self._attrs) + ) + + return self + + def add_setattr(self): + sa_attrs = {} + for a in self._attrs: + on_setattr = a.on_setattr or self._on_setattr + if on_setattr and on_setattr is not setters.NO_OP: + sa_attrs[a.name] = a, on_setattr + + if not sa_attrs: + return self + + if self._has_custom_setattr: + # We need to write a __setattr__ but there already is one! + msg = "Can't combine custom __setattr__ with on_setattr hooks." + raise ValueError(msg) + + # docstring comes from _add_method_dunders + def __setattr__(self, name, val): + try: + a, hook = sa_attrs[name] + except KeyError: + nval = val + else: + nval = hook(self, a, val) + + _OBJ_SETATTR(self, name, nval) + + self._cls_dict["__attrs_own_setattr__"] = True + self._cls_dict["__setattr__"] = self._add_method_dunders(__setattr__) + self._wrote_own_setattr = True + + return self + + def _add_method_dunders_unsafe(self, method: Callable) -> Callable: + """ + Add __module__ and __qualname__ to a *method*. + """ + method.__module__ = self._cls.__module__ + + method.__qualname__ = f"{self._cls.__qualname__}.{method.__name__}" + + method.__doc__ = ( + f"Method generated by attrs for class {self._cls.__qualname__}." + ) + + return method + + def _add_method_dunders_safe(self, method: Callable) -> Callable: + """ + Add __module__ and __qualname__ to a *method* if possible. + """ + with contextlib.suppress(AttributeError): + method.__module__ = self._cls.__module__ + + with contextlib.suppress(AttributeError): + method.__qualname__ = f"{self._cls.__qualname__}.{method.__name__}" + + with contextlib.suppress(AttributeError): + method.__doc__ = f"Method generated by attrs for class {self._cls.__qualname__}." + + return method + + +def _determine_attrs_eq_order(cmp, eq, order, default_eq): + """ + Validate the combination of *cmp*, *eq*, and *order*. Derive the effective + values of eq and order. If *eq* is None, set it to *default_eq*. + """ + if cmp is not None and any((eq is not None, order is not None)): + msg = "Don't mix `cmp` with `eq' and `order`." + raise ValueError(msg) + + # cmp takes precedence due to bw-compatibility. + if cmp is not None: + return cmp, cmp + + # If left None, equality is set to the specified default and ordering + # mirrors equality. + if eq is None: + eq = default_eq + + if order is None: + order = eq + + if eq is False and order is True: + msg = "`order` can only be True if `eq` is True too." + raise ValueError(msg) + + return eq, order + + +def _determine_attrib_eq_order(cmp, eq, order, default_eq): + """ + Validate the combination of *cmp*, *eq*, and *order*. Derive the effective + values of eq and order. If *eq* is None, set it to *default_eq*. + """ + if cmp is not None and any((eq is not None, order is not None)): + msg = "Don't mix `cmp` with `eq' and `order`." + raise ValueError(msg) + + def decide_callable_or_boolean(value): + """ + Decide whether a key function is used. + """ + if callable(value): + value, key = True, value + else: + key = None + return value, key + + # cmp takes precedence due to bw-compatibility. + if cmp is not None: + cmp, cmp_key = decide_callable_or_boolean(cmp) + return cmp, cmp_key, cmp, cmp_key + + # If left None, equality is set to the specified default and ordering + # mirrors equality. + if eq is None: + eq, eq_key = default_eq, None + else: + eq, eq_key = decide_callable_or_boolean(eq) + + if order is None: + order, order_key = eq, eq_key + else: + order, order_key = decide_callable_or_boolean(order) + + if eq is False and order is True: + msg = "`order` can only be True if `eq` is True too." + raise ValueError(msg) + + return eq, eq_key, order, order_key + + +def _determine_whether_to_implement( + cls, flag, auto_detect, dunders, default=True +): + """ + Check whether we should implement a set of methods for *cls*. + + *flag* is the argument passed into @attr.s like 'init', *auto_detect* the + same as passed into @attr.s and *dunders* is a tuple of attribute names + whose presence signal that the user has implemented it themselves. + + Return *default* if no reason for either for or against is found. + """ + if flag is True or flag is False: + return flag + + if flag is None and auto_detect is False: + return default + + # Logically, flag is None and auto_detect is True here. + for dunder in dunders: + if _has_own_attribute(cls, dunder): + return False + + return default + + +def attrs( + maybe_cls=None, + these=None, + repr_ns=None, + repr=None, + cmp=None, + hash=None, + init=None, + slots=False, + frozen=False, + weakref_slot=True, + str=False, + auto_attribs=False, + kw_only=False, + cache_hash=False, + auto_exc=False, + eq=None, + order=None, + auto_detect=False, + collect_by_mro=False, + getstate_setstate=None, + on_setattr=None, + field_transformer=None, + match_args=True, + unsafe_hash=None, + force_kw_only=True, +): + r""" + A class decorator that adds :term:`dunder methods` according to the + specified attributes using `attr.ib` or the *these* argument. + + Consider using `attrs.define` / `attrs.frozen` in new code (``attr.s`` will + *never* go away, though). + + Args: + repr_ns (str): + When using nested classes, there was no way in Python 2 to + automatically detect that. This argument allows to set a custom + name for a more meaningful ``repr`` output. This argument is + pointless in Python 3 and is therefore deprecated. + + .. caution:: + Refer to `attrs.define` for the rest of the parameters, but note that they + can have different defaults. + + Notably, leaving *on_setattr* as `None` will **not** add any hooks. + + .. versionadded:: 16.0.0 *slots* + .. versionadded:: 16.1.0 *frozen* + .. versionadded:: 16.3.0 *str* + .. versionadded:: 16.3.0 Support for ``__attrs_post_init__``. + .. versionchanged:: 17.1.0 + *hash* supports `None` as value which is also the default now. + .. versionadded:: 17.3.0 *auto_attribs* + .. versionchanged:: 18.1.0 + If *these* is passed, no attributes are deleted from the class body. + .. versionchanged:: 18.1.0 If *these* is ordered, the order is retained. + .. versionadded:: 18.2.0 *weakref_slot* + .. deprecated:: 18.2.0 + ``__lt__``, ``__le__``, ``__gt__``, and ``__ge__`` now raise a + `DeprecationWarning` if the classes compared are subclasses of + each other. ``__eq`` and ``__ne__`` never tried to compared subclasses + to each other. + .. versionchanged:: 19.2.0 + ``__lt__``, ``__le__``, ``__gt__``, and ``__ge__`` now do not consider + subclasses comparable anymore. + .. versionadded:: 18.2.0 *kw_only* + .. versionadded:: 18.2.0 *cache_hash* + .. versionadded:: 19.1.0 *auto_exc* + .. deprecated:: 19.2.0 *cmp* Removal on or after 2021-06-01. + .. versionadded:: 19.2.0 *eq* and *order* + .. versionadded:: 20.1.0 *auto_detect* + .. versionadded:: 20.1.0 *collect_by_mro* + .. versionadded:: 20.1.0 *getstate_setstate* + .. versionadded:: 20.1.0 *on_setattr* + .. versionadded:: 20.3.0 *field_transformer* + .. versionchanged:: 21.1.0 + ``init=False`` injects ``__attrs_init__`` + .. versionchanged:: 21.1.0 Support for ``__attrs_pre_init__`` + .. versionchanged:: 21.1.0 *cmp* undeprecated + .. versionadded:: 21.3.0 *match_args* + .. versionadded:: 22.2.0 + *unsafe_hash* as an alias for *hash* (for :pep:`681` compliance). + .. deprecated:: 24.1.0 *repr_ns* + .. versionchanged:: 24.1.0 + Instances are not compared as tuples of attributes anymore, but using a + big ``and`` condition. This is faster and has more correct behavior for + uncomparable values like `math.nan`. + .. versionadded:: 24.1.0 + If a class has an *inherited* classmethod called + ``__attrs_init_subclass__``, it is executed after the class is created. + .. deprecated:: 24.1.0 *hash* is deprecated in favor of *unsafe_hash*. + .. versionchanged:: 25.4.0 + *kw_only* now only applies to attributes defined in the current class, + and respects attribute-level ``kw_only=False`` settings. + .. versionadded:: 25.4.0 *force_kw_only* + """ + if repr_ns is not None: + import warnings + + warnings.warn( + DeprecationWarning( + "The `repr_ns` argument is deprecated and will be removed in or after August 2025." + ), + stacklevel=2, + ) + + eq_, order_ = _determine_attrs_eq_order(cmp, eq, order, None) + + # unsafe_hash takes precedence due to PEP 681. + if unsafe_hash is not None: + hash = unsafe_hash + + if isinstance(on_setattr, (list, tuple)): + on_setattr = setters.pipe(*on_setattr) + + def wrap(cls): + nonlocal hash + is_frozen = frozen or _has_frozen_base_class(cls) + is_exc = auto_exc is True and issubclass(cls, BaseException) + has_own_setattr = auto_detect and _has_own_attribute( + cls, "__setattr__" + ) + + if has_own_setattr and is_frozen: + msg = "Can't freeze a class with a custom __setattr__." + raise ValueError(msg) + + eq = not is_exc and _determine_whether_to_implement( + cls, eq_, auto_detect, ("__eq__", "__ne__") + ) + + Hashability = ClassProps.Hashability + + if is_exc: + hashability = Hashability.LEAVE_ALONE + elif hash is True: + hashability = ( + Hashability.HASHABLE_CACHED + if cache_hash + else Hashability.HASHABLE + ) + elif hash is False: + hashability = Hashability.LEAVE_ALONE + elif hash is None: + if auto_detect is True and _has_own_attribute(cls, "__hash__"): + hashability = Hashability.LEAVE_ALONE + elif eq is True and is_frozen is True: + hashability = ( + Hashability.HASHABLE_CACHED + if cache_hash + else Hashability.HASHABLE + ) + elif eq is False: + hashability = Hashability.LEAVE_ALONE + else: + hashability = Hashability.UNHASHABLE + else: + msg = "Invalid value for hash. Must be True, False, or None." + raise TypeError(msg) + + KeywordOnly = ClassProps.KeywordOnly + if kw_only: + kwo = KeywordOnly.FORCE if force_kw_only else KeywordOnly.YES + else: + kwo = KeywordOnly.NO + + props = ClassProps( + is_exception=is_exc, + is_frozen=is_frozen, + is_slotted=slots, + collected_fields_by_mro=collect_by_mro, + added_init=_determine_whether_to_implement( + cls, init, auto_detect, ("__init__",) + ), + added_repr=_determine_whether_to_implement( + cls, repr, auto_detect, ("__repr__",) + ), + added_eq=eq, + added_ordering=not is_exc + and _determine_whether_to_implement( + cls, + order_, + auto_detect, + ("__lt__", "__le__", "__gt__", "__ge__"), + ), + hashability=hashability, + added_match_args=match_args, + kw_only=kwo, + has_weakref_slot=weakref_slot, + added_str=str, + added_pickling=_determine_whether_to_implement( + cls, + getstate_setstate, + auto_detect, + ("__getstate__", "__setstate__"), + default=slots, + ), + on_setattr_hook=on_setattr, + field_transformer=field_transformer, + ) + + if not props.is_hashable and cache_hash: + msg = "Invalid value for cache_hash. To use hash caching, hashing must be either explicitly or implicitly enabled." + raise TypeError(msg) + + builder = _ClassBuilder( + cls, + these, + auto_attribs=auto_attribs, + props=props, + has_custom_setattr=has_own_setattr, + ) + + if props.added_repr: + builder.add_repr(repr_ns) + + if props.added_str: + builder.add_str() + + if props.added_eq: + builder.add_eq() + if props.added_ordering: + builder.add_order() + + if not frozen: + builder.add_setattr() + + if props.is_hashable: + builder.add_hash() + elif props.hashability is Hashability.UNHASHABLE: + builder.make_unhashable() + + if props.added_init: + builder.add_init() + else: + builder.add_attrs_init() + if cache_hash: + msg = "Invalid value for cache_hash. To use hash caching, init must be True." + raise TypeError(msg) + + if PY_3_13_PLUS and not _has_own_attribute(cls, "__replace__"): + builder.add_replace() + + if ( + PY_3_10_PLUS + and match_args + and not _has_own_attribute(cls, "__match_args__") + ): + builder.add_match_args() + + return builder.build_class() + + # maybe_cls's type depends on the usage of the decorator. It's a class + # if it's used as `@attrs` but `None` if used as `@attrs()`. + if maybe_cls is None: + return wrap + + return wrap(maybe_cls) + + +_attrs = attrs +""" +Internal alias so we can use it in functions that take an argument called +*attrs*. +""" + + +def _has_frozen_base_class(cls): + """ + Check whether *cls* has a frozen ancestor by looking at its + __setattr__. + """ + return cls.__setattr__ is _frozen_setattrs + + +def _generate_unique_filename(cls: type, func_name: str) -> str: + """ + Create a "filename" suitable for a function being generated. + """ + return ( + f"" + ) + + +def _make_hash_script( + cls: type, attrs: list[Attribute], frozen: bool, cache_hash: bool +) -> tuple[str, dict]: + attrs = tuple( + a for a in attrs if a.hash is True or (a.hash is None and a.eq is True) + ) + + tab = " " + + type_hash = hash(_generate_unique_filename(cls, "hash")) + # If eq is custom generated, we need to include the functions in globs + globs = {} + + hash_def = "def __hash__(self" + hash_func = "hash((" + closing_braces = "))" + if not cache_hash: + hash_def += "):" + else: + hash_def += ", *" + + hash_def += ", _cache_wrapper=__import__('attr._make')._make._CacheHashWrapper):" + hash_func = "_cache_wrapper(" + hash_func + closing_braces += ")" + + method_lines = [hash_def] + + def append_hash_computation_lines(prefix, indent): + """ + Generate the code for actually computing the hash code. + Below this will either be returned directly or used to compute + a value which is then cached, depending on the value of cache_hash + """ + + method_lines.extend( + [ + indent + prefix + hash_func, + indent + f" {type_hash},", + ] + ) + + for a in attrs: + if a.eq_key: + cmp_name = f"_{a.name}_key" + globs[cmp_name] = a.eq_key + method_lines.append( + indent + f" {cmp_name}(self.{a.name})," + ) + else: + method_lines.append(indent + f" self.{a.name},") + + method_lines.append(indent + " " + closing_braces) + + if cache_hash: + method_lines.append(tab + f"if self.{_HASH_CACHE_FIELD} is None:") + if frozen: + append_hash_computation_lines( + f"object.__setattr__(self, '{_HASH_CACHE_FIELD}', ", tab * 2 + ) + method_lines.append(tab * 2 + ")") # close __setattr__ + else: + append_hash_computation_lines( + f"self.{_HASH_CACHE_FIELD} = ", tab * 2 + ) + method_lines.append(tab + f"return self.{_HASH_CACHE_FIELD}") + else: + append_hash_computation_lines("return ", tab) + + script = "\n".join(method_lines) + return script, globs + + +def _add_hash(cls: type, attrs: list[Attribute]): + """ + Add a hash method to *cls*. + """ + script, globs = _make_hash_script( + cls, attrs, frozen=False, cache_hash=False + ) + _compile_and_eval( + script, globs, filename=_generate_unique_filename(cls, "__hash__") + ) + cls.__hash__ = globs["__hash__"] + return cls + + +def __ne__(self, other): + """ + Check equality and either forward a NotImplemented or + return the result negated. + """ + result = self.__eq__(other) + if result is NotImplemented: + return NotImplemented + + return not result + + +def _make_eq_script(attrs: list) -> tuple[str, dict]: + """ + Create __eq__ method for *cls* with *attrs*. + """ + attrs = [a for a in attrs if a.eq] + + lines = [ + "def __eq__(self, other):", + " if other.__class__ is not self.__class__:", + " return NotImplemented", + ] + + globs = {} + if attrs: + lines.append(" return (") + for a in attrs: + if a.eq_key: + cmp_name = f"_{a.name}_key" + # Add the key function to the global namespace + # of the evaluated function. + globs[cmp_name] = a.eq_key + lines.append( + f" {cmp_name}(self.{a.name}) == {cmp_name}(other.{a.name})" + ) + else: + lines.append(f" self.{a.name} == other.{a.name}") + if a is not attrs[-1]: + lines[-1] = f"{lines[-1]} and" + lines.append(" )") + else: + lines.append(" return True") + + script = "\n".join(lines) + + return script, globs + + +def _make_order(cls, attrs): + """ + Create ordering methods for *cls* with *attrs*. + """ + attrs = [a for a in attrs if a.order] + + def attrs_to_tuple(obj): + """ + Save us some typing. + """ + return tuple( + key(value) if key else value + for value, key in ( + (getattr(obj, a.name), a.order_key) for a in attrs + ) + ) + + def __lt__(self, other): + """ + Automatically created by attrs. + """ + if other.__class__ is self.__class__: + return attrs_to_tuple(self) < attrs_to_tuple(other) + + return NotImplemented + + def __le__(self, other): + """ + Automatically created by attrs. + """ + if other.__class__ is self.__class__: + return attrs_to_tuple(self) <= attrs_to_tuple(other) + + return NotImplemented + + def __gt__(self, other): + """ + Automatically created by attrs. + """ + if other.__class__ is self.__class__: + return attrs_to_tuple(self) > attrs_to_tuple(other) + + return NotImplemented + + def __ge__(self, other): + """ + Automatically created by attrs. + """ + if other.__class__ is self.__class__: + return attrs_to_tuple(self) >= attrs_to_tuple(other) + + return NotImplemented + + return __lt__, __le__, __gt__, __ge__ + + +def _add_eq(cls, attrs=None): + """ + Add equality methods to *cls* with *attrs*. + """ + if attrs is None: + attrs = cls.__attrs_attrs__ + + script, globs = _make_eq_script(attrs) + _compile_and_eval( + script, globs, filename=_generate_unique_filename(cls, "__eq__") + ) + cls.__eq__ = globs["__eq__"] + cls.__ne__ = __ne__ + + return cls + + +def _make_repr_script(attrs, ns) -> tuple[str, dict]: + """ + Create the source and globs for a __repr__ and return it. + """ + # Figure out which attributes to include, and which function to use to + # format them. The a.repr value can be either bool or a custom + # callable. + attr_names_with_reprs = tuple( + (a.name, (repr if a.repr is True else a.repr), a.init) + for a in attrs + if a.repr is not False + ) + globs = { + name + "_repr": r for name, r, _ in attr_names_with_reprs if r != repr + } + globs["_compat"] = _compat + globs["AttributeError"] = AttributeError + globs["NOTHING"] = NOTHING + attribute_fragments = [] + for name, r, i in attr_names_with_reprs: + accessor = ( + "self." + name if i else 'getattr(self, "' + name + '", NOTHING)' + ) + fragment = ( + "%s={%s!r}" % (name, accessor) + if r == repr + else "%s={%s_repr(%s)}" % (name, name, accessor) + ) + attribute_fragments.append(fragment) + repr_fragment = ", ".join(attribute_fragments) + + if ns is None: + cls_name_fragment = '{self.__class__.__qualname__.rsplit(">.", 1)[-1]}' + else: + cls_name_fragment = ns + ".{self.__class__.__name__}" + + lines = [ + "def __repr__(self):", + " try:", + " already_repring = _compat.repr_context.already_repring", + " except AttributeError:", + " already_repring = {id(self),}", + " _compat.repr_context.already_repring = already_repring", + " else:", + " if id(self) in already_repring:", + " return '...'", + " else:", + " already_repring.add(id(self))", + " try:", + f" return f'{cls_name_fragment}({repr_fragment})'", + " finally:", + " already_repring.remove(id(self))", + ] + + return "\n".join(lines), globs + + +def _add_repr(cls, ns=None, attrs=None): + """ + Add a repr method to *cls*. + """ + if attrs is None: + attrs = cls.__attrs_attrs__ + + script, globs = _make_repr_script(attrs, ns) + _compile_and_eval( + script, globs, filename=_generate_unique_filename(cls, "__repr__") + ) + cls.__repr__ = globs["__repr__"] + return cls + + +def fields(cls): + """ + Return the tuple of *attrs* attributes for a class or instance. + + The tuple also allows accessing the fields by their names (see below for + examples). + + Args: + cls (type): Class or instance to introspect. + + Raises: + TypeError: If *cls* is neither a class nor an *attrs* instance. + + attrs.exceptions.NotAnAttrsClassError: + If *cls* is not an *attrs* class. + + Returns: + tuple (with name accessors) of `attrs.Attribute` + + .. versionchanged:: 16.2.0 Returned tuple allows accessing the fields + by name. + .. versionchanged:: 23.1.0 Add support for generic classes. + .. versionchanged:: 26.1.0 Add support for instances. + """ + generic_base = get_generic_base(cls) + + if generic_base is None and not isinstance(cls, type): + type_ = type(cls) + if getattr(type_, "__attrs_attrs__", None) is None: + msg = "Passed object must be a class or attrs instance." + raise TypeError(msg) + + return fields(type_) + + attrs = getattr(cls, "__attrs_attrs__", None) + + if attrs is None: + if generic_base is not None: + attrs = getattr(generic_base, "__attrs_attrs__", None) + if attrs is not None: + # Even though this is global state, stick it on here to speed + # it up. We rely on `cls` being cached for this to be + # efficient. + cls.__attrs_attrs__ = attrs + return attrs + msg = f"{cls!r} is not an attrs-decorated class." + raise NotAnAttrsClassError(msg) + + return attrs + + +def fields_dict(cls): + """ + Return an ordered dictionary of *attrs* attributes for a class, whose keys + are the attribute names. + + Args: + cls (type): Class to introspect. + + Raises: + TypeError: If *cls* is not a class. + + attrs.exceptions.NotAnAttrsClassError: + If *cls* is not an *attrs* class. + + Returns: + dict[str, attrs.Attribute]: Dict of attribute name to definition + + .. versionadded:: 18.1.0 + """ + if not isinstance(cls, type): + msg = "Passed object must be a class." + raise TypeError(msg) + attrs = getattr(cls, "__attrs_attrs__", None) + if attrs is None: + msg = f"{cls!r} is not an attrs-decorated class." + raise NotAnAttrsClassError(msg) + return {a.name: a for a in attrs} + + +def validate(inst): + """ + Validate all attributes on *inst* that have a validator. + + Leaves all exceptions through. + + Args: + inst: Instance of a class with *attrs* attributes. + """ + if _config._run_validators is False: + return + + for a in fields(inst.__class__): + v = a.validator + if v is not None: + v(inst, a, getattr(inst, a.name)) + + +def _is_slot_attr(a_name, base_attr_map): + """ + Check if the attribute name comes from a slot class. + """ + cls = base_attr_map.get(a_name) + return cls and "__slots__" in cls.__dict__ + + +def _make_init_script( + cls, + attrs, + pre_init, + pre_init_has_args, + post_init, + frozen, + slots, + cache_hash, + base_attr_map, + is_exc, + cls_on_setattr, + attrs_init, +) -> tuple[str, dict, dict]: + has_cls_on_setattr = ( + cls_on_setattr is not None and cls_on_setattr is not setters.NO_OP + ) + + if frozen and has_cls_on_setattr: + msg = "Frozen classes can't use on_setattr." + raise ValueError(msg) + + needs_cached_setattr = cache_hash or frozen + filtered_attrs = [] + attr_dict = {} + for a in attrs: + if not a.init and a.default is NOTHING: + continue + + filtered_attrs.append(a) + attr_dict[a.name] = a + + if a.on_setattr is not None: + if frozen is True and a.on_setattr is not setters.NO_OP: + msg = "Frozen classes can't use on_setattr." + raise ValueError(msg) + + needs_cached_setattr = True + elif has_cls_on_setattr and a.on_setattr is not setters.NO_OP: + needs_cached_setattr = True + + script, globs, annotations = _attrs_to_init_script( + filtered_attrs, + frozen, + slots, + pre_init, + pre_init_has_args, + post_init, + cache_hash, + base_attr_map, + is_exc, + needs_cached_setattr, + has_cls_on_setattr, + "__attrs_init__" if attrs_init else "__init__", + ) + if cls.__module__ in sys.modules: + # This makes typing.get_type_hints(CLS.__init__) resolve string types. + globs.update(sys.modules[cls.__module__].__dict__) + + globs.update({"NOTHING": NOTHING, "attr_dict": attr_dict}) + + if needs_cached_setattr: + # Save the lookup overhead in __init__ if we need to circumvent + # setattr hooks. + globs["_cached_setattr_get"] = _OBJ_SETATTR.__get__ + + return script, globs, annotations + + +def _setattr(attr_name: str, value_var: str, has_on_setattr: bool) -> str: + """ + Use the cached object.setattr to set *attr_name* to *value_var*. + """ + return f"_setattr('{attr_name}', {value_var})" + + +def _setattr_with_converter( + attr_name: str, value_var: str, has_on_setattr: bool, converter: Converter +) -> str: + """ + Use the cached object.setattr to set *attr_name* to *value_var*, but run + its converter first. + """ + return f"_setattr('{attr_name}', {converter._fmt_converter_call(attr_name, value_var)})" + + +def _assign(attr_name: str, value: str, has_on_setattr: bool) -> str: + """ + Unless *attr_name* has an on_setattr hook, use normal assignment. Otherwise + relegate to _setattr. + """ + if has_on_setattr: + return _setattr(attr_name, value, True) + + return f"self.{attr_name} = {value}" + + +def _assign_with_converter( + attr_name: str, value_var: str, has_on_setattr: bool, converter: Converter +) -> str: + """ + Unless *attr_name* has an on_setattr hook, use normal assignment after + conversion. Otherwise relegate to _setattr_with_converter. + """ + if has_on_setattr: + return _setattr_with_converter(attr_name, value_var, True, converter) + + return f"self.{attr_name} = {converter._fmt_converter_call(attr_name, value_var)}" + + +def _determine_setters( + frozen: bool, slots: bool, base_attr_map: dict[str, type] +): + """ + Determine the correct setter functions based on whether a class is frozen + and/or slotted. + """ + if frozen is True: + if slots is True: + return (), _setattr, _setattr_with_converter + + # Dict frozen classes assign directly to __dict__. + # But only if the attribute doesn't come from an ancestor slot + # class. + # Note _inst_dict will be used again below if cache_hash is True + + def fmt_setter( + attr_name: str, value_var: str, has_on_setattr: bool + ) -> str: + if _is_slot_attr(attr_name, base_attr_map): + return _setattr(attr_name, value_var, has_on_setattr) + + return f"_inst_dict['{attr_name}'] = {value_var}" + + def fmt_setter_with_converter( + attr_name: str, + value_var: str, + has_on_setattr: bool, + converter: Converter, + ) -> str: + if has_on_setattr or _is_slot_attr(attr_name, base_attr_map): + return _setattr_with_converter( + attr_name, value_var, has_on_setattr, converter + ) + + return f"_inst_dict['{attr_name}'] = {converter._fmt_converter_call(attr_name, value_var)}" + + return ( + ("_inst_dict = self.__dict__",), + fmt_setter, + fmt_setter_with_converter, + ) + + # Not frozen -- we can just assign directly. + return (), _assign, _assign_with_converter + + +def _attrs_to_init_script( + attrs: list[Attribute], + is_frozen: bool, + is_slotted: bool, + call_pre_init: bool, + pre_init_has_args: bool, + call_post_init: bool, + does_cache_hash: bool, + base_attr_map: dict[str, type], + is_exc: bool, + needs_cached_setattr: bool, + has_cls_on_setattr: bool, + method_name: str, +) -> tuple[str, dict, dict]: + """ + Return a script of an initializer for *attrs*, a dict of globals, and + annotations for the initializer. + + The globals are required by the generated script. + """ + lines = ["self.__attrs_pre_init__()"] if call_pre_init else [] + + if needs_cached_setattr: + lines.append( + # Circumvent the __setattr__ descriptor to save one lookup per + # assignment. Note _setattr will be used again below if + # does_cache_hash is True. + "_setattr = _cached_setattr_get(self)" + ) + + extra_lines, fmt_setter, fmt_setter_with_converter = _determine_setters( + is_frozen, is_slotted, base_attr_map + ) + lines.extend(extra_lines) + + args = [] # Parameters in the definition of __init__ + pre_init_args = [] # Parameters in the call to __attrs_pre_init__ + kw_only_args = [] # Used for both 'args' and 'pre_init_args' above + attrs_to_validate = [] + + # This is a dictionary of names to validator and converter callables. + # Injecting this into __init__ globals lets us avoid lookups. + names_for_globals = {} + annotations = {"return": None} + + for a in attrs: + if a.validator: + attrs_to_validate.append(a) + + attr_name = a.name + has_on_setattr = a.on_setattr is not None or ( + a.on_setattr is not setters.NO_OP and has_cls_on_setattr + ) + # a.alias is set to maybe-mangled attr_name in _ClassBuilder if not + # explicitly provided + arg_name = a.alias + + has_factory = isinstance(a.default, Factory) + maybe_self = "self" if has_factory and a.default.takes_self else "" + + if a.converter is not None and not isinstance(a.converter, Converter): + converter = Converter(a.converter) + else: + converter = a.converter + + if a.init is False: + if has_factory: + init_factory_name = _INIT_FACTORY_PAT % (a.name,) + if converter is not None: + lines.append( + fmt_setter_with_converter( + attr_name, + init_factory_name + f"({maybe_self})", + has_on_setattr, + converter, + ) + ) + names_for_globals[converter._get_global_name(a.name)] = ( + converter.converter + ) + else: + lines.append( + fmt_setter( + attr_name, + init_factory_name + f"({maybe_self})", + has_on_setattr, + ) + ) + names_for_globals[init_factory_name] = a.default.factory + elif converter is not None: + lines.append( + fmt_setter_with_converter( + attr_name, + f"attr_dict['{attr_name}'].default", + has_on_setattr, + converter, + ) + ) + names_for_globals[converter._get_global_name(a.name)] = ( + converter.converter + ) + else: + lines.append( + fmt_setter( + attr_name, + f"attr_dict['{attr_name}'].default", + has_on_setattr, + ) + ) + elif a.default is not NOTHING and not has_factory: + arg = f"{arg_name}=attr_dict['{attr_name}'].default" + if a.kw_only: + kw_only_args.append(arg) + else: + args.append(arg) + pre_init_args.append(arg_name) + + if converter is not None: + lines.append( + fmt_setter_with_converter( + attr_name, arg_name, has_on_setattr, converter + ) + ) + names_for_globals[converter._get_global_name(a.name)] = ( + converter.converter + ) + else: + lines.append(fmt_setter(attr_name, arg_name, has_on_setattr)) + + elif has_factory: + arg = f"{arg_name}=NOTHING" + if a.kw_only: + kw_only_args.append(arg) + else: + args.append(arg) + pre_init_args.append(arg_name) + lines.append(f"if {arg_name} is not NOTHING:") + + init_factory_name = _INIT_FACTORY_PAT % (a.name,) + if converter is not None: + lines.append( + " " + + fmt_setter_with_converter( + attr_name, arg_name, has_on_setattr, converter + ) + ) + lines.append("else:") + lines.append( + " " + + fmt_setter_with_converter( + attr_name, + init_factory_name + "(" + maybe_self + ")", + has_on_setattr, + converter, + ) + ) + names_for_globals[converter._get_global_name(a.name)] = ( + converter.converter + ) + else: + lines.append( + " " + fmt_setter(attr_name, arg_name, has_on_setattr) + ) + lines.append("else:") + lines.append( + " " + + fmt_setter( + attr_name, + init_factory_name + "(" + maybe_self + ")", + has_on_setattr, + ) + ) + names_for_globals[init_factory_name] = a.default.factory + else: + if a.kw_only: + kw_only_args.append(arg_name) + else: + args.append(arg_name) + pre_init_args.append(arg_name) + + if converter is not None: + lines.append( + fmt_setter_with_converter( + attr_name, arg_name, has_on_setattr, converter + ) + ) + names_for_globals[converter._get_global_name(a.name)] = ( + converter.converter + ) + else: + lines.append(fmt_setter(attr_name, arg_name, has_on_setattr)) + + if a.init is True: + if a.type is not None and converter is None: + annotations[arg_name] = a.type + elif converter is not None and converter._first_param_type: + # Use the type from the converter if present. + annotations[arg_name] = converter._first_param_type + + if attrs_to_validate: # we can skip this if there are no validators. + names_for_globals["_config"] = _config + lines.append("if _config._run_validators is True:") + for a in attrs_to_validate: + val_name = "__attr_validator_" + a.name + attr_name = "__attr_" + a.name + lines.append(f" {val_name}(self, {attr_name}, self.{a.name})") + names_for_globals[val_name] = a.validator + names_for_globals[attr_name] = a + + if call_post_init: + lines.append("self.__attrs_post_init__()") + + # Because this is set only after __attrs_post_init__ is called, a crash + # will result if post-init tries to access the hash code. This seemed + # preferable to setting this beforehand, in which case alteration to field + # values during post-init combined with post-init accessing the hash code + # would result in silent bugs. + if does_cache_hash: + if is_frozen: + if is_slotted: + init_hash_cache = f"_setattr('{_HASH_CACHE_FIELD}', None)" + else: + init_hash_cache = f"_inst_dict['{_HASH_CACHE_FIELD}'] = None" + else: + init_hash_cache = f"self.{_HASH_CACHE_FIELD} = None" + lines.append(init_hash_cache) + + # For exceptions we rely on BaseException.__init__ for proper + # initialization. + if is_exc: + vals = ",".join(f"self.{a.name}" for a in attrs if a.init) + + lines.append(f"BaseException.__init__(self, {vals})") + + args = ", ".join(args) + pre_init_args = ", ".join(pre_init_args) + if kw_only_args: + # leading comma & kw_only args + args += f"{', ' if args else ''}*, {', '.join(kw_only_args)}" + pre_init_kw_only_args = ", ".join( + [ + f"{kw_arg_name}={kw_arg_name}" + # We need to remove the defaults from the kw_only_args. + for kw_arg_name in (kwa.split("=")[0] for kwa in kw_only_args) + ] + ) + pre_init_args += ", " if pre_init_args else "" + pre_init_args += pre_init_kw_only_args + + if call_pre_init and pre_init_has_args: + # If pre init method has arguments, pass the values given to __init__. + lines[0] = f"self.__attrs_pre_init__({pre_init_args})" + + # Python <3.12 doesn't allow backslashes in f-strings. + NL = "\n " + return ( + f"""def {method_name}(self, {args}): + {NL.join(lines) if lines else "pass"} +""", + names_for_globals, + annotations, + ) + + +def _default_init_alias_for(name: str) -> str: + """ + The default __init__ parameter name for a field. + + This performs private-name adjustment via leading-unscore stripping, + and is the default value of Attribute.alias if not provided. + """ + + return name.lstrip("_") + + +class Attribute: + """ + *Read-only* representation of an attribute. + + .. warning:: + + You should never instantiate this class yourself. + + The class has *all* arguments of `attr.ib` (except for ``factory`` which is + only syntactic sugar for ``default=Factory(...)`` plus the following: + + - ``name`` (`str`): The name of the attribute. + - ``alias`` (`str`): The __init__ parameter name of the attribute, after + any explicit overrides and default private-attribute-name handling. + - ``alias_is_default`` (`bool`): Whether the ``alias`` was automatically + generated (``True``) or explicitly provided by the user (``False``). + - ``inherited`` (`bool`): Whether or not that attribute has been inherited + from a base class. + - ``eq_key`` and ``order_key`` (`typing.Callable` or `None`): The + callables that are used for comparing and ordering objects by this + attribute, respectively. These are set by passing a callable to + `attr.ib`'s ``eq``, ``order``, or ``cmp`` arguments. See also + :ref:`comparison customization `. + + Instances of this class are frequently used for introspection purposes + like: + + - `fields` returns a tuple of them. + - Validators get them passed as the first argument. + - The :ref:`field transformer ` hook receives a list of + them. + - The ``alias`` property exposes the __init__ parameter name of the field, + with any overrides and default private-attribute handling applied. + + + .. versionadded:: 20.1.0 *inherited* + .. versionadded:: 20.1.0 *on_setattr* + .. versionchanged:: 20.2.0 *inherited* is not taken into account for + equality checks and hashing anymore. + .. versionadded:: 21.1.0 *eq_key* and *order_key* + .. versionadded:: 22.2.0 *alias* + .. versionadded:: 26.1.0 *alias_is_default* + + For the full version history of the fields, see `attr.ib`. + """ + + # These slots must NOT be reordered because we use them later for + # instantiation. + __slots__ = ( # noqa: RUF023 + "name", + "default", + "validator", + "repr", + "eq", + "eq_key", + "order", + "order_key", + "hash", + "init", + "metadata", + "type", + "converter", + "kw_only", + "inherited", + "on_setattr", + "alias", + "alias_is_default", + ) + + def __init__( + self, + name, + default, + validator, + repr, + cmp, # XXX: unused, remove along with other cmp code. + hash, + init, + inherited, + metadata=None, + type=None, + converter=None, + kw_only=False, + eq=None, + eq_key=None, + order=None, + order_key=None, + on_setattr=None, + alias=None, + alias_is_default=None, + ): + eq, eq_key, order, order_key = _determine_attrib_eq_order( + cmp, eq_key or eq, order_key or order, True + ) + + # Cache this descriptor here to speed things up later. + bound_setattr = _OBJ_SETATTR.__get__(self) + + # Despite the big red warning, people *do* instantiate `Attribute` + # themselves. + bound_setattr("name", name) + bound_setattr("default", default) + bound_setattr("validator", validator) + bound_setattr("repr", repr) + bound_setattr("eq", eq) + bound_setattr("eq_key", eq_key) + bound_setattr("order", order) + bound_setattr("order_key", order_key) + bound_setattr("hash", hash) + bound_setattr("init", init) + bound_setattr("converter", converter) + bound_setattr( + "metadata", + ( + types.MappingProxyType(dict(metadata)) # Shallow copy + if metadata + else _EMPTY_METADATA_SINGLETON + ), + ) + bound_setattr("type", type) + bound_setattr("kw_only", kw_only) + bound_setattr("inherited", inherited) + bound_setattr("on_setattr", on_setattr) + bound_setattr("alias", alias) + bound_setattr( + "alias_is_default", + alias is None if alias_is_default is None else alias_is_default, + ) + + def __setattr__(self, name, value): + raise FrozenInstanceError + + @classmethod + def from_counting_attr( + cls, name: str, ca: _CountingAttr, kw_only: bool, type=None + ): + # The 'kw_only' argument is the class-level setting, and is used if the + # attribute itself does not explicitly set 'kw_only'. + # type holds the annotated value. deal with conflicts: + if type is None: + type = ca.type + elif ca.type is not None: + msg = f"Type annotation and type argument cannot both be present for '{name}'." + raise ValueError(msg) + return cls( + name, + ca._default, + ca._validator, + ca.repr, + None, + ca.hash, + ca.init, + False, + ca.metadata, + type, + ca.converter, + kw_only if ca.kw_only is None else ca.kw_only, + ca.eq, + ca.eq_key, + ca.order, + ca.order_key, + ca.on_setattr, + ca.alias, + ca.alias is None, + ) + + # Don't use attrs.evolve since fields(Attribute) doesn't work + def evolve(self, **changes): + """ + Copy *self* and apply *changes*. + + This works similarly to `attrs.evolve` but that function does not work + with :class:`attrs.Attribute`. + + It is mainly meant to be used for `transform-fields`. + + .. versionadded:: 20.3.0 + """ + new = copy.copy(self) + + new._setattrs(changes.items()) + + if "alias" in changes and "alias_is_default" not in changes: + # Explicit alias provided -- no longer the default. + _OBJ_SETATTR.__get__(new)("alias_is_default", False) + elif ( + "name" in changes + and "alias" not in changes + # Don't auto-generate alias if the user picked picked the old one. + and self.alias_is_default + ): + # Name changed, alias was auto-generated -- update it. + _OBJ_SETATTR.__get__(new)( + "alias", _default_init_alias_for(new.name) + ) + + return new + + # Don't use _add_pickle since fields(Attribute) doesn't work + def __getstate__(self): + """ + Play nice with pickle. + """ + return tuple( + getattr(self, name) if name != "metadata" else dict(self.metadata) + for name in self.__slots__ + ) + + def __setstate__(self, state): + """ + Play nice with pickle. + """ + if len(state) < len(self.__slots__): + # Pre-26.1.0 pickle without alias_is_default -- infer it + # heuristically. + state_dict = dict(zip(self.__slots__, state)) + alias_is_default = state_dict.get( + "alias" + ) is None or state_dict.get("alias") == _default_init_alias_for( + state_dict["name"] + ) + state = (*state, alias_is_default) + + self._setattrs(zip(self.__slots__, state)) + + def _setattrs(self, name_values_pairs): + bound_setattr = _OBJ_SETATTR.__get__(self) + for name, value in name_values_pairs: + if name != "metadata": + bound_setattr(name, value) + else: + bound_setattr( + name, + ( + types.MappingProxyType(dict(value)) + if value + else _EMPTY_METADATA_SINGLETON + ), + ) + + +_a = [ + Attribute( + name=name, + default=NOTHING, + validator=None, + repr=(name != "alias_is_default"), + cmp=None, + eq=True, + order=False, + hash=(name != "metadata"), + init=True, + inherited=False, + alias=_default_init_alias_for(name), + ) + for name in Attribute.__slots__ +] + +Attribute = _add_hash( + _add_eq( + _add_repr(Attribute, attrs=_a), + attrs=[a for a in _a if a.name != "inherited"], + ), + attrs=[a for a in _a if a.hash and a.name != "inherited"], +) + + +class _CountingAttr: + """ + Intermediate representation of attributes that uses a counter to preserve + the order in which the attributes have been defined. + + *Internal* data structure of the attrs library. Running into is most + likely the result of a bug like a forgotten `@attr.s` decorator. + """ + + __slots__ = ( + "_default", + "_validator", + "alias", + "converter", + "counter", + "eq", + "eq_key", + "hash", + "init", + "kw_only", + "metadata", + "on_setattr", + "order", + "order_key", + "repr", + "type", + ) + __attrs_attrs__ = ( + *tuple( + Attribute( + name=name, + alias=_default_init_alias_for(name), + default=NOTHING, + validator=None, + repr=True, + cmp=None, + hash=True, + init=True, + kw_only=False, + eq=True, + eq_key=None, + order=False, + order_key=None, + inherited=False, + on_setattr=None, + ) + for name in ( + "counter", + "_default", + "repr", + "eq", + "order", + "hash", + "init", + "on_setattr", + "alias", + ) + ), + Attribute( + name="metadata", + alias="metadata", + default=None, + validator=None, + repr=True, + cmp=None, + hash=False, + init=True, + kw_only=False, + eq=True, + eq_key=None, + order=False, + order_key=None, + inherited=False, + on_setattr=None, + ), + ) + cls_counter = 0 + + def __init__( + self, + default, + validator, + repr, + cmp, + hash, + init, + converter, + metadata, + type, + kw_only, + eq, + eq_key, + order, + order_key, + on_setattr, + alias, + ): + _CountingAttr.cls_counter += 1 + self.counter = _CountingAttr.cls_counter + self._default = default + self._validator = validator + self.converter = converter + self.repr = repr + self.eq = eq + self.eq_key = eq_key + self.order = order + self.order_key = order_key + self.hash = hash + self.init = init + self.metadata = metadata + self.type = type + self.kw_only = kw_only + self.on_setattr = on_setattr + self.alias = alias + + def validator(self, meth): + """ + Decorator that adds *meth* to the list of validators. + + Returns *meth* unchanged. + + .. versionadded:: 17.1.0 + """ + if self._validator is None: + self._validator = meth + else: + self._validator = and_(self._validator, meth) + return meth + + def default(self, meth): + """ + Decorator that allows to set the default for an attribute. + + Returns *meth* unchanged. + + Raises: + DefaultAlreadySetError: If default has been set before. + + .. versionadded:: 17.1.0 + """ + if self._default is not NOTHING: + raise DefaultAlreadySetError + + self._default = Factory(meth, takes_self=True) + + return meth + + +_CountingAttr = _add_eq(_add_repr(_CountingAttr)) + + +class ClassProps: + """ + Effective class properties as derived from parameters to `attr.s()` or + `define()` decorators. + + This is the same data structure that *attrs* uses internally to decide how + to construct the final class. + + Warning: + + This feature is currently **experimental** and is not covered by our + strict backwards-compatibility guarantees. + + + Attributes: + is_exception (bool): + Whether the class is treated as an exception class. + + is_slotted (bool): + Whether the class is `slotted `. + + has_weakref_slot (bool): + Whether the class has a slot for weak references. + + is_frozen (bool): + Whether the class is frozen. + + kw_only (KeywordOnly): + Whether / how the class enforces keyword-only arguments on the + ``__init__`` method. + + collected_fields_by_mro (bool): + Whether the class fields were collected by method resolution order. + That is, correctly but unlike `dataclasses`. + + added_init (bool): + Whether the class has an *attrs*-generated ``__init__`` method. + + added_repr (bool): + Whether the class has an *attrs*-generated ``__repr__`` method. + + added_eq (bool): + Whether the class has *attrs*-generated equality methods. + + added_ordering (bool): + Whether the class has *attrs*-generated ordering methods. + + hashability (Hashability): How `hashable ` the class is. + + added_match_args (bool): + Whether the class supports positional `match ` over its + fields. + + added_str (bool): + Whether the class has an *attrs*-generated ``__str__`` method. + + added_pickling (bool): + Whether the class has *attrs*-generated ``__getstate__`` and + ``__setstate__`` methods for `pickle`. + + on_setattr_hook (Callable[[Any, Attribute[Any], Any], Any] | None): + The class's ``__setattr__`` hook. + + field_transformer (Callable[[Attribute[Any]], Attribute[Any]] | None): + The class's `field transformers `. + + .. versionadded:: 25.4.0 + """ + + class Hashability(enum.Enum): + """ + The hashability of a class. + + .. versionadded:: 25.4.0 + """ + + HASHABLE = "hashable" + """Write a ``__hash__``.""" + HASHABLE_CACHED = "hashable_cache" + """Write a ``__hash__`` and cache the hash.""" + UNHASHABLE = "unhashable" + """Set ``__hash__`` to ``None``.""" + LEAVE_ALONE = "leave_alone" + """Don't touch ``__hash__``.""" + + class KeywordOnly(enum.Enum): + """ + How attributes should be treated regarding keyword-only parameters. + + .. versionadded:: 25.4.0 + """ + + NO = "no" + """Attributes are not keyword-only.""" + YES = "yes" + """Attributes in current class without kw_only=False are keyword-only.""" + FORCE = "force" + """All attributes are keyword-only.""" + + __slots__ = ( # noqa: RUF023 -- order matters for __init__ + "is_exception", + "is_slotted", + "has_weakref_slot", + "is_frozen", + "kw_only", + "collected_fields_by_mro", + "added_init", + "added_repr", + "added_eq", + "added_ordering", + "hashability", + "added_match_args", + "added_str", + "added_pickling", + "on_setattr_hook", + "field_transformer", + ) + + def __init__( + self, + is_exception, + is_slotted, + has_weakref_slot, + is_frozen, + kw_only, + collected_fields_by_mro, + added_init, + added_repr, + added_eq, + added_ordering, + hashability, + added_match_args, + added_str, + added_pickling, + on_setattr_hook, + field_transformer, + ): + self.is_exception = is_exception + self.is_slotted = is_slotted + self.has_weakref_slot = has_weakref_slot + self.is_frozen = is_frozen + self.kw_only = kw_only + self.collected_fields_by_mro = collected_fields_by_mro + self.added_init = added_init + self.added_repr = added_repr + self.added_eq = added_eq + self.added_ordering = added_ordering + self.hashability = hashability + self.added_match_args = added_match_args + self.added_str = added_str + self.added_pickling = added_pickling + self.on_setattr_hook = on_setattr_hook + self.field_transformer = field_transformer + + @property + def is_hashable(self): + return ( + self.hashability is ClassProps.Hashability.HASHABLE + or self.hashability is ClassProps.Hashability.HASHABLE_CACHED + ) + + +_cas = [ + Attribute( + name=name, + default=NOTHING, + validator=None, + repr=True, + cmp=None, + eq=True, + order=False, + hash=True, + init=True, + inherited=False, + alias=_default_init_alias_for(name), + ) + for name in ClassProps.__slots__ +] + +ClassProps = _add_eq(_add_repr(ClassProps, attrs=_cas), attrs=_cas) + + +class Factory: + """ + Stores a factory callable. + + If passed as the default value to `attrs.field`, the factory is used to + generate a new value. + + Args: + factory (typing.Callable): + A callable that takes either none or exactly one mandatory + positional argument depending on *takes_self*. + + takes_self (bool): + Pass the partially initialized instance that is being initialized + as a positional argument. + + .. versionadded:: 17.1.0 *takes_self* + """ + + __slots__ = ("factory", "takes_self") + + def __init__(self, factory, takes_self=False): + self.factory = factory + self.takes_self = takes_self + + def __getstate__(self): + """ + Play nice with pickle. + """ + return tuple(getattr(self, name) for name in self.__slots__) + + def __setstate__(self, state): + """ + Play nice with pickle. + """ + for name, value in zip(self.__slots__, state): + setattr(self, name, value) + + +_f = [ + Attribute( + name=name, + default=NOTHING, + validator=None, + repr=True, + cmp=None, + eq=True, + order=False, + hash=True, + init=True, + inherited=False, + ) + for name in Factory.__slots__ +] + +Factory = _add_hash(_add_eq(_add_repr(Factory, attrs=_f), attrs=_f), attrs=_f) + + +class Converter: + """ + Stores a converter callable. + + Allows for the wrapped converter to take additional arguments. The + arguments are passed in the order they are documented. + + Args: + converter (Callable): A callable that converts the passed value. + + takes_self (bool): + Pass the partially initialized instance that is being initialized + as a positional argument. (default: `False`) + + takes_field (bool): + Pass the field definition (an :class:`Attribute`) into the + converter as a positional argument. (default: `False`) + + .. versionadded:: 24.1.0 + """ + + __slots__ = ( + "__call__", + "_first_param_type", + "_global_name", + "converter", + "takes_field", + "takes_self", + ) + + def __init__(self, converter, *, takes_self=False, takes_field=False): + self.converter = converter + self.takes_self = takes_self + self.takes_field = takes_field + + ex = _AnnotationExtractor(converter) + self._first_param_type = ex.get_first_param_type() + + if not (self.takes_self or self.takes_field): + self.__call__ = lambda value, _, __: self.converter(value) + elif self.takes_self and not self.takes_field: + self.__call__ = lambda value, instance, __: self.converter( + value, instance + ) + elif not self.takes_self and self.takes_field: + self.__call__ = lambda value, __, field: self.converter( + value, field + ) + else: + self.__call__ = self.converter + + rt = ex.get_return_type() + if rt is not None: + self.__call__.__annotations__["return"] = rt + + @staticmethod + def _get_global_name(attr_name: str) -> str: + """ + Return the name that a converter for an attribute name *attr_name* + would have. + """ + return f"__attr_converter_{attr_name}" + + def _fmt_converter_call(self, attr_name: str, value_var: str) -> str: + """ + Return a string that calls the converter for an attribute name + *attr_name* and the value in variable named *value_var* according to + `self.takes_self` and `self.takes_field`. + """ + if not (self.takes_self or self.takes_field): + return f"{self._get_global_name(attr_name)}({value_var})" + + if self.takes_self and self.takes_field: + return f"{self._get_global_name(attr_name)}({value_var}, self, attr_dict['{attr_name}'])" + + if self.takes_self: + return f"{self._get_global_name(attr_name)}({value_var}, self)" + + return f"{self._get_global_name(attr_name)}({value_var}, attr_dict['{attr_name}'])" + + def __getstate__(self): + """ + Return a dict containing only converter and takes_self -- the rest gets + computed when loading. + """ + return { + "converter": self.converter, + "takes_self": self.takes_self, + "takes_field": self.takes_field, + } + + def __setstate__(self, state): + """ + Load instance from state. + """ + self.__init__(**state) + + +_f = [ + Attribute( + name=name, + default=NOTHING, + validator=None, + repr=True, + cmp=None, + eq=True, + order=False, + hash=True, + init=True, + inherited=False, + ) + for name in ("converter", "takes_self", "takes_field") +] + +Converter = _add_hash( + _add_eq(_add_repr(Converter, attrs=_f), attrs=_f), attrs=_f +) + + +def make_class( + name, attrs, bases=(object,), class_body=None, **attributes_arguments +): + r""" + A quick way to create a new class called *name* with *attrs*. + + .. note:: + + ``make_class()`` is a thin wrapper around `attr.s`, not `attrs.define` + which means that it doesn't come with some of the improved defaults. + + For example, if you want the same ``on_setattr`` behavior as in + `attrs.define`, you have to pass the hooks yourself: ``make_class(..., + on_setattr=setters.pipe(setters.convert, setters.validate)`` + + .. warning:: + + It is *your* duty to ensure that the class name and the attribute names + are valid identifiers. ``make_class()`` will *not* validate them for + you. + + Args: + name (str): The name for the new class. + + attrs (list | dict): + A list of names or a dictionary of mappings of names to `attr.ib`\ + s / `attrs.field`\ s. + + The order is deduced from the order of the names or attributes + inside *attrs*. Otherwise the order of the definition of the + attributes is used. + + bases (tuple[type, ...]): Classes that the new class will subclass. + + class_body (dict): + An optional dictionary of class attributes for the new class. + + attributes_arguments: Passed unmodified to `attr.s`. + + Returns: + type: A new class with *attrs*. + + .. versionadded:: 17.1.0 *bases* + .. versionchanged:: 18.1.0 If *attrs* is ordered, the order is retained. + .. versionchanged:: 23.2.0 *class_body* + .. versionchanged:: 25.2.0 Class names can now be unicode. + """ + # Class identifiers are converted into the normal form NFKC while parsing + name = unicodedata.normalize("NFKC", name) + + if isinstance(attrs, dict): + cls_dict = attrs + elif isinstance(attrs, (list, tuple)): + cls_dict = {a: attrib() for a in attrs} + else: + msg = "attrs argument must be a dict or a list." + raise TypeError(msg) + + pre_init = cls_dict.pop("__attrs_pre_init__", None) + post_init = cls_dict.pop("__attrs_post_init__", None) + user_init = cls_dict.pop("__init__", None) + + body = {} + if class_body is not None: + body.update(class_body) + if pre_init is not None: + body["__attrs_pre_init__"] = pre_init + if post_init is not None: + body["__attrs_post_init__"] = post_init + if user_init is not None: + body["__init__"] = user_init + + type_ = types.new_class(name, bases, {}, lambda ns: ns.update(body)) + + # For pickling to work, the __module__ variable needs to be set to the + # frame where the class is created. Bypass this step in environments where + # sys._getframe is not defined (Jython for example) or sys._getframe is not + # defined for arguments greater than 0 (IronPython). + with contextlib.suppress(AttributeError, ValueError): + type_.__module__ = sys._getframe(1).f_globals.get( + "__name__", "__main__" + ) + + # We do it here for proper warnings with meaningful stacklevel. + cmp = attributes_arguments.pop("cmp", None) + ( + attributes_arguments["eq"], + attributes_arguments["order"], + ) = _determine_attrs_eq_order( + cmp, + attributes_arguments.get("eq"), + attributes_arguments.get("order"), + True, + ) + + cls = _attrs(these=cls_dict, **attributes_arguments)(type_) + # Only add type annotations now or "_attrs()" will complain: + cls.__annotations__ = { + k: v.type for k, v in cls_dict.items() if v.type is not None + } + return cls + + +# These are required by within this module so we define them here and merely +# import into .validators / .converters. + + +@attrs(slots=True, unsafe_hash=True) +class _AndValidator: + """ + Compose many validators to a single one. + """ + + _validators = attrib() + + def __call__(self, inst, attr, value): + for v in self._validators: + v(inst, attr, value) + + +def and_(*validators): + """ + A validator that composes multiple validators into one. + + When called on a value, it runs all wrapped validators. + + Args: + validators (~collections.abc.Iterable[typing.Callable]): + Arbitrary number of validators. + + .. versionadded:: 17.1.0 + """ + vals = [] + for validator in validators: + vals.extend( + validator._validators + if isinstance(validator, _AndValidator) + else [validator] + ) + + return _AndValidator(tuple(vals)) + + +def pipe(*converters): + """ + A converter that composes multiple converters into one. + + When called on a value, it runs all wrapped converters, returning the + *last* value. + + Type annotations will be inferred from the wrapped converters', if they + have any. + + converters (~collections.abc.Iterable[typing.Callable]): + Arbitrary number of converters. + + .. versionadded:: 20.1.0 + """ + + return_instance = any(isinstance(c, Converter) for c in converters) + + if return_instance: + + def pipe_converter(val, inst, field): + for c in converters: + val = ( + c(val, inst, field) if isinstance(c, Converter) else c(val) + ) + + return val + + else: + + def pipe_converter(val): + for c in converters: + val = c(val) + + return val + + if not converters: + # If the converter list is empty, pipe_converter is the identity. + A = TypeVar("A") + pipe_converter.__annotations__.update({"val": A, "return": A}) + else: + # Get parameter type from first converter. + t = _AnnotationExtractor(converters[0]).get_first_param_type() + if t: + pipe_converter.__annotations__["val"] = t + + last = converters[-1] + if not PY_3_11_PLUS and isinstance(last, Converter): + last = last.__call__ + + # Get return type from last converter. + rt = _AnnotationExtractor(last).get_return_type() + if rt: + pipe_converter.__annotations__["return"] = rt + + if return_instance: + return Converter(pipe_converter, takes_self=True, takes_field=True) + return pipe_converter diff --git a/py_modules/attr/_next_gen.py b/py_modules/attr/_next_gen.py new file mode 100644 index 0000000..4ccd0da --- /dev/null +++ b/py_modules/attr/_next_gen.py @@ -0,0 +1,674 @@ +# SPDX-License-Identifier: MIT + +""" +These are keyword-only APIs that call `attr.s` and `attr.ib` with different +default values. +""" + +from functools import partial + +from . import setters +from ._funcs import asdict as _asdict +from ._funcs import astuple as _astuple +from ._make import ( + _DEFAULT_ON_SETATTR, + NOTHING, + _frozen_setattrs, + attrib, + attrs, +) +from .exceptions import NotAnAttrsClassError, UnannotatedAttributeError + + +def define( + maybe_cls=None, + *, + these=None, + repr=None, + unsafe_hash=None, + hash=None, + init=None, + slots=True, + frozen=False, + weakref_slot=True, + str=False, + auto_attribs=None, + kw_only=False, + cache_hash=False, + auto_exc=True, + eq=None, + order=False, + auto_detect=True, + getstate_setstate=None, + on_setattr=None, + field_transformer=None, + match_args=True, + force_kw_only=False, +): + r""" + A class decorator that adds :term:`dunder methods` according to + :term:`fields ` specified using :doc:`type annotations `, + `field()` calls, or the *these* argument. + + Since *attrs* patches or replaces an existing class, you cannot use + `object.__init_subclass__` with *attrs* classes, because it runs too early. + As a replacement, you can define ``__attrs_init_subclass__`` on your class. + It will be called by *attrs* classes that subclass it after they're + created. See also :ref:`init-subclass`. + + Args: + slots (bool): + Create a :term:`slotted class ` that's more + memory-efficient. Slotted classes are generally superior to the + default dict classes, but have some gotchas you should know about, + so we encourage you to read the :term:`glossary entry `. + + auto_detect (bool): + Instead of setting the *init*, *repr*, *eq*, and *hash* arguments + explicitly, assume they are set to True **unless any** of the + involved methods for one of the arguments is implemented in the + *current* class (meaning, it is *not* inherited from some base + class). + + So, for example by implementing ``__eq__`` on a class yourself, + *attrs* will deduce ``eq=False`` and will create *neither* + ``__eq__`` *nor* ``__ne__`` (but Python classes come with a + sensible ``__ne__`` by default, so it *should* be enough to only + implement ``__eq__`` in most cases). + + Passing :data:`True` or :data:`False` to *init*, *repr*, *eq*, or *hash* + overrides whatever *auto_detect* would determine. + + auto_exc (bool): + If the class subclasses `BaseException` (which implicitly includes + any subclass of any exception), the following happens to behave + like a well-behaved Python exception class: + + - the values for *eq*, *order*, and *hash* are ignored and the + instances compare and hash by the instance's ids [#]_ , + - all attributes that are either passed into ``__init__`` or have a + default value are additionally available as a tuple in the + ``args`` attribute, + - the value of *str* is ignored leaving ``__str__`` to base + classes. + + .. [#] + Note that *attrs* will *not* remove existing implementations of + ``__hash__`` or the equality methods. It just won't add own + ones. + + on_setattr (~typing.Callable | list[~typing.Callable] | None | ~typing.Literal[attrs.setters.NO_OP]): + A callable that is run whenever the user attempts to set an + attribute (either by assignment like ``i.x = 42`` or by using + `setattr` like ``setattr(i, "x", 42)``). It receives the same + arguments as validators: the instance, the attribute that is being + modified, and the new value. + + If no exception is raised, the attribute is set to the return value + of the callable. + + If a list of callables is passed, they're automatically wrapped in + an `attrs.setters.pipe`. + + If left None, the default behavior is to run converters and + validators whenever an attribute is set. + + init (bool): + Create a ``__init__`` method that initializes the *attrs* + attributes. Leading underscores are stripped for the argument name, + unless an alias is set on the attribute. + + .. seealso:: + `init` shows advanced ways to customize the generated + ``__init__`` method, including executing code before and after. + + repr(bool): + Create a ``__repr__`` method with a human readable representation + of *attrs* attributes. + + str (bool): + Create a ``__str__`` method that is identical to ``__repr__``. This + is usually not necessary except for `Exception`\ s. + + eq (bool | None): + If True or None (default), add ``__eq__`` and ``__ne__`` methods + that check two instances for equality. + + .. seealso:: + `comparison` describes how to customize the comparison behavior + going as far comparing NumPy arrays. + + order (bool | None): + If True, add ``__lt__``, ``__le__``, ``__gt__``, and ``__ge__`` + methods that behave like *eq* above and allow instances to be + ordered. + + They compare the instances as if they were tuples of their *attrs* + attributes if and only if the types of both classes are + *identical*. + + If `None` mirror value of *eq*. + + .. seealso:: `comparison` + + unsafe_hash (bool | None): + If None (default), the ``__hash__`` method is generated according + how *eq* and *frozen* are set. + + 1. If *both* are True, *attrs* will generate a ``__hash__`` for + you. + 2. If *eq* is True and *frozen* is False, ``__hash__`` will be set + to None, marking it unhashable (which it is). + 3. If *eq* is False, ``__hash__`` will be left untouched meaning + the ``__hash__`` method of the base class will be used. If the + base class is `object`, this means it will fall back to id-based + hashing. + + Although not recommended, you can decide for yourself and force + *attrs* to create one (for example, if the class is immutable even + though you didn't freeze it programmatically) by passing True or + not. Both of these cases are rather special and should be used + carefully. + + .. seealso:: + + - Our documentation on `hashing`, + - Python's documentation on `object.__hash__`, + - and the `GitHub issue that led to the default \ behavior + `_ for more + details. + + hash (bool | None): + Deprecated alias for *unsafe_hash*. *unsafe_hash* takes precedence. + + cache_hash (bool): + Ensure that the object's hash code is computed only once and stored + on the object. If this is set to True, hashing must be either + explicitly or implicitly enabled for this class. If the hash code + is cached, avoid any reassignments of fields involved in hash code + computation or mutations of the objects those fields point to after + object creation. If such changes occur, the behavior of the + object's hash code is undefined. + + frozen (bool): + Make instances immutable after initialization. If someone attempts + to modify a frozen instance, `attrs.exceptions.FrozenInstanceError` + is raised. + + .. note:: + + 1. This is achieved by installing a custom ``__setattr__`` + method on your class, so you can't implement your own. + + 2. True immutability is impossible in Python. + + 3. This *does* have a minor a runtime performance `impact + ` when initializing new instances. In other + words: ``__init__`` is slightly slower with ``frozen=True``. + + 4. If a class is frozen, you cannot modify ``self`` in + ``__attrs_post_init__`` or a self-written ``__init__``. You + can circumvent that limitation by using + ``object.__setattr__(self, "attribute_name", value)``. + + 5. Subclasses of a frozen class are frozen too. + + kw_only (bool): + Make attributes keyword-only in the generated ``__init__`` (if + *init* is False, this parameter is ignored). Attributes that + explicitly set ``kw_only=False`` are not affected; base class + attributes are also not affected. + + Also see *force_kw_only*. + + weakref_slot (bool): + Make instances weak-referenceable. This has no effect unless + *slots* is True. + + field_transformer (~typing.Callable | None): + A function that is called with the original class object and all + fields right before *attrs* finalizes the class. You can use this, + for example, to automatically add converters or validators to + fields based on their types. + + .. seealso:: `transform-fields` + + match_args (bool): + If True (default), set ``__match_args__`` on the class to support + :pep:`634` (*Structural Pattern Matching*). It is a tuple of all + non-keyword-only ``__init__`` parameter names on Python 3.10 and + later. Ignored on older Python versions. + + collect_by_mro (bool): + If True, *attrs* collects attributes from base classes correctly + according to the `method resolution order + `_. If False, *attrs* + will mimic the (wrong) behavior of `dataclasses` and :pep:`681`. + + See also `issue #428 + `_. + + force_kw_only (bool): + A back-compat flag for restoring pre-25.4.0 behavior. If True and + ``kw_only=True``, all attributes are made keyword-only, including + base class attributes, and those set to ``kw_only=False`` at the + attribute level. Defaults to False. + + See also `issue #980 + `_. + + getstate_setstate (bool | None): + .. note:: + + This is usually only interesting for slotted classes and you + should probably just set *auto_detect* to True. + + If True, ``__getstate__`` and ``__setstate__`` are generated and + attached to the class. This is necessary for slotted classes to be + pickleable. If left None, it's True by default for slotted classes + and False for dict classes. + + If *auto_detect* is True, and *getstate_setstate* is left None, and + **either** ``__getstate__`` or ``__setstate__`` is detected + directly on the class (meaning: not inherited), it is set to False + (this is usually what you want). + + auto_attribs (bool | None): + If True, look at type annotations to determine which attributes to + use, like `dataclasses`. If False, it will only look for explicit + :func:`field` class attributes, like classic *attrs*. + + If left None, it will guess: + + 1. If any attributes are annotated and no unannotated + `attrs.field`\ s are found, it assumes *auto_attribs=True*. + 2. Otherwise it assumes *auto_attribs=False* and tries to collect + `attrs.field`\ s. + + If *attrs* decides to look at type annotations, **all** fields + **must** be annotated. If *attrs* encounters a field that is set to + a :func:`field` / `attr.ib` but lacks a type annotation, an + `attrs.exceptions.UnannotatedAttributeError` is raised. Use + ``field_name: typing.Any = field(...)`` if you don't want to set a + type. + + .. warning:: + + For features that use the attribute name to create decorators + (for example, :ref:`validators `), you still *must* + assign :func:`field` / `attr.ib` to them. Otherwise Python will + either not find the name or try to use the default value to + call, for example, ``validator`` on it. + + Attributes annotated as `typing.ClassVar`, and attributes that are + neither annotated nor set to an `field()` are **ignored**. + + these (dict[str, object]): + A dictionary of name to the (private) return value of `field()` + mappings. This is useful to avoid the definition of your attributes + within the class body because you can't (for example, if you want + to add ``__repr__`` methods to Django models) or don't want to. + + If *these* is not `None`, *attrs* will *not* search the class body + for attributes and will *not* remove any attributes from it. + + The order is deduced from the order of the attributes inside + *these*. + + Arguably, this is a rather obscure feature. + + .. versionadded:: 20.1.0 + .. versionchanged:: 21.3.0 Converters are also run ``on_setattr``. + .. versionadded:: 22.2.0 + *unsafe_hash* as an alias for *hash* (for :pep:`681` compliance). + .. versionchanged:: 24.1.0 + Instances are not compared as tuples of attributes anymore, but using a + big ``and`` condition. This is faster and has more correct behavior for + uncomparable values like `math.nan`. + .. versionadded:: 24.1.0 + If a class has an *inherited* classmethod called + ``__attrs_init_subclass__``, it is executed after the class is created. + .. deprecated:: 24.1.0 *hash* is deprecated in favor of *unsafe_hash*. + .. versionadded:: 24.3.0 + Unless already present, a ``__replace__`` method is automatically + created for `copy.replace` (Python 3.13+ only). + .. versionchanged:: 25.4.0 + *kw_only* now only applies to attributes defined in the current class, + and respects attribute-level ``kw_only=False`` settings. + .. versionadded:: 25.4.0 + Added *force_kw_only* to go back to the previous *kw_only* behavior. + + .. note:: + + The main differences to the classic `attr.s` are: + + - Automatically detect whether or not *auto_attribs* should be `True` + (c.f. *auto_attribs* parameter). + - Converters and validators run when attributes are set by default -- + if *frozen* is `False`. + - *slots=True* + + Usually, this has only upsides and few visible effects in everyday + programming. But it *can* lead to some surprising behaviors, so + please make sure to read :term:`slotted classes`. + + - *auto_exc=True* + - *auto_detect=True* + - *order=False* + - *force_kw_only=False* + - Some options that were only relevant on Python 2 or were kept around + for backwards-compatibility have been removed. + + """ + + def do_it(cls, auto_attribs): + return attrs( + maybe_cls=cls, + these=these, + repr=repr, + hash=hash, + unsafe_hash=unsafe_hash, + init=init, + slots=slots, + frozen=frozen, + weakref_slot=weakref_slot, + str=str, + auto_attribs=auto_attribs, + kw_only=kw_only, + cache_hash=cache_hash, + auto_exc=auto_exc, + eq=eq, + order=order, + auto_detect=auto_detect, + collect_by_mro=True, + getstate_setstate=getstate_setstate, + on_setattr=on_setattr, + field_transformer=field_transformer, + match_args=match_args, + force_kw_only=force_kw_only, + ) + + def wrap(cls): + """ + Making this a wrapper ensures this code runs during class creation. + + We also ensure that frozen-ness of classes is inherited. + """ + nonlocal frozen, on_setattr + + had_on_setattr = on_setattr not in (None, setters.NO_OP) + + # By default, mutable classes convert & validate on setattr. + if frozen is False and on_setattr is None: + on_setattr = _DEFAULT_ON_SETATTR + + # However, if we subclass a frozen class, we inherit the immutability + # and disable on_setattr. + for base_cls in cls.__bases__: + if base_cls.__setattr__ is _frozen_setattrs: + if had_on_setattr: + msg = "Frozen classes can't use on_setattr (frozen-ness was inherited)." + raise ValueError(msg) + + on_setattr = setters.NO_OP + break + + if auto_attribs is not None: + return do_it(cls, auto_attribs) + + try: + return do_it(cls, True) + except UnannotatedAttributeError: + return do_it(cls, False) + + # maybe_cls's type depends on the usage of the decorator. It's a class + # if it's used as `@attrs` but `None` if used as `@attrs()`. + if maybe_cls is None: + return wrap + + return wrap(maybe_cls) + + +mutable = define +frozen = partial(define, frozen=True, on_setattr=None) + + +def field( + *, + default=NOTHING, + validator=None, + repr=True, + hash=None, + init=True, + metadata=None, + type=None, + converter=None, + factory=None, + kw_only=None, + eq=None, + order=None, + on_setattr=None, + alias=None, +): + """ + Create a new :term:`field` / :term:`attribute` on a class. + + .. warning:: + + Does **nothing** unless the class is also decorated with + `attrs.define` (or similar)! + + Args: + default: + A value that is used if an *attrs*-generated ``__init__`` is used + and no value is passed while instantiating or the attribute is + excluded using ``init=False``. + + If the value is an instance of `attrs.Factory`, its callable will + be used to construct a new value (useful for mutable data types + like lists or dicts). + + If a default is not set (or set manually to `attrs.NOTHING`), a + value *must* be supplied when instantiating; otherwise a + `TypeError` will be raised. + + .. seealso:: `defaults` + + factory (~typing.Callable): + Syntactic sugar for ``default=attr.Factory(factory)``. + + validator (~typing.Callable | list[~typing.Callable]): + Callable that is called by *attrs*-generated ``__init__`` methods + after the instance has been initialized. They receive the + initialized instance, the :func:`~attrs.Attribute`, and the passed + value. + + The return value is *not* inspected so the validator has to throw + an exception itself. + + If a `list` is passed, its items are treated as validators and must + all pass. + + Validators can be globally disabled and re-enabled using + `attrs.validators.get_disabled` / `attrs.validators.set_disabled`. + + The validator can also be set using decorator notation as shown + below. + + .. seealso:: :ref:`validators` + + repr (bool | ~typing.Callable): + Include this attribute in the generated ``__repr__`` method. If + True, include the attribute; if False, omit it. By default, the + built-in ``repr()`` function is used. To override how the attribute + value is formatted, pass a ``callable`` that takes a single value + and returns a string. Note that the resulting string is used as-is, + which means it will be used directly *instead* of calling + ``repr()`` (the default). + + eq (bool | ~typing.Callable): + If True (default), include this attribute in the generated + ``__eq__`` and ``__ne__`` methods that check two instances for + equality. To override how the attribute value is compared, pass a + callable that takes a single value and returns the value to be + compared. + + .. seealso:: `comparison` + + order (bool | ~typing.Callable): + If True (default), include this attributes in the generated + ``__lt__``, ``__le__``, ``__gt__`` and ``__ge__`` methods. To + override how the attribute value is ordered, pass a callable that + takes a single value and returns the value to be ordered. + + .. seealso:: `comparison` + + hash (bool | None): + Include this attribute in the generated ``__hash__`` method. If + None (default), mirror *eq*'s value. This is the correct behavior + according the Python spec. Setting this value to anything else + than None is *discouraged*. + + .. seealso:: `hashing` + + init (bool): + Include this attribute in the generated ``__init__`` method. + + It is possible to set this to False and set a default value. In + that case this attributed is unconditionally initialized with the + specified default value or factory. + + .. seealso:: `init` + + converter (typing.Callable | Converter): + A callable that is called by *attrs*-generated ``__init__`` methods + to convert attribute's value to the desired format. + + If a vanilla callable is passed, it is given the passed-in value as + the only positional argument. It is possible to receive additional + arguments by wrapping the callable in a `Converter`. + + Either way, the returned value will be used as the new value of the + attribute. The value is converted before being passed to the + validator, if any. + + .. seealso:: :ref:`converters` + + metadata (dict | None): + An arbitrary mapping, to be used by third-party code. + + .. seealso:: `extending-metadata`. + + type (type): + The type of the attribute. Nowadays, the preferred method to + specify the type is using a variable annotation (see :pep:`526`). + This argument is provided for backwards-compatibility and for usage + with `make_class`. Regardless of the approach used, the type will + be stored on ``Attribute.type``. + + Please note that *attrs* doesn't do anything with this metadata by + itself. You can use it as part of your own code or for `static type + checking `. + + kw_only (bool | None): + Make this attribute keyword-only in the generated ``__init__`` (if + *init* is False, this parameter is ignored). If None (default), + mirror the setting from `attrs.define`. + + on_setattr (~typing.Callable | list[~typing.Callable] | None | ~typing.Literal[attrs.setters.NO_OP]): + Allows to overwrite the *on_setattr* setting from `attr.s`. If left + None, the *on_setattr* value from `attr.s` is used. Set to + `attrs.setters.NO_OP` to run **no** `setattr` hooks for this + attribute -- regardless of the setting in `define()`. + + alias (str | None): + Override this attribute's parameter name in the generated + ``__init__`` method. If left None, default to ``name`` stripped + of leading underscores. See `private-attributes`. + + .. versionadded:: 20.1.0 + .. versionchanged:: 21.1.0 + *eq*, *order*, and *cmp* also accept a custom callable + .. versionadded:: 22.2.0 *alias* + .. versionadded:: 23.1.0 + The *type* parameter has been re-added; mostly for `attrs.make_class`. + Please note that type checkers ignore this metadata. + .. versionchanged:: 25.4.0 + *kw_only* can now be None, and its default is also changed from False to + None. + + .. seealso:: + + `attr.ib` + """ + return attrib( + default=default, + validator=validator, + repr=repr, + hash=hash, + init=init, + metadata=metadata, + type=type, + converter=converter, + factory=factory, + kw_only=kw_only, + eq=eq, + order=order, + on_setattr=on_setattr, + alias=alias, + ) + + +def asdict(inst, *, recurse=True, filter=None, value_serializer=None): + """ + Same as `attr.asdict`, except that collections types are always retained + and dict is always used as *dict_factory*. + + .. versionadded:: 21.3.0 + """ + return _asdict( + inst=inst, + recurse=recurse, + filter=filter, + value_serializer=value_serializer, + retain_collection_types=True, + ) + + +def astuple(inst, *, recurse=True, filter=None): + """ + Same as `attr.astuple`, except that collections types are always retained + and `tuple` is always used as the *tuple_factory*. + + .. versionadded:: 21.3.0 + """ + return _astuple( + inst=inst, recurse=recurse, filter=filter, retain_collection_types=True + ) + + +def inspect(cls): + """ + Inspect the class and return its effective build parameters. + + Warning: + This feature is currently **experimental** and is not covered by our + strict backwards-compatibility guarantees. + + Args: + cls: The *attrs*-decorated class to inspect. + + Returns: + The effective build parameters of the class. + + Raises: + NotAnAttrsClassError: If the class is not an *attrs*-decorated class. + + .. versionadded:: 25.4.0 + """ + try: + return cls.__dict__["__attrs_props__"] + except KeyError: + msg = f"{cls!r} is not an attrs-decorated class." + raise NotAnAttrsClassError(msg) from None diff --git a/py_modules/attr/_typing_compat.pyi b/py_modules/attr/_typing_compat.pyi new file mode 100644 index 0000000..ca7b71e --- /dev/null +++ b/py_modules/attr/_typing_compat.pyi @@ -0,0 +1,15 @@ +from typing import Any, ClassVar, Protocol + +# MYPY is a special constant in mypy which works the same way as `TYPE_CHECKING`. +MYPY = False + +if MYPY: + # A protocol to be able to statically accept an attrs class. + class AttrsInstance_(Protocol): + __attrs_attrs__: ClassVar[Any] + +else: + # For type checkers without plug-in support use an empty protocol that + # will (hopefully) be combined into a union. + class AttrsInstance_(Protocol): + pass diff --git a/py_modules/attr/_version_info.py b/py_modules/attr/_version_info.py new file mode 100644 index 0000000..27f1888 --- /dev/null +++ b/py_modules/attr/_version_info.py @@ -0,0 +1,89 @@ +# SPDX-License-Identifier: MIT + + +from functools import total_ordering + +from ._funcs import astuple +from ._make import attrib, attrs + + +@total_ordering +@attrs(eq=False, order=False, slots=True, frozen=True) +class VersionInfo: + """ + A version object that can be compared to tuple of length 1--4: + + >>> attr.VersionInfo(19, 1, 0, "final") <= (19, 2) + True + >>> attr.VersionInfo(19, 1, 0, "final") < (19, 1, 1) + True + >>> vi = attr.VersionInfo(19, 2, 0, "final") + >>> vi < (19, 1, 1) + False + >>> vi < (19,) + False + >>> vi == (19, 2,) + True + >>> vi == (19, 2, 1) + False + + .. versionadded:: 19.2 + """ + + year = attrib(type=int) + minor = attrib(type=int) + micro = attrib(type=int) + releaselevel = attrib(type=str) + + @classmethod + def _from_version_string(cls, s): + """ + Parse *s* and return a _VersionInfo. + """ + v = s.split(".") + if len(v) == 3: + v.append("final") + + return cls( + year=int(v[0]), minor=int(v[1]), micro=int(v[2]), releaselevel=v[3] + ) + + def _ensure_tuple(self, other): + """ + Ensure *other* is a tuple of a valid length. + + Returns a possibly transformed *other* and ourselves as a tuple of + the same length as *other*. + """ + + if self.__class__ is other.__class__: + other = astuple(other) + + if not isinstance(other, tuple): + raise NotImplementedError + + if not (1 <= len(other) <= 4): + raise NotImplementedError + + return astuple(self)[: len(other)], other + + def __eq__(self, other): + try: + us, them = self._ensure_tuple(other) + except NotImplementedError: + return NotImplemented + + return us == them + + def __lt__(self, other): + try: + us, them = self._ensure_tuple(other) + except NotImplementedError: + return NotImplemented + + # Since alphabetically "dev0" < "final" < "post1" < "post2", we don't + # have to do anything special with releaselevel for now. + return us < them + + def __hash__(self): + return hash((self.year, self.minor, self.micro, self.releaselevel)) diff --git a/py_modules/attr/_version_info.pyi b/py_modules/attr/_version_info.pyi new file mode 100644 index 0000000..45ced08 --- /dev/null +++ b/py_modules/attr/_version_info.pyi @@ -0,0 +1,9 @@ +class VersionInfo: + @property + def year(self) -> int: ... + @property + def minor(self) -> int: ... + @property + def micro(self) -> int: ... + @property + def releaselevel(self) -> str: ... diff --git a/py_modules/attr/converters.py b/py_modules/attr/converters.py new file mode 100644 index 0000000..0a79dee --- /dev/null +++ b/py_modules/attr/converters.py @@ -0,0 +1,162 @@ +# SPDX-License-Identifier: MIT + +""" +Commonly useful converters. +""" + +import typing + +from ._compat import _AnnotationExtractor +from ._make import NOTHING, Converter, Factory, pipe + + +__all__ = [ + "default_if_none", + "optional", + "pipe", + "to_bool", +] + + +def optional(converter): + """ + A converter that allows an attribute to be optional. An optional attribute + is one which can be set to `None`. + + Type annotations will be inferred from the wrapped converter's, if it has + any. + + Args: + converter (typing.Callable): + the converter that is used for non-`None` values. + + .. versionadded:: 17.1.0 + """ + + if isinstance(converter, Converter): + + def optional_converter(val, inst, field): + if val is None: + return None + return converter(val, inst, field) + + else: + + def optional_converter(val): + if val is None: + return None + return converter(val) + + xtr = _AnnotationExtractor(converter) + + t = xtr.get_first_param_type() + if t: + optional_converter.__annotations__["val"] = typing.Optional[t] + + rt = xtr.get_return_type() + if rt: + optional_converter.__annotations__["return"] = typing.Optional[rt] + + if isinstance(converter, Converter): + return Converter(optional_converter, takes_self=True, takes_field=True) + + return optional_converter + + +def default_if_none(default=NOTHING, factory=None): + """ + A converter that allows to replace `None` values by *default* or the result + of *factory*. + + Args: + default: + Value to be used if `None` is passed. Passing an instance of + `attrs.Factory` is supported, however the ``takes_self`` option is + *not*. + + factory (typing.Callable): + A callable that takes no parameters whose result is used if `None` + is passed. + + Raises: + TypeError: If **neither** *default* or *factory* is passed. + + TypeError: If **both** *default* and *factory* are passed. + + ValueError: + If an instance of `attrs.Factory` is passed with + ``takes_self=True``. + + .. versionadded:: 18.2.0 + """ + if default is NOTHING and factory is None: + msg = "Must pass either `default` or `factory`." + raise TypeError(msg) + + if default is not NOTHING and factory is not None: + msg = "Must pass either `default` or `factory` but not both." + raise TypeError(msg) + + if factory is not None: + default = Factory(factory) + + if isinstance(default, Factory): + if default.takes_self: + msg = "`takes_self` is not supported by default_if_none." + raise ValueError(msg) + + def default_if_none_converter(val): + if val is not None: + return val + + return default.factory() + + else: + + def default_if_none_converter(val): + if val is not None: + return val + + return default + + return default_if_none_converter + + +def to_bool(val): + """ + Convert "boolean" strings (for example, from environment variables) to real + booleans. + + Values mapping to `True`: + + - ``True`` + - ``"true"`` / ``"t"`` + - ``"yes"`` / ``"y"`` + - ``"on"`` + - ``"1"`` + - ``1`` + + Values mapping to `False`: + + - ``False`` + - ``"false"`` / ``"f"`` + - ``"no"`` / ``"n"`` + - ``"off"`` + - ``"0"`` + - ``0`` + + Raises: + ValueError: For any other value. + + .. versionadded:: 21.3.0 + """ + if isinstance(val, str): + val = val.lower() + + if val in (True, "true", "t", "yes", "y", "on", "1", 1): + return True + if val in (False, "false", "f", "no", "n", "off", "0", 0): + return False + + msg = f"Cannot convert value to bool: {val!r}" + raise ValueError(msg) diff --git a/py_modules/attr/converters.pyi b/py_modules/attr/converters.pyi new file mode 100644 index 0000000..12bd0c4 --- /dev/null +++ b/py_modules/attr/converters.pyi @@ -0,0 +1,19 @@ +from typing import Callable, Any, overload + +from attrs import _ConverterType, _CallableConverterType + +@overload +def pipe(*validators: _CallableConverterType) -> _CallableConverterType: ... +@overload +def pipe(*validators: _ConverterType) -> _ConverterType: ... +@overload +def optional(converter: _CallableConverterType) -> _CallableConverterType: ... +@overload +def optional(converter: _ConverterType) -> _ConverterType: ... +@overload +def default_if_none(default: Any) -> _CallableConverterType: ... +@overload +def default_if_none( + *, factory: Callable[[], Any] +) -> _CallableConverterType: ... +def to_bool(val: str | int | bool) -> bool: ... diff --git a/py_modules/attr/exceptions.py b/py_modules/attr/exceptions.py new file mode 100644 index 0000000..a207df4 --- /dev/null +++ b/py_modules/attr/exceptions.py @@ -0,0 +1,95 @@ +# SPDX-License-Identifier: MIT + +from __future__ import annotations + + +class FrozenError(AttributeError): + """ + A frozen/immutable instance or attribute have been attempted to be + modified. + + It mirrors the behavior of ``namedtuples`` by using the same error message + and subclassing `AttributeError`. + + .. versionadded:: 20.1.0 + """ + + def __init__(self): + msg = "can't set attribute" + super().__init__(msg) + self.msg = msg + + +class FrozenInstanceError(FrozenError): + """ + A frozen instance has been attempted to be modified. + + .. versionadded:: 16.1.0 + """ + + +class FrozenAttributeError(FrozenError): + """ + A frozen attribute has been attempted to be modified. + + .. versionadded:: 20.1.0 + """ + + +class AttrsAttributeNotFoundError(ValueError): + """ + An *attrs* function couldn't find an attribute that the user asked for. + + .. versionadded:: 16.2.0 + """ + + +class NotAnAttrsClassError(ValueError): + """ + A non-*attrs* class has been passed into an *attrs* function. + + .. versionadded:: 16.2.0 + """ + + +class DefaultAlreadySetError(RuntimeError): + """ + A default has been set when defining the field and is attempted to be reset + using the decorator. + + .. versionadded:: 17.1.0 + """ + + +class UnannotatedAttributeError(RuntimeError): + """ + A class with ``auto_attribs=True`` has a field without a type annotation. + + .. versionadded:: 17.3.0 + """ + + +class PythonTooOldError(RuntimeError): + """ + It was attempted to use an *attrs* feature that requires a newer Python + version. + + .. versionadded:: 18.2.0 + """ + + +class NotCallableError(TypeError): + """ + A field requiring a callable has been set with a value that is not + callable. + + .. versionadded:: 19.2.0 + """ + + def __init__(self, msg, value): + super(TypeError, self).__init__(msg, value) + self.msg = msg + self.value = value + + def __str__(self): + return str(self.msg) diff --git a/py_modules/attr/exceptions.pyi b/py_modules/attr/exceptions.pyi new file mode 100644 index 0000000..f268011 --- /dev/null +++ b/py_modules/attr/exceptions.pyi @@ -0,0 +1,17 @@ +from typing import Any + +class FrozenError(AttributeError): + msg: str = ... + +class FrozenInstanceError(FrozenError): ... +class FrozenAttributeError(FrozenError): ... +class AttrsAttributeNotFoundError(ValueError): ... +class NotAnAttrsClassError(ValueError): ... +class DefaultAlreadySetError(RuntimeError): ... +class UnannotatedAttributeError(RuntimeError): ... +class PythonTooOldError(RuntimeError): ... + +class NotCallableError(TypeError): + msg: str = ... + value: Any = ... + def __init__(self, msg: str, value: Any) -> None: ... diff --git a/py_modules/attr/filters.py b/py_modules/attr/filters.py new file mode 100644 index 0000000..689b170 --- /dev/null +++ b/py_modules/attr/filters.py @@ -0,0 +1,72 @@ +# SPDX-License-Identifier: MIT + +""" +Commonly useful filters for `attrs.asdict` and `attrs.astuple`. +""" + +from ._make import Attribute + + +def _split_what(what): + """ + Returns a tuple of `frozenset`s of classes and attributes. + """ + return ( + frozenset(cls for cls in what if isinstance(cls, type)), + frozenset(cls for cls in what if isinstance(cls, str)), + frozenset(cls for cls in what if isinstance(cls, Attribute)), + ) + + +def include(*what): + """ + Create a filter that only allows *what*. + + Args: + what (list[type, str, attrs.Attribute]): + What to include. Can be a type, a name, or an attribute. + + Returns: + Callable: + A callable that can be passed to `attrs.asdict`'s and + `attrs.astuple`'s *filter* argument. + + .. versionchanged:: 23.1.0 Accept strings with field names. + """ + cls, names, attrs = _split_what(what) + + def include_(attribute, value): + return ( + value.__class__ in cls + or attribute.name in names + or attribute in attrs + ) + + return include_ + + +def exclude(*what): + """ + Create a filter that does **not** allow *what*. + + Args: + what (list[type, str, attrs.Attribute]): + What to exclude. Can be a type, a name, or an attribute. + + Returns: + Callable: + A callable that can be passed to `attrs.asdict`'s and + `attrs.astuple`'s *filter* argument. + + .. versionchanged:: 23.3.0 Accept field name string as input argument + """ + cls, names, attrs = _split_what(what) + + def exclude_(attribute, value): + return not ( + value.__class__ in cls + or attribute.name in names + or attribute in attrs + ) + + return exclude_ diff --git a/py_modules/attr/filters.pyi b/py_modules/attr/filters.pyi new file mode 100644 index 0000000..974abdc --- /dev/null +++ b/py_modules/attr/filters.pyi @@ -0,0 +1,6 @@ +from typing import Any + +from . import Attribute, _FilterType + +def include(*what: type | str | Attribute[Any]) -> _FilterType[Any]: ... +def exclude(*what: type | str | Attribute[Any]) -> _FilterType[Any]: ... diff --git a/py_modules/attr/py.typed b/py_modules/attr/py.typed new file mode 100644 index 0000000..e69de29 diff --git a/py_modules/attr/setters.py b/py_modules/attr/setters.py new file mode 100644 index 0000000..78b0839 --- /dev/null +++ b/py_modules/attr/setters.py @@ -0,0 +1,79 @@ +# SPDX-License-Identifier: MIT + +""" +Commonly used hooks for on_setattr. +""" + +from . import _config +from .exceptions import FrozenAttributeError + + +def pipe(*setters): + """ + Run all *setters* and return the return value of the last one. + + .. versionadded:: 20.1.0 + """ + + def wrapped_pipe(instance, attrib, new_value): + rv = new_value + + for setter in setters: + rv = setter(instance, attrib, rv) + + return rv + + return wrapped_pipe + + +def frozen(_, __, ___): + """ + Prevent an attribute to be modified. + + .. versionadded:: 20.1.0 + """ + raise FrozenAttributeError + + +def validate(instance, attrib, new_value): + """ + Run *attrib*'s validator on *new_value* if it has one. + + .. versionadded:: 20.1.0 + """ + if _config._run_validators is False: + return new_value + + v = attrib.validator + if not v: + return new_value + + v(instance, attrib, new_value) + + return new_value + + +def convert(instance, attrib, new_value): + """ + Run *attrib*'s converter -- if it has one -- on *new_value* and return the + result. + + .. versionadded:: 20.1.0 + """ + c = attrib.converter + if c: + # This can be removed once we drop 3.8 and use attrs.Converter instead. + from ._make import Converter + + if not isinstance(c, Converter): + return c(new_value) + + return c(new_value, instance, attrib) + + return new_value + + +# Sentinel for disabling class-wide *on_setattr* hooks for certain attributes. +# Sphinx's autodata stopped working, so the docstring is inlined in the API +# docs. +NO_OP = object() diff --git a/py_modules/attr/setters.pyi b/py_modules/attr/setters.pyi new file mode 100644 index 0000000..73abf36 --- /dev/null +++ b/py_modules/attr/setters.pyi @@ -0,0 +1,20 @@ +from typing import Any, NewType, NoReturn, TypeVar + +from . import Attribute +from attrs import _OnSetAttrType + +_T = TypeVar("_T") + +def frozen( + instance: Any, attribute: Attribute[Any], new_value: Any +) -> NoReturn: ... +def pipe(*setters: _OnSetAttrType) -> _OnSetAttrType: ... +def validate(instance: Any, attribute: Attribute[_T], new_value: _T) -> _T: ... + +# convert is allowed to return Any, because they can be chained using pipe. +def convert( + instance: Any, attribute: Attribute[Any], new_value: Any +) -> Any: ... + +_NoOpType = NewType("_NoOpType", object) +NO_OP: _NoOpType diff --git a/py_modules/attr/validators.py b/py_modules/attr/validators.py new file mode 100644 index 0000000..0b1a294 --- /dev/null +++ b/py_modules/attr/validators.py @@ -0,0 +1,750 @@ +# SPDX-License-Identifier: MIT + +""" +Commonly useful validators. +""" + +import operator +import re + +from contextlib import contextmanager +from re import Pattern + +from ._config import get_run_validators, set_run_validators +from ._make import _AndValidator, and_, attrib, attrs +from .converters import default_if_none +from .exceptions import NotCallableError + + +__all__ = [ + "and_", + "deep_iterable", + "deep_mapping", + "disabled", + "ge", + "get_disabled", + "gt", + "in_", + "instance_of", + "is_callable", + "le", + "lt", + "matches_re", + "max_len", + "min_len", + "not_", + "optional", + "or_", + "set_disabled", +] + + +def set_disabled(disabled): + """ + Globally disable or enable running validators. + + By default, they are run. + + Args: + disabled (bool): If `True`, disable running all validators. + + .. warning:: + + This function is not thread-safe! + + .. versionadded:: 21.3.0 + """ + set_run_validators(not disabled) + + +def get_disabled(): + """ + Return a bool indicating whether validators are currently disabled or not. + + Returns: + bool:`True` if validators are currently disabled. + + .. versionadded:: 21.3.0 + """ + return not get_run_validators() + + +@contextmanager +def disabled(): + """ + Context manager that disables running validators within its context. + + .. warning:: + + This context manager is not thread-safe! + + .. versionadded:: 21.3.0 + .. versionchanged:: 26.1.0 The contextmanager is nestable. + """ + prev = get_run_validators() + set_run_validators(False) + try: + yield + finally: + set_run_validators(prev) + + +@attrs(repr=False, slots=True, unsafe_hash=True) +class _InstanceOfValidator: + type = attrib() + + def __call__(self, inst, attr, value): + """ + We use a callable class to be able to change the ``__repr__``. + """ + if not isinstance(value, self.type): + msg = f"'{attr.name}' must be {self.type!r} (got {value!r} that is a {value.__class__!r})." + raise TypeError( + msg, + attr, + self.type, + value, + ) + + def __repr__(self): + return f"" + + +def instance_of(type): + """ + A validator that raises a `TypeError` if the initializer is called with a + wrong type for this particular attribute (checks are performed using + `isinstance` therefore it's also valid to pass a tuple of types). + + Args: + type (type | tuple[type]): The type to check for. + + Raises: + TypeError: + With a human readable error message, the attribute (of type + `attrs.Attribute`), the expected type, and the value it got. + """ + return _InstanceOfValidator(type) + + +@attrs(repr=False, frozen=True, slots=True) +class _MatchesReValidator: + pattern = attrib() + match_func = attrib() + + def __call__(self, inst, attr, value): + """ + We use a callable class to be able to change the ``__repr__``. + """ + if not self.match_func(value): + msg = f"'{attr.name}' must match regex {self.pattern.pattern!r} ({value!r} doesn't)" + raise ValueError( + msg, + attr, + self.pattern, + value, + ) + + def __repr__(self): + return f"" + + +def matches_re(regex, flags=0, func=None): + r""" + A validator that raises `ValueError` if the initializer is called with a + string that doesn't match *regex*. + + Args: + regex (str, re.Pattern): + A regex string or precompiled pattern to match against + + flags (int): + Flags that will be passed to the underlying re function (default 0) + + func (typing.Callable): + Which underlying `re` function to call. Valid options are + `re.fullmatch`, `re.search`, and `re.match`; the default `None` + means `re.fullmatch`. For performance reasons, the pattern is + always precompiled using `re.compile`. + + .. versionadded:: 19.2.0 + .. versionchanged:: 21.3.0 *regex* can be a pre-compiled pattern. + """ + valid_funcs = (re.fullmatch, None, re.search, re.match) + if func not in valid_funcs: + msg = "'func' must be one of {}.".format( + ", ".join( + sorted((e and e.__name__) or "None" for e in set(valid_funcs)) + ) + ) + raise ValueError(msg) + + if isinstance(regex, Pattern): + if flags: + msg = "'flags' can only be used with a string pattern; pass flags to re.compile() instead" + raise TypeError(msg) + pattern = regex + else: + pattern = re.compile(regex, flags) + + if func is re.match: + match_func = pattern.match + elif func is re.search: + match_func = pattern.search + else: + match_func = pattern.fullmatch + + return _MatchesReValidator(pattern, match_func) + + +@attrs(repr=False, slots=True, unsafe_hash=True) +class _OptionalValidator: + validator = attrib() + + def __call__(self, inst, attr, value): + if value is None: + return + + self.validator(inst, attr, value) + + def __repr__(self): + return f"" + + +def optional(validator): + """ + A validator that makes an attribute optional. An optional attribute is one + which can be set to `None` in addition to satisfying the requirements of + the sub-validator. + + Args: + validator + (typing.Callable | tuple[typing.Callable] | list[typing.Callable]): + A validator (or validators) that is used for non-`None` values. + + .. versionadded:: 15.1.0 + .. versionchanged:: 17.1.0 *validator* can be a list of validators. + .. versionchanged:: 23.1.0 *validator* can also be a tuple of validators. + """ + if isinstance(validator, (list, tuple)): + return _OptionalValidator(_AndValidator(validator)) + + return _OptionalValidator(validator) + + +@attrs(repr=False, slots=True, unsafe_hash=True) +class _InValidator: + options = attrib() + _original_options = attrib(hash=False) + + def __call__(self, inst, attr, value): + try: + in_options = value in self.options + except TypeError: # e.g. `1 in "abc"` + in_options = False + + if not in_options: + msg = f"'{attr.name}' must be in {self._original_options!r} (got {value!r})" + raise ValueError( + msg, + attr, + self._original_options, + value, + ) + + def __repr__(self): + return f"" + + +def in_(options): + """ + A validator that raises a `ValueError` if the initializer is called with a + value that does not belong in the *options* provided. + + The check is performed using ``value in options``, so *options* has to + support that operation. + + To keep the validator hashable, dicts, lists, and sets are transparently + transformed into a `tuple`. + + Args: + options: Allowed options. + + Raises: + ValueError: + With a human readable error message, the attribute (of type + `attrs.Attribute`), the expected options, and the value it got. + + .. versionadded:: 17.1.0 + .. versionchanged:: 22.1.0 + The ValueError was incomplete until now and only contained the human + readable error message. Now it contains all the information that has + been promised since 17.1.0. + .. versionchanged:: 24.1.0 + *options* that are a list, dict, or a set are now transformed into a + tuple to keep the validator hashable. + """ + repr_options = options + if isinstance(options, (list, dict, set)): + options = tuple(options) + + return _InValidator(options, repr_options) + + +@attrs(repr=False, slots=False, unsafe_hash=True) +class _IsCallableValidator: + def __call__(self, inst, attr, value): + """ + We use a callable class to be able to change the ``__repr__``. + """ + if not callable(value): + message = ( + "'{name}' must be callable " + "(got {value!r} that is a {actual!r})." + ) + raise NotCallableError( + msg=message.format( + name=attr.name, value=value, actual=value.__class__ + ), + value=value, + ) + + def __repr__(self): + return "" + + +def is_callable(): + """ + A validator that raises a `attrs.exceptions.NotCallableError` if the + initializer is called with a value for this particular attribute that is + not callable. + + .. versionadded:: 19.1.0 + + Raises: + attrs.exceptions.NotCallableError: + With a human readable error message containing the attribute + (`attrs.Attribute`) name, and the value it got. + """ + return _IsCallableValidator() + + +@attrs(repr=False, slots=True, unsafe_hash=True) +class _DeepIterable: + member_validator = attrib(validator=is_callable()) + iterable_validator = attrib( + default=None, validator=optional(is_callable()) + ) + + def __call__(self, inst, attr, value): + """ + We use a callable class to be able to change the ``__repr__``. + """ + if self.iterable_validator is not None: + self.iterable_validator(inst, attr, value) + + for member in value: + self.member_validator(inst, attr, member) + + def __repr__(self): + iterable_identifier = ( + "" + if self.iterable_validator is None + else f" {self.iterable_validator!r}" + ) + return ( + f"" + ) + + +def deep_iterable(member_validator, iterable_validator=None): + """ + A validator that performs deep validation of an iterable. + + Args: + member_validator: Validator(s) to apply to iterable members. + + iterable_validator: + Validator(s) to apply to iterable itself (optional). + + Raises + TypeError: if any sub-validators fail + + .. versionadded:: 19.1.0 + + .. versionchanged:: 25.4.0 + *member_validator* and *iterable_validator* can now be a list or tuple + of validators. + """ + if isinstance(member_validator, (list, tuple)): + member_validator = and_(*member_validator) + if isinstance(iterable_validator, (list, tuple)): + iterable_validator = and_(*iterable_validator) + return _DeepIterable(member_validator, iterable_validator) + + +@attrs(repr=False, slots=True, unsafe_hash=True) +class _DeepMapping: + key_validator = attrib(validator=optional(is_callable())) + value_validator = attrib(validator=optional(is_callable())) + mapping_validator = attrib(validator=optional(is_callable())) + + def __call__(self, inst, attr, value): + """ + We use a callable class to be able to change the ``__repr__``. + """ + if self.mapping_validator is not None: + self.mapping_validator(inst, attr, value) + + for key in value: + if self.key_validator is not None: + self.key_validator(inst, attr, key) + if self.value_validator is not None: + self.value_validator(inst, attr, value[key]) + + def __repr__(self): + return f"" + + +def deep_mapping( + key_validator=None, value_validator=None, mapping_validator=None +): + """ + A validator that performs deep validation of a dictionary. + + All validators are optional, but at least one of *key_validator* or + *value_validator* must be provided. + + Args: + key_validator: Validator(s) to apply to dictionary keys. + + value_validator: Validator(s) to apply to dictionary values. + + mapping_validator: + Validator(s) to apply to top-level mapping attribute. + + .. versionadded:: 19.1.0 + + .. versionchanged:: 25.4.0 + *key_validator* and *value_validator* are now optional, but at least one + of them must be provided. + + .. versionchanged:: 25.4.0 + *key_validator*, *value_validator*, and *mapping_validator* can now be a + list or tuple of validators. + + Raises: + TypeError: If any sub-validator fails on validation. + + ValueError: + If neither *key_validator* nor *value_validator* is provided on + instantiation. + """ + if key_validator is None and value_validator is None: + msg = ( + "At least one of key_validator or value_validator must be provided" + ) + raise ValueError(msg) + + if isinstance(key_validator, (list, tuple)): + key_validator = and_(*key_validator) + if isinstance(value_validator, (list, tuple)): + value_validator = and_(*value_validator) + if isinstance(mapping_validator, (list, tuple)): + mapping_validator = and_(*mapping_validator) + + return _DeepMapping(key_validator, value_validator, mapping_validator) + + +@attrs(repr=False, frozen=True, slots=True) +class _NumberValidator: + bound = attrib() + compare_op = attrib() + compare_func = attrib() + + def __call__(self, inst, attr, value): + """ + We use a callable class to be able to change the ``__repr__``. + """ + if not self.compare_func(value, self.bound): + msg = f"'{attr.name}' must be {self.compare_op} {self.bound}: {value}" + raise ValueError(msg) + + def __repr__(self): + return f"" + + +def lt(val): + """ + A validator that raises `ValueError` if the initializer is called with a + number larger or equal to *val*. + + The validator uses `operator.lt` to compare the values. + + Args: + val: Exclusive upper bound for values. + + .. versionadded:: 21.3.0 + """ + return _NumberValidator(val, "<", operator.lt) + + +def le(val): + """ + A validator that raises `ValueError` if the initializer is called with a + number greater than *val*. + + The validator uses `operator.le` to compare the values. + + Args: + val: Inclusive upper bound for values. + + .. versionadded:: 21.3.0 + """ + return _NumberValidator(val, "<=", operator.le) + + +def ge(val): + """ + A validator that raises `ValueError` if the initializer is called with a + number smaller than *val*. + + The validator uses `operator.ge` to compare the values. + + Args: + val: Inclusive lower bound for values + + .. versionadded:: 21.3.0 + """ + return _NumberValidator(val, ">=", operator.ge) + + +def gt(val): + """ + A validator that raises `ValueError` if the initializer is called with a + number smaller or equal to *val*. + + The validator uses `operator.gt` to compare the values. + + Args: + val: Exclusive lower bound for values + + .. versionadded:: 21.3.0 + """ + return _NumberValidator(val, ">", operator.gt) + + +@attrs(repr=False, frozen=True, slots=True) +class _MaxLengthValidator: + max_length = attrib() + + def __call__(self, inst, attr, value): + """ + We use a callable class to be able to change the ``__repr__``. + """ + if len(value) > self.max_length: + msg = f"Length of '{attr.name}' must be <= {self.max_length}: {len(value)}" + raise ValueError(msg) + + def __repr__(self): + return f"" + + +def max_len(length): + """ + A validator that raises `ValueError` if the initializer is called + with a string or iterable that is longer than *length*. + + Args: + length (int): Maximum length of the string or iterable + + .. versionadded:: 21.3.0 + """ + return _MaxLengthValidator(length) + + +@attrs(repr=False, frozen=True, slots=True) +class _MinLengthValidator: + min_length = attrib() + + def __call__(self, inst, attr, value): + """ + We use a callable class to be able to change the ``__repr__``. + """ + if len(value) < self.min_length: + msg = f"Length of '{attr.name}' must be >= {self.min_length}: {len(value)}" + raise ValueError(msg) + + def __repr__(self): + return f"" + + +def min_len(length): + """ + A validator that raises `ValueError` if the initializer is called + with a string or iterable that is shorter than *length*. + + Args: + length (int): Minimum length of the string or iterable + + .. versionadded:: 22.1.0 + """ + return _MinLengthValidator(length) + + +@attrs(repr=False, slots=True, unsafe_hash=True) +class _SubclassOfValidator: + type = attrib() + + def __call__(self, inst, attr, value): + """ + We use a callable class to be able to change the ``__repr__``. + """ + if not issubclass(value, self.type): + msg = f"'{attr.name}' must be a subclass of {self.type!r} (got {value!r})." + raise TypeError( + msg, + attr, + self.type, + value, + ) + + def __repr__(self): + return f"" + + +def _subclass_of(type): + """ + A validator that raises a `TypeError` if the initializer is called with a + wrong type for this particular attribute (checks are performed using + `issubclass` therefore it's also valid to pass a tuple of types). + + Args: + type (type | tuple[type, ...]): The type(s) to check for. + + Raises: + TypeError: + With a human readable error message, the attribute (of type + `attrs.Attribute`), the expected type, and the value it got. + """ + return _SubclassOfValidator(type) + + +@attrs(repr=False, slots=True, unsafe_hash=True) +class _NotValidator: + validator = attrib() + msg = attrib( + converter=default_if_none( + "not_ validator child '{validator!r}' " + "did not raise a captured error" + ) + ) + exc_types = attrib( + validator=deep_iterable( + member_validator=_subclass_of(Exception), + iterable_validator=instance_of(tuple), + ), + ) + + def __call__(self, inst, attr, value): + try: + self.validator(inst, attr, value) + except self.exc_types: + pass # suppress error to invert validity + else: + raise ValueError( + self.msg.format( + validator=self.validator, + exc_types=self.exc_types, + ), + attr, + self.validator, + value, + self.exc_types, + ) + + def __repr__(self): + return f"" + + +def not_(validator, *, msg=None, exc_types=(ValueError, TypeError)): + """ + A validator that wraps and logically 'inverts' the validator passed to it. + It will raise a `ValueError` if the provided validator *doesn't* raise a + `ValueError` or `TypeError` (by default), and will suppress the exception + if the provided validator *does*. + + Intended to be used with existing validators to compose logic without + needing to create inverted variants, for example, ``not_(in_(...))``. + + Args: + validator: A validator to be logically inverted. + + msg (str): + Message to raise if validator fails. Formatted with keys + ``exc_types`` and ``validator``. + + exc_types (tuple[type, ...]): + Exception type(s) to capture. Other types raised by child + validators will not be intercepted and pass through. + + Raises: + ValueError: + With a human readable error message, the attribute (of type + `attrs.Attribute`), the validator that failed to raise an + exception, the value it got, and the expected exception types. + + .. versionadded:: 22.2.0 + """ + try: + exc_types = tuple(exc_types) + except TypeError: + exc_types = (exc_types,) + return _NotValidator(validator, msg, exc_types) + + +@attrs(repr=False, slots=True, unsafe_hash=True) +class _OrValidator: + validators = attrib() + + def __call__(self, inst, attr, value): + for v in self.validators: + try: + v(inst, attr, value) + except Exception: # noqa: BLE001, PERF203, S112 + continue + else: + return + + msg = f"None of {self.validators!r} satisfied for value {value!r}" + raise ValueError(msg) + + def __repr__(self): + return f"" + + +def or_(*validators): + """ + A validator that composes multiple validators into one. + + When called on a value, it runs all wrapped validators until one of them is + satisfied. + + Args: + validators (~collections.abc.Iterable[typing.Callable]): + Arbitrary number of validators. + + Raises: + ValueError: + If no validator is satisfied. Raised with a human-readable error + message listing all the wrapped validators and the value that + failed all of them. + + .. versionadded:: 24.1.0 + """ + vals = [] + for v in validators: + vals.extend(v.validators if isinstance(v, _OrValidator) else [v]) + + return _OrValidator(tuple(vals)) diff --git a/py_modules/attr/validators.pyi b/py_modules/attr/validators.pyi new file mode 100644 index 0000000..18fb112 --- /dev/null +++ b/py_modules/attr/validators.pyi @@ -0,0 +1,140 @@ +from types import UnionType +from typing import ( + Any, + AnyStr, + Callable, + Container, + ContextManager, + Iterable, + Mapping, + Match, + Pattern, + TypeVar, + overload, +) + +from attrs import _ValidatorType +from attrs import _ValidatorArgType + +_T = TypeVar("_T") +_T1 = TypeVar("_T1") +_T2 = TypeVar("_T2") +_T3 = TypeVar("_T3") +_T4 = TypeVar("_T4") +_T5 = TypeVar("_T5") +_T6 = TypeVar("_T6") +_I = TypeVar("_I", bound=Iterable) +_K = TypeVar("_K") +_V = TypeVar("_V") +_M = TypeVar("_M", bound=Mapping) + +def set_disabled(run: bool) -> None: ... +def get_disabled() -> bool: ... +def disabled() -> ContextManager[None]: ... + +# To be more precise on instance_of use some overloads. +# If there are more than 3 items in the tuple then we fall back to Any +@overload +def instance_of(type: type[_T]) -> _ValidatorType[_T]: ... +@overload +def instance_of(type: tuple[type[_T]]) -> _ValidatorType[_T]: ... +@overload +def instance_of( + type: tuple[type[_T1], type[_T2]], +) -> _ValidatorType[_T1 | _T2]: ... +@overload +def instance_of( + type: tuple[type[_T1], type[_T2], type[_T3]], +) -> _ValidatorType[_T1 | _T2 | _T3]: ... +@overload +def instance_of(type: tuple[type, ...]) -> _ValidatorType[Any]: ... +@overload +def instance_of(type: UnionType) -> _ValidatorType[Any]: ... +def optional( + validator: ( + _ValidatorType[_T] + | list[_ValidatorType[_T]] + | tuple[_ValidatorType[_T], ...] + ), +) -> _ValidatorType[_T | None]: ... +def in_(options: Container[_T]) -> _ValidatorType[_T]: ... +def and_(*validators: _ValidatorType[_T]) -> _ValidatorType[_T]: ... +def matches_re( + regex: Pattern[AnyStr] | AnyStr, + flags: int = ..., + func: Callable[[AnyStr, AnyStr, int], Match[AnyStr] | None] | None = ..., +) -> _ValidatorType[AnyStr]: ... +def deep_iterable( + member_validator: _ValidatorArgType[_T], + iterable_validator: _ValidatorArgType[_I] | None = ..., +) -> _ValidatorType[_I]: ... +@overload +def deep_mapping( + key_validator: _ValidatorArgType[_K], + value_validator: _ValidatorArgType[_V] | None = ..., + mapping_validator: _ValidatorArgType[_M] | None = ..., +) -> _ValidatorType[_M]: ... +@overload +def deep_mapping( + key_validator: _ValidatorArgType[_K] | None = ..., + value_validator: _ValidatorArgType[_V] = ..., + mapping_validator: _ValidatorArgType[_M] | None = ..., +) -> _ValidatorType[_M]: ... +def is_callable() -> _ValidatorType[_T]: ... +def lt(val: _T) -> _ValidatorType[_T]: ... +def le(val: _T) -> _ValidatorType[_T]: ... +def ge(val: _T) -> _ValidatorType[_T]: ... +def gt(val: _T) -> _ValidatorType[_T]: ... +def max_len(length: int) -> _ValidatorType[_T]: ... +def min_len(length: int) -> _ValidatorType[_T]: ... +def not_( + validator: _ValidatorType[_T], + *, + msg: str | None = None, + exc_types: type[Exception] | Iterable[type[Exception]] = ..., +) -> _ValidatorType[_T]: ... +@overload +def or_( + __v1: _ValidatorType[_T1], + __v2: _ValidatorType[_T2], +) -> _ValidatorType[_T1 | _T2]: ... +@overload +def or_( + __v1: _ValidatorType[_T1], + __v2: _ValidatorType[_T2], + __v3: _ValidatorType[_T3], +) -> _ValidatorType[_T1 | _T2 | _T3]: ... +@overload +def or_( + __v1: _ValidatorType[_T1], + __v2: _ValidatorType[_T2], + __v3: _ValidatorType[_T3], + __v4: _ValidatorType[_T4], +) -> _ValidatorType[_T1 | _T2 | _T3 | _T4]: ... +@overload +def or_( + __v1: _ValidatorType[_T1], + __v2: _ValidatorType[_T2], + __v3: _ValidatorType[_T3], + __v4: _ValidatorType[_T4], + __v5: _ValidatorType[_T5], +) -> _ValidatorType[_T1 | _T2 | _T3 | _T4 | _T5]: ... +@overload +def or_( + __v1: _ValidatorType[_T1], + __v2: _ValidatorType[_T2], + __v3: _ValidatorType[_T3], + __v4: _ValidatorType[_T4], + __v5: _ValidatorType[_T5], + __v6: _ValidatorType[_T6], +) -> _ValidatorType[_T1 | _T2 | _T3 | _T4 | _T5 | _T6]: ... +@overload +def or_( + __v1: _ValidatorType[Any], + __v2: _ValidatorType[Any], + __v3: _ValidatorType[Any], + __v4: _ValidatorType[Any], + __v5: _ValidatorType[Any], + __v6: _ValidatorType[Any], + *validators: _ValidatorType[Any], +) -> _ValidatorType[Any]: ... diff --git a/py_modules/attrs/__init__.py b/py_modules/attrs/__init__.py new file mode 100644 index 0000000..dc1ce4b --- /dev/null +++ b/py_modules/attrs/__init__.py @@ -0,0 +1,72 @@ +# SPDX-License-Identifier: MIT + +from attr import ( + NOTHING, + Attribute, + AttrsInstance, + Converter, + Factory, + NothingType, + _make_getattr, + assoc, + cmp_using, + define, + evolve, + field, + fields, + fields_dict, + frozen, + has, + make_class, + mutable, + resolve_types, + validate, +) +from attr._make import ClassProps +from attr._next_gen import asdict, astuple, inspect + +from . import converters, exceptions, filters, setters, validators + + +__all__ = [ + "NOTHING", + "Attribute", + "AttrsInstance", + "ClassProps", + "Converter", + "Factory", + "NothingType", + "__author__", + "__copyright__", + "__description__", + "__doc__", + "__email__", + "__license__", + "__title__", + "__url__", + "__version__", + "__version_info__", + "asdict", + "assoc", + "astuple", + "cmp_using", + "converters", + "define", + "evolve", + "exceptions", + "field", + "fields", + "fields_dict", + "filters", + "frozen", + "has", + "inspect", + "make_class", + "mutable", + "resolve_types", + "setters", + "validate", + "validators", +] + +__getattr__ = _make_getattr(__name__) diff --git a/py_modules/attrs/__init__.pyi b/py_modules/attrs/__init__.pyi new file mode 100644 index 0000000..6364bac --- /dev/null +++ b/py_modules/attrs/__init__.pyi @@ -0,0 +1,314 @@ +import sys + +from typing import ( + Any, + Callable, + Mapping, + Sequence, + overload, + TypeVar, +) + +# Because we need to type our own stuff, we have to make everything from +# attr explicitly public too. +from attr import __author__ as __author__ +from attr import __copyright__ as __copyright__ +from attr import __description__ as __description__ +from attr import __email__ as __email__ +from attr import __license__ as __license__ +from attr import __title__ as __title__ +from attr import __url__ as __url__ +from attr import __version__ as __version__ +from attr import __version_info__ as __version_info__ +from attr import assoc as assoc +from attr import Attribute as Attribute +from attr import AttrsInstance as AttrsInstance +from attr import cmp_using as cmp_using +from attr import converters as converters +from attr import Converter as Converter +from attr import evolve as evolve +from attr import exceptions as exceptions +from attr import Factory as Factory +from attr import fields as fields +from attr import fields_dict as fields_dict +from attr import filters as filters +from attr import has as has +from attr import make_class as make_class +from attr import NOTHING as NOTHING +from attr import resolve_types as resolve_types +from attr import setters as setters +from attr import validate as validate +from attr import validators as validators +from attr import attrib, asdict as asdict, astuple as astuple +from attr import NothingType as NothingType + +if sys.version_info >= (3, 11): + from typing import dataclass_transform +else: + from typing_extensions import dataclass_transform + +_T = TypeVar("_T") +_C = TypeVar("_C", bound=type) + +_EqOrderType = bool | Callable[[Any], Any] +_ValidatorType = Callable[[Any, "Attribute[_T]", _T], Any] +_CallableConverterType = Callable[[Any], Any] +_ConverterType = _CallableConverterType | Converter[Any, Any] +_ReprType = Callable[[Any], str] +_ReprArgType = bool | _ReprType +_OnSetAttrType = Callable[[Any, "Attribute[Any]", Any], Any] +_OnSetAttrArgType = _OnSetAttrType | list[_OnSetAttrType] | setters._NoOpType +_FieldTransformer = Callable[ + [type, list["Attribute[Any]"]], list["Attribute[Any]"] +] +# FIXME: in reality, if multiple validators are passed they must be in a list +# or tuple, but those are invariant and so would prevent subtypes of +# _ValidatorType from working when passed in a list or tuple. +_ValidatorArgType = _ValidatorType[_T] | Sequence[_ValidatorType[_T]] + +@overload +def field( + *, + default: None = ..., + validator: None = ..., + repr: _ReprArgType = ..., + hash: bool | None = ..., + init: bool = ..., + metadata: Mapping[Any, Any] | None = ..., + converter: None = ..., + factory: None = ..., + kw_only: bool | None = ..., + eq: bool | None = ..., + order: bool | None = ..., + on_setattr: _OnSetAttrArgType | None = ..., + alias: str | None = ..., + type: type | None = ..., +) -> Any: ... + +# This form catches an explicit None or no default and infers the type from the +# other arguments. +@overload +def field( + *, + default: None = ..., + validator: _ValidatorArgType[_T] | None = ..., + repr: _ReprArgType = ..., + hash: bool | None = ..., + init: bool = ..., + metadata: Mapping[Any, Any] | None = ..., + converter: _ConverterType + | list[_ConverterType] + | tuple[_ConverterType, ...] + | None = ..., + factory: Callable[[], _T] | None = ..., + kw_only: bool | None = ..., + eq: _EqOrderType | None = ..., + order: _EqOrderType | None = ..., + on_setattr: _OnSetAttrArgType | None = ..., + alias: str | None = ..., + type: type | None = ..., +) -> _T: ... + +# This form catches an explicit default argument. +@overload +def field( + *, + default: _T, + validator: _ValidatorArgType[_T] | None = ..., + repr: _ReprArgType = ..., + hash: bool | None = ..., + init: bool = ..., + metadata: Mapping[Any, Any] | None = ..., + converter: _ConverterType + | list[_ConverterType] + | tuple[_ConverterType, ...] + | None = ..., + factory: Callable[[], _T] | None = ..., + kw_only: bool | None = ..., + eq: _EqOrderType | None = ..., + order: _EqOrderType | None = ..., + on_setattr: _OnSetAttrArgType | None = ..., + alias: str | None = ..., + type: type | None = ..., +) -> _T: ... + +# This form covers type=non-Type: e.g. forward references (str), Any +@overload +def field( + *, + default: _T | None = ..., + validator: _ValidatorArgType[_T] | None = ..., + repr: _ReprArgType = ..., + hash: bool | None = ..., + init: bool = ..., + metadata: Mapping[Any, Any] | None = ..., + converter: _ConverterType + | list[_ConverterType] + | tuple[_ConverterType, ...] + | None = ..., + factory: Callable[[], _T] | None = ..., + kw_only: bool | None = ..., + eq: _EqOrderType | None = ..., + order: _EqOrderType | None = ..., + on_setattr: _OnSetAttrArgType | None = ..., + alias: str | None = ..., + type: type | None = ..., +) -> Any: ... +@overload +@dataclass_transform(field_specifiers=(attrib, field)) +def define( + maybe_cls: _C, + *, + these: dict[str, Any] | None = ..., + repr: bool = ..., + unsafe_hash: bool | None = ..., + hash: bool | None = ..., + init: bool = ..., + slots: bool = ..., + frozen: bool = ..., + weakref_slot: bool = ..., + str: bool = ..., + auto_attribs: bool = ..., + kw_only: bool = ..., + cache_hash: bool = ..., + auto_exc: bool = ..., + eq: bool | None = ..., + order: bool | None = ..., + auto_detect: bool = ..., + getstate_setstate: bool | None = ..., + on_setattr: _OnSetAttrArgType | None = ..., + field_transformer: _FieldTransformer | None = ..., + match_args: bool = ..., +) -> _C: ... +@overload +@dataclass_transform(field_specifiers=(attrib, field)) +def define( + maybe_cls: None = ..., + *, + these: dict[str, Any] | None = ..., + repr: bool = ..., + unsafe_hash: bool | None = ..., + hash: bool | None = ..., + init: bool = ..., + slots: bool = ..., + frozen: bool = ..., + weakref_slot: bool = ..., + str: bool = ..., + auto_attribs: bool = ..., + kw_only: bool = ..., + cache_hash: bool = ..., + auto_exc: bool = ..., + eq: bool | None = ..., + order: bool | None = ..., + auto_detect: bool = ..., + getstate_setstate: bool | None = ..., + on_setattr: _OnSetAttrArgType | None = ..., + field_transformer: _FieldTransformer | None = ..., + match_args: bool = ..., +) -> Callable[[_C], _C]: ... + +mutable = define + +@overload +@dataclass_transform(frozen_default=True, field_specifiers=(attrib, field)) +def frozen( + maybe_cls: _C, + *, + these: dict[str, Any] | None = ..., + repr: bool = ..., + unsafe_hash: bool | None = ..., + hash: bool | None = ..., + init: bool = ..., + slots: bool = ..., + frozen: bool = ..., + weakref_slot: bool = ..., + str: bool = ..., + auto_attribs: bool = ..., + kw_only: bool = ..., + cache_hash: bool = ..., + auto_exc: bool = ..., + eq: bool | None = ..., + order: bool | None = ..., + auto_detect: bool = ..., + getstate_setstate: bool | None = ..., + on_setattr: _OnSetAttrArgType | None = ..., + field_transformer: _FieldTransformer | None = ..., + match_args: bool = ..., +) -> _C: ... +@overload +@dataclass_transform(frozen_default=True, field_specifiers=(attrib, field)) +def frozen( + maybe_cls: None = ..., + *, + these: dict[str, Any] | None = ..., + repr: bool = ..., + unsafe_hash: bool | None = ..., + hash: bool | None = ..., + init: bool = ..., + slots: bool = ..., + frozen: bool = ..., + weakref_slot: bool = ..., + str: bool = ..., + auto_attribs: bool = ..., + kw_only: bool = ..., + cache_hash: bool = ..., + auto_exc: bool = ..., + eq: bool | None = ..., + order: bool | None = ..., + auto_detect: bool = ..., + getstate_setstate: bool | None = ..., + on_setattr: _OnSetAttrArgType | None = ..., + field_transformer: _FieldTransformer | None = ..., + match_args: bool = ..., +) -> Callable[[_C], _C]: ... + +class ClassProps: + # XXX: somehow when defining/using enums Mypy starts looking at our own + # (untyped) code and causes tons of errors. + Hashability: Any + KeywordOnly: Any + + is_exception: bool + is_slotted: bool + has_weakref_slot: bool + is_frozen: bool + # kw_only: ClassProps.KeywordOnly + kw_only: Any + collected_fields_by_mro: bool + added_init: bool + added_repr: bool + added_eq: bool + added_ordering: bool + # hashability: ClassProps.Hashability + hashability: Any + added_match_args: bool + added_str: bool + added_pickling: bool + on_setattr_hook: _OnSetAttrType | None + field_transformer: Callable[[Attribute[Any]], Attribute[Any]] | None + + def __init__( + self, + is_exception: bool, + is_slotted: bool, + has_weakref_slot: bool, + is_frozen: bool, + # kw_only: ClassProps.KeywordOnly + kw_only: Any, + collected_fields_by_mro: bool, + added_init: bool, + added_repr: bool, + added_eq: bool, + added_ordering: bool, + # hashability: ClassProps.Hashability + hashability: Any, + added_match_args: bool, + added_str: bool, + added_pickling: bool, + on_setattr_hook: _OnSetAttrType, + field_transformer: Callable[[Attribute[Any]], Attribute[Any]], + ) -> None: ... + @property + def is_hashable(self) -> bool: ... + +def inspect(cls: type) -> ClassProps: ... diff --git a/py_modules/attrs/converters.py b/py_modules/attrs/converters.py new file mode 100644 index 0000000..7821f6c --- /dev/null +++ b/py_modules/attrs/converters.py @@ -0,0 +1,3 @@ +# SPDX-License-Identifier: MIT + +from attr.converters import * # noqa: F403 diff --git a/py_modules/attrs/exceptions.py b/py_modules/attrs/exceptions.py new file mode 100644 index 0000000..3323f9d --- /dev/null +++ b/py_modules/attrs/exceptions.py @@ -0,0 +1,3 @@ +# SPDX-License-Identifier: MIT + +from attr.exceptions import * # noqa: F403 diff --git a/py_modules/attrs/filters.py b/py_modules/attrs/filters.py new file mode 100644 index 0000000..3080f48 --- /dev/null +++ b/py_modules/attrs/filters.py @@ -0,0 +1,3 @@ +# SPDX-License-Identifier: MIT + +from attr.filters import * # noqa: F403 diff --git a/py_modules/attrs/py.typed b/py_modules/attrs/py.typed new file mode 100644 index 0000000..e69de29 diff --git a/py_modules/attrs/setters.py b/py_modules/attrs/setters.py new file mode 100644 index 0000000..f3d73bb --- /dev/null +++ b/py_modules/attrs/setters.py @@ -0,0 +1,3 @@ +# SPDX-License-Identifier: MIT + +from attr.setters import * # noqa: F403 diff --git a/py_modules/attrs/validators.py b/py_modules/attrs/validators.py new file mode 100644 index 0000000..037e124 --- /dev/null +++ b/py_modules/attrs/validators.py @@ -0,0 +1,3 @@ +# SPDX-License-Identifier: MIT + +from attr.validators import * # noqa: F403 diff --git a/py_modules/cffi/__init__.py b/py_modules/cffi/__init__.py new file mode 100644 index 0000000..93dcdb3 --- /dev/null +++ b/py_modules/cffi/__init__.py @@ -0,0 +1,14 @@ +__all__ = ['FFI', 'VerificationError', 'VerificationMissing', 'CDefError', + 'FFIError'] + +from .api import FFI +from .error import CDefError, FFIError, VerificationError, VerificationMissing +from .error import PkgConfigError + +__version__ = "2.1.1" +__version_info__ = (2, 1, 1) + +# The verifier module file names are based on the CRC32 of a string that +# contains the following version number. It may be older than __version__ +# if nothing is clearly incompatible. +__version_verifier_modules__ = "0.8.6" diff --git a/py_modules/cffi/_cffi_errors.h b/py_modules/cffi/_cffi_errors.h new file mode 100644 index 0000000..0931379 --- /dev/null +++ b/py_modules/cffi/_cffi_errors.h @@ -0,0 +1,135 @@ +#ifndef CFFI_MESSAGEBOX +# ifdef _MSC_VER +# define CFFI_MESSAGEBOX 1 +# else +# define CFFI_MESSAGEBOX 0 +# endif +#endif + + +#if CFFI_MESSAGEBOX +/* Windows only: logic to take the Python-CFFI embedding logic + initialization errors and display them in a background thread + with MessageBox. The idea is that if the whole program closes + as a result of this problem, then likely it is already a console + program and you can read the stderr output in the console too. + If it is not a console program, then it will likely show its own + dialog to complain, or generally not abruptly close, and for this + case the background thread should stay alive. +*/ +static void *volatile _cffi_bootstrap_text; + +static PyObject *_cffi_start_error_capture(void) +{ + PyObject *result = NULL; + PyObject *x, *m, *bi; + + if (InterlockedCompareExchangePointer(&_cffi_bootstrap_text, + (void *)1, NULL) != NULL) + return (PyObject *)1; + + m = PyImport_AddModule("_cffi_error_capture"); + if (m == NULL) + goto error; + + result = PyModule_GetDict(m); + if (result == NULL) + goto error; + + bi = PyImport_ImportModule("builtins"); + if (bi == NULL) + goto error; + PyDict_SetItemString(result, "__builtins__", bi); + Py_DECREF(bi); + + x = PyRun_String( + "import sys\n" + "class FileLike:\n" + " def write(self, x):\n" + " try:\n" + " of.write(x)\n" + " except: pass\n" + " self.buf += x\n" + " def flush(self):\n" + " pass\n" + "fl = FileLike()\n" + "fl.buf = ''\n" + "of = sys.stderr\n" + "sys.stderr = fl\n" + "def done():\n" + " sys.stderr = of\n" + " return fl.buf\n", /* make sure the returned value stays alive */ + Py_file_input, + result, result); + Py_XDECREF(x); + + error: + if (PyErr_Occurred()) + { + PyErr_WriteUnraisable(Py_None); + PyErr_Clear(); + } + return result; +} + +#pragma comment(lib, "user32.lib") + +static DWORD WINAPI _cffi_bootstrap_dialog(LPVOID ignored) +{ + Sleep(666); /* may be interrupted if the whole process is closing */ + MessageBoxW(NULL, (wchar_t *)_cffi_bootstrap_text, + L"Python-CFFI error", + MB_OK | MB_ICONERROR); + _cffi_bootstrap_text = NULL; + return 0; +} + +static void _cffi_stop_error_capture(PyObject *ecap) +{ + PyObject *s; + void *text; + + if (ecap == (PyObject *)1) + return; + + if (ecap == NULL) + goto error; + + s = PyRun_String("done()", Py_eval_input, ecap, ecap); + if (s == NULL) + goto error; + + /* Show a dialog box, but in a background thread, and + never show multiple dialog boxes at once. */ + text = PyUnicode_AsWideCharString(s, NULL); + + _cffi_bootstrap_text = text; + + if (text != NULL) + { + HANDLE h; + h = CreateThread(NULL, 0, _cffi_bootstrap_dialog, + NULL, 0, NULL); + if (h != NULL) + CloseHandle(h); + } + /* decref the string, but it should stay alive as 'fl.buf' + in the small module above. It will really be freed only if + we later get another similar error. So it's a leak of at + most one copy of the small module. That's fine for this + situation which is usually a "fatal error" anyway. */ + Py_DECREF(s); + PyErr_Clear(); + return; + + error: + _cffi_bootstrap_text = NULL; + PyErr_Clear(); +} + +#else + +static PyObject *_cffi_start_error_capture(void) { return NULL; } +static void _cffi_stop_error_capture(PyObject *ecap) { } + +#endif diff --git a/py_modules/cffi/_cffi_gen_src.py b/py_modules/cffi/_cffi_gen_src.py new file mode 100644 index 0000000..3d4fb7e --- /dev/null +++ b/py_modules/cffi/_cffi_gen_src.py @@ -0,0 +1,216 @@ +# Integrated from the cffi-buildtool project by Rose Davidson +# (https://github.com/inklesspen/cffi-buildtool), under the following +# license: +# +# MIT License +# +# Copyright (c) 2024, Rose Davidson +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, +# distribute, sublicense, and/or sell copies of the Software, and to +# permit persons to whom the Software is furnished to do so, subject to +# the following conditions: +# +# The above copyright notice and this permission notice (including the +# next paragraph) shall be included in all copies or substantial portions +# of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. +# IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY +# CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, +# TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE +# SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +"""Implementation of the ``cffi-gen-src`` command-line tool. + +This module is private; the command line is the only supported +interface. Two subcommands: + +``exec-python`` + Execute a Python script that constructs a :class:`cffi.FFI` + (the same kind of script that the CFFI docs' "Main mode of usage" + describes) and emit the generated C source. + +``read-sources`` + Create the :class:`cffi.FFI` from a separate ``cdef`` file and C + source prelude, then emit the generated C source. +""" + +import argparse +import io +import os +import sys + +from .api import FFI + + +def _execfile(pysrc, filename, globs): + compiled = compile(source=pysrc, filename=filename, mode='exec') + exec(compiled, globs, globs) + + +def find_ffi_in_python_script(pysrc, filename, ffivar): + """Execute ``pysrc`` and return the :class:`FFI` object it defines. + + The script is executed with ``__name__`` set to ``"cffi.gen_src"``, + so a trailing ``if __name__ == "__main__": ffibuilder.compile()`` + block in the script is skipped. + + ``ffivar`` is the name bound by the script to the :class:`FFI` + object, or to a callable that returns one. + + Raises :class:`NameError` if the name is not bound by the script, + or :class:`TypeError` if the name does not resolve to an + :class:`FFI` instance. + """ + filename = os.path.abspath(filename) + globs = {'__name__': 'cffi.gen_src', '__file__': filename} + old_path = sys.path[:] + sys.path.insert(0, os.path.dirname(filename)) + try: + _execfile(pysrc, filename, globs) + if ffivar not in globs: + raise NameError( + "Expected to find the FFI object with the name %r, " + "but it was not found." % (ffivar,) + ) + ffi = globs[ffivar] + if not isinstance(ffi, FFI) and callable(ffi): + # Maybe it's a callable that returns a FFI + ffi = ffi() + if not isinstance(ffi, FFI): + raise TypeError( + "Found an object with the name %r but it was not an " + "instance of cffi.api.FFI" % (ffivar,) + ) + return ffi + finally: + sys.path[:] = old_path + + +def make_ffi_from_sources(modulename, cdef, csrc): + """Create an :class:`FFI` from ``cdef`` text and a C source prelude.""" + ffibuilder = FFI() + ffibuilder.cdef(cdef) + ffibuilder.set_source(modulename, csrc) + return ffibuilder + + +def generate_c_source(ffi): + """Return the C source that :meth:`FFI.emit_c_code` would write.""" + output = io.StringIO() + ffi.emit_c_code(output) + return output.getvalue() + + +def write_c_source(output, generated): + if output == '-': + sys.stdout.write(generated) + return + with open(output, 'w', encoding='utf-8') as f: + f.write(generated) + + +def same_input_file(file1, file2): + if file1 is file2: + return True + try: + return os.path.samefile(file1.name, file2.name) + except OSError: + return False + + +def exec_python(*, output, pyfile, ffi_var): + with pyfile: + ffi = find_ffi_in_python_script(pyfile.read(), pyfile.name, ffi_var) + generated = generate_c_source(ffi) + write_c_source(output, generated) + + +def read_sources(*, output, module_name, cdef_input, csrc_input): + with csrc_input, cdef_input: + csrc = csrc_input.read() + cdef = cdef_input.read() + ffi = make_ffi_from_sources(module_name, cdef, csrc) + generated = generate_c_source(ffi) + write_c_source(output, generated) + + +def _prog(): + # The same parser serves both documented invocations; make --help + # and usage messages show the one that was actually used. + argv0 = os.path.basename(sys.argv[0]) if sys.argv else '' + if argv0.startswith('cffi-gen-src'): + return 'cffi-gen-src' + return 'python -m cffi.gen_src' + + +parser = argparse.ArgumentParser( + prog=_prog(), + description='Generate CFFI C source for extension modules.', +) +subparsers = parser.add_subparsers(dest='mode') + +exec_python_parser = subparsers.add_parser( + 'exec-python', + help='Execute a Python script that defines an FFI object', +) +exec_python_parser.add_argument( + '--ffi-var', + default='ffibuilder', + help="Name of the FFI object in the Python script; defaults to 'ffibuilder'.", +) +exec_python_parser.add_argument( + 'pyfile', + type=argparse.FileType('r', encoding='utf-8'), + help='Path to the Python script', +) +exec_python_parser.add_argument( + 'output', + help='Output path for the C source', +) + +read_sources_parser = subparsers.add_parser( + 'read-sources', + help='Read cdef and C source prelude files that define an FFI object', +) +read_sources_parser.add_argument( + 'module_name', + help='Full name of the generated module, including packages', +) +read_sources_parser.add_argument( + 'cdef', + type=argparse.FileType('r', encoding='utf-8'), + help='File containing C definitions', +) +read_sources_parser.add_argument( + 'csrc', + type=argparse.FileType('r', encoding='utf-8'), + help='File containing C source prelude', +) +read_sources_parser.add_argument( + 'output', + help='Output path for the C source', +) + + +def run(args=None): + args = parser.parse_args(args=args) + if args.mode == 'exec-python': + exec_python(output=args.output, pyfile=args.pyfile, ffi_var=args.ffi_var) + elif args.mode == 'read-sources': + if same_input_file(args.cdef, args.csrc): + parser.error('cdef and csrc are the same file and should not be') + read_sources( + output=args.output, + module_name=args.module_name, + cdef_input=args.cdef, + csrc_input=args.csrc, + ) + else: + parser.error('a subcommand is required: exec-python or read-sources') + parser.exit(0) diff --git a/py_modules/cffi/_cffi_include.h b/py_modules/cffi/_cffi_include.h new file mode 100644 index 0000000..c8c6d34 --- /dev/null +++ b/py_modules/cffi/_cffi_include.h @@ -0,0 +1,397 @@ +#define _CFFI_ + +/* We try to define Py_LIMITED_API before including Python.h. + + Mess: we can only define it if Py_DEBUG, Py_TRACE_REFS and + Py_REF_DEBUG are not defined. This is a best-effort approximation: + we can learn about Py_DEBUG from pyconfig.h, but it is unclear if + the same works for the other two macros. Py_DEBUG implies them, + but not the other way around. + + The implementation is messy (issue #350): on Windows, with _MSC_VER, + we have to define Py_LIMITED_API even before including pyconfig.h. + In that case, we guess what pyconfig.h will do to the macros above, + and check our guess after the #include. + + Note that on Windows, with CPython 3.x, you need >= 3.5 and virtualenv + version >= 16.0.0. With older versions of either, you don't get a + copy of PYTHON3.DLL in the virtualenv. We can't check the version of + CPython *before* we even include pyconfig.h. ffi.set_source() puts + a ``#define _CFFI_NO_LIMITED_API'' at the start of this file if it is + running on Windows < 3.5, as an attempt at fixing it, but that's + arguably wrong because it may not be the target version of Python. + Still better than nothing I guess. As another workaround, you can + remove the definition of Py_LIMITED_API here. + + See also 'py_limited_api' in cffi/setuptools_ext.py. +*/ +#if !defined(_CFFI_USE_EMBEDDING) && !defined(Py_LIMITED_API) +# ifdef _MSC_VER +# if !defined(_DEBUG) && !defined(Py_DEBUG) && !defined(Py_TRACE_REFS) && !defined(Py_REF_DEBUG) && !defined(_CFFI_NO_LIMITED_API) +# if !defined(Py_GIL_DISABLED) +# define Py_LIMITED_API +# else +# define Py_LIMITED_API 0x030f0000 +# endif +# endif + +# include + /* sanity-check: Py_LIMITED_API will cause crashes if any of these + are also defined. Normally, the Python file PC/pyconfig.h does not + cause any of these to be defined, with the exception that _DEBUG + causes Py_DEBUG. Double-check that. */ +# ifdef Py_LIMITED_API +# if defined(Py_DEBUG) +# error "pyconfig.h unexpectedly defines Py_DEBUG, but Py_LIMITED_API is set" +# endif +# if defined(Py_TRACE_REFS) +# error "pyconfig.h unexpectedly defines Py_TRACE_REFS, but Py_LIMITED_API is set" +# endif +# if defined(Py_REF_DEBUG) +# error "pyconfig.h unexpectedly defines Py_REF_DEBUG, but Py_LIMITED_API is set" +# endif +# endif +# else +# include +# if !defined(Py_DEBUG) && !defined(Py_TRACE_REFS) && !defined(Py_REF_DEBUG) && !defined(_CFFI_NO_LIMITED_API) +# if !defined(Py_GIL_DISABLED) +# define Py_LIMITED_API +# else +# define Py_LIMITED_API 0x030f0000 +# endif +# endif +# endif +#endif + +#include +#ifdef __cplusplus +extern "C" { +#endif +#include +#include +#include + +#include "parse_c_type.h" + +/* this block of #ifs should be kept exactly identical between + c/_cffi_backend.c, cffi/vengine_cpy.py, cffi/vengine_gen.py + and cffi/_cffi_include.h */ +#if defined(_MSC_VER) +# include /* for alloca() */ +# if _MSC_VER < 1600 /* MSVC < 2010 */ + typedef __int8 int8_t; + typedef __int16 int16_t; + typedef __int32 int32_t; + typedef __int64 int64_t; + typedef unsigned __int8 uint8_t; + typedef unsigned __int16 uint16_t; + typedef unsigned __int32 uint32_t; + typedef unsigned __int64 uint64_t; + typedef __int8 int_least8_t; + typedef __int16 int_least16_t; + typedef __int32 int_least32_t; + typedef __int64 int_least64_t; + typedef unsigned __int8 uint_least8_t; + typedef unsigned __int16 uint_least16_t; + typedef unsigned __int32 uint_least32_t; + typedef unsigned __int64 uint_least64_t; + typedef __int8 int_fast8_t; + typedef __int16 int_fast16_t; + typedef __int32 int_fast32_t; + typedef __int64 int_fast64_t; + typedef unsigned __int8 uint_fast8_t; + typedef unsigned __int16 uint_fast16_t; + typedef unsigned __int32 uint_fast32_t; + typedef unsigned __int64 uint_fast64_t; + typedef __int64 intmax_t; + typedef unsigned __int64 uintmax_t; +# else +# include +# endif +# if _MSC_VER < 1800 /* MSVC < 2013 */ +# ifndef __cplusplus + typedef unsigned char _Bool; +# endif +# endif +# define _cffi_float_complex_t _Fcomplex /* include for it */ +# define _cffi_double_complex_t _Dcomplex /* include for it */ +#else +# include +# if (defined (__SVR4) && defined (__sun)) || defined(_AIX) || defined(__hpux) +# include +# endif +# define _cffi_float_complex_t float _Complex +# define _cffi_double_complex_t double _Complex +#endif + +#ifdef __GNUC__ +# define _CFFI_UNUSED_FN __attribute__((unused)) +#else +# define _CFFI_UNUSED_FN /* nothing */ +#endif + +#ifdef __cplusplus +# ifndef _Bool + typedef bool _Bool; /* semi-hackish: C++ has no _Bool; bool is builtin */ +# endif +#endif + +/********** CPython-specific section **********/ +#ifndef PYPY_VERSION + + +#define _cffi_from_c_double PyFloat_FromDouble +#define _cffi_from_c_float PyFloat_FromDouble +#define _cffi_from_c_long PyLong_FromLong +#define _cffi_from_c_ulong PyLong_FromUnsignedLong +#define _cffi_from_c_longlong PyLong_FromLongLong +#define _cffi_from_c_ulonglong PyLong_FromUnsignedLongLong +#define _cffi_from_c__Bool PyBool_FromLong + +#define _cffi_to_c_double PyFloat_AsDouble +#define _cffi_to_c_float PyFloat_AsDouble + +#define _cffi_from_c_int(x, type) \ + (((type)-1) > 0 ? /* unsigned */ \ + (sizeof(type) < sizeof(long) ? \ + PyLong_FromLong((long)x) : \ + sizeof(type) == sizeof(long) ? \ + PyLong_FromUnsignedLong((unsigned long)x) : \ + PyLong_FromUnsignedLongLong((unsigned long long)x)) : \ + (sizeof(type) <= sizeof(long) ? \ + PyLong_FromLong((long)x) : \ + PyLong_FromLongLong((long long)x))) + +#define _cffi_to_c_int(o, type) \ + ((type)( \ + sizeof(type) == 1 ? (((type)-1) > 0 ? (type)_cffi_to_c_u8(o) \ + : (type)_cffi_to_c_i8(o)) : \ + sizeof(type) == 2 ? (((type)-1) > 0 ? (type)_cffi_to_c_u16(o) \ + : (type)_cffi_to_c_i16(o)) : \ + sizeof(type) == 4 ? (((type)-1) > 0 ? (type)_cffi_to_c_u32(o) \ + : (type)_cffi_to_c_i32(o)) : \ + sizeof(type) == 8 ? (((type)-1) > 0 ? (type)_cffi_to_c_u64(o) \ + : (type)_cffi_to_c_i64(o)) : \ + (Py_FatalError("unsupported size for type " #type), (type)0))) + +#define _cffi_to_c_i8 \ + ((int(*)(PyObject *))_cffi_exports[1]) +#define _cffi_to_c_u8 \ + ((int(*)(PyObject *))_cffi_exports[2]) +#define _cffi_to_c_i16 \ + ((int(*)(PyObject *))_cffi_exports[3]) +#define _cffi_to_c_u16 \ + ((int(*)(PyObject *))_cffi_exports[4]) +#define _cffi_to_c_i32 \ + ((int(*)(PyObject *))_cffi_exports[5]) +#define _cffi_to_c_u32 \ + ((unsigned int(*)(PyObject *))_cffi_exports[6]) +#define _cffi_to_c_i64 \ + ((long long(*)(PyObject *))_cffi_exports[7]) +#define _cffi_to_c_u64 \ + ((unsigned long long(*)(PyObject *))_cffi_exports[8]) +#define _cffi_to_c_char \ + ((int(*)(PyObject *))_cffi_exports[9]) +#define _cffi_from_c_pointer \ + ((PyObject *(*)(char *, struct _cffi_ctypedescr *))_cffi_exports[10]) +#define _cffi_to_c_pointer \ + ((char *(*)(PyObject *, struct _cffi_ctypedescr *))_cffi_exports[11]) +#define _cffi_get_struct_layout \ + not used any more +#define _cffi_restore_errno \ + ((void(*)(void))_cffi_exports[13]) +#define _cffi_save_errno \ + ((void(*)(void))_cffi_exports[14]) +#define _cffi_from_c_char \ + ((PyObject *(*)(char))_cffi_exports[15]) +#define _cffi_from_c_deref \ + ((PyObject *(*)(char *, struct _cffi_ctypedescr *))_cffi_exports[16]) +#define _cffi_to_c \ + ((int(*)(char *, struct _cffi_ctypedescr *, PyObject *))_cffi_exports[17]) +#define _cffi_from_c_struct \ + ((PyObject *(*)(char *, struct _cffi_ctypedescr *))_cffi_exports[18]) +#define _cffi_to_c_wchar_t \ + ((_cffi_wchar_t(*)(PyObject *))_cffi_exports[19]) +#define _cffi_from_c_wchar_t \ + ((PyObject *(*)(_cffi_wchar_t))_cffi_exports[20]) +#define _cffi_to_c_long_double \ + ((long double(*)(PyObject *))_cffi_exports[21]) +#define _cffi_to_c__Bool \ + ((_Bool(*)(PyObject *))_cffi_exports[22]) +#define _cffi_prepare_pointer_call_argument \ + ((Py_ssize_t(*)(struct _cffi_ctypedescr *, \ + PyObject *, char **))_cffi_exports[23]) +#define _cffi_convert_array_from_object \ + ((int(*)(char *, struct _cffi_ctypedescr *, PyObject *))_cffi_exports[24]) +#define _CFFI_CPIDX 25 +#define _cffi_call_python \ + ((void(*)(struct _cffi_externpy_s *, char *))_cffi_exports[_CFFI_CPIDX]) +#define _cffi_to_c_wchar3216_t \ + ((int(*)(PyObject *))_cffi_exports[26]) +#define _cffi_from_c_wchar3216_t \ + ((PyObject *(*)(int))_cffi_exports[27]) +#define _CFFI_NUM_EXPORTS 28 + +struct _cffi_ctypedescr; + +static void *_cffi_exports[_CFFI_NUM_EXPORTS]; + +#define _cffi_type(index) ( \ + assert((((uintptr_t)_cffi_types[index]) & 1) == 0), \ + (struct _cffi_ctypedescr *)_cffi_types[index]) + +static PyObject *_cffi_init(const char *module_name, Py_ssize_t version, + const struct _cffi_type_context_s *ctx) +{ + PyObject *module, *o_arg, *new_module; + void *raw[] = { + (void *)module_name, + (void *)version, + (void *)_cffi_exports, + (void *)ctx, + }; + + module = PyImport_ImportModule("_cffi_backend"); + if (module == NULL) + goto failure; + + o_arg = PyLong_FromVoidPtr((void *)raw); + if (o_arg == NULL) + goto failure; + + new_module = PyObject_CallMethod( + module, (char *)"_init_cffi_1_0_external_module", (char *)"O", o_arg); + + Py_DECREF(o_arg); + Py_DECREF(module); + return new_module; + + failure: + Py_XDECREF(module); + return NULL; +} + + +#ifdef HAVE_WCHAR_H +typedef wchar_t _cffi_wchar_t; +#else +typedef uint16_t _cffi_wchar_t; /* same random pick as _cffi_backend.c */ +#endif + +_CFFI_UNUSED_FN static uint16_t _cffi_to_c_char16_t(PyObject *o) +{ + if (sizeof(_cffi_wchar_t) == 2) + return (uint16_t)_cffi_to_c_wchar_t(o); + else + return (uint16_t)_cffi_to_c_wchar3216_t(o); +} + +_CFFI_UNUSED_FN static PyObject *_cffi_from_c_char16_t(uint16_t x) +{ + if (sizeof(_cffi_wchar_t) == 2) + return _cffi_from_c_wchar_t((_cffi_wchar_t)x); + else + return _cffi_from_c_wchar3216_t((int)x); +} + +_CFFI_UNUSED_FN static int _cffi_to_c_char32_t(PyObject *o) +{ + if (sizeof(_cffi_wchar_t) == 4) + return (int)_cffi_to_c_wchar_t(o); + else + return (int)_cffi_to_c_wchar3216_t(o); +} + +_CFFI_UNUSED_FN static PyObject *_cffi_from_c_char32_t(unsigned int x) +{ + if (sizeof(_cffi_wchar_t) == 4) + return _cffi_from_c_wchar_t((_cffi_wchar_t)x); + else + return _cffi_from_c_wchar3216_t((int)x); +} + +union _cffi_union_alignment_u { + unsigned char m_char; + unsigned short m_short; + unsigned int m_int; + unsigned long m_long; + unsigned long long m_longlong; + float m_float; + double m_double; + long double m_longdouble; +}; + +struct _cffi_freeme_s { + struct _cffi_freeme_s *next; + union _cffi_union_alignment_u alignment; +}; + +_CFFI_UNUSED_FN static int +_cffi_convert_array_argument(struct _cffi_ctypedescr *ctptr, PyObject *arg, + char **output_data, Py_ssize_t datasize, + struct _cffi_freeme_s **freeme) +{ + char *p; + if (datasize < 0) + return -1; + + p = *output_data; + if (p == NULL) { + struct _cffi_freeme_s *fp = (struct _cffi_freeme_s *)PyObject_Malloc( + offsetof(struct _cffi_freeme_s, alignment) + (size_t)datasize); + if (fp == NULL) + return -1; + fp->next = *freeme; + *freeme = fp; + p = *output_data = (char *)&fp->alignment; + } + memset((void *)p, 0, (size_t)datasize); + return _cffi_convert_array_from_object(p, ctptr, arg); +} + +_CFFI_UNUSED_FN static void +_cffi_free_array_arguments(struct _cffi_freeme_s *freeme) +{ + do { + void *p = (void *)freeme; + freeme = freeme->next; + PyObject_Free(p); + } while (freeme != NULL); +} + +/********** end CPython-specific section **********/ +#else +_CFFI_UNUSED_FN +static void (*_cffi_call_python_org)(struct _cffi_externpy_s *, char *); +# define _cffi_call_python _cffi_call_python_org +#endif + + +#define _cffi_array_len(array) (sizeof(array) / sizeof((array)[0])) + +#define _cffi_prim_int(size, sign) \ + ((size) == 1 ? ((sign) ? _CFFI_PRIM_INT8 : _CFFI_PRIM_UINT8) : \ + (size) == 2 ? ((sign) ? _CFFI_PRIM_INT16 : _CFFI_PRIM_UINT16) : \ + (size) == 4 ? ((sign) ? _CFFI_PRIM_INT32 : _CFFI_PRIM_UINT32) : \ + (size) == 8 ? ((sign) ? _CFFI_PRIM_INT64 : _CFFI_PRIM_UINT64) : \ + _CFFI__UNKNOWN_PRIM) + +#define _cffi_prim_float(size) \ + ((size) == sizeof(float) ? _CFFI_PRIM_FLOAT : \ + (size) == sizeof(double) ? _CFFI_PRIM_DOUBLE : \ + (size) == sizeof(long double) ? _CFFI__UNKNOWN_LONG_DOUBLE : \ + _CFFI__UNKNOWN_FLOAT_PRIM) + +#define _cffi_check_int(got, got_nonpos, expected) \ + ((got_nonpos) == (expected <= 0) && \ + (got) == (unsigned long long)expected) + +#ifdef MS_WIN32 +# define _cffi_stdcall __stdcall +#else +# define _cffi_stdcall /* nothing */ +#endif + +#ifdef __cplusplus +} +#endif diff --git a/py_modules/cffi/_embedding.h b/py_modules/cffi/_embedding.h new file mode 100644 index 0000000..ba6e79b --- /dev/null +++ b/py_modules/cffi/_embedding.h @@ -0,0 +1,512 @@ + +/***** Support code for embedding *****/ + +#ifdef __cplusplus +extern "C" { +#endif + + +#if defined(_WIN32) +# define CFFI_DLLEXPORT __declspec(dllexport) +#elif defined(__GNUC__) +# define CFFI_DLLEXPORT __attribute__((visibility("default"))) +#else +# define CFFI_DLLEXPORT /* nothing */ +#endif + + +/* There are two global variables of type _cffi_call_python_fnptr: + + * _cffi_call_python, which we declare just below, is the one called + by ``extern "Python"`` implementations. + + * _cffi_call_python_org, which on CPython is actually part of the + _cffi_exports[] array, is the function pointer copied from + _cffi_backend. If _cffi_start_python() fails, then this is set + to NULL; otherwise, it should never be NULL. + + After initialization is complete, both are equal. However, the + first one remains equal to &_cffi_start_and_call_python until the + very end of initialization, when we are (or should be) sure that + concurrent threads also see a completely initialized world, and + only then is it changed. +*/ +#undef _cffi_call_python +typedef void (*_cffi_call_python_fnptr)(struct _cffi_externpy_s *, char *); +static void _cffi_start_and_call_python(struct _cffi_externpy_s *, char *); +static _cffi_call_python_fnptr _cffi_call_python = &_cffi_start_and_call_python; + + +#ifndef _MSC_VER + /* --- Assuming a GCC not infinitely old --- */ +# define cffi_compare_and_swap(l,o,n) __sync_bool_compare_and_swap(l,o,n) +# define cffi_write_barrier() __sync_synchronize() +# if !defined(__amd64__) && !defined(__x86_64__) && \ + !defined(__i386__) && !defined(__i386) +# define cffi_read_barrier() __sync_synchronize() +# else +# define cffi_read_barrier() (void)0 +# endif +#else + /* --- Windows threads version --- */ +# include +# define cffi_compare_and_swap(l,o,n) \ + (InterlockedCompareExchangePointer(l,n,o) == (o)) +# define cffi_write_barrier() InterlockedCompareExchange(&_cffi_dummy,0,0) +# define cffi_read_barrier() (void)0 +static volatile LONG _cffi_dummy; +#endif + +#ifdef WITH_THREAD +# ifndef _MSC_VER +# include + static pthread_mutex_t _cffi_embed_startup_lock; +# else + static CRITICAL_SECTION _cffi_embed_startup_lock; +# endif + static char _cffi_embed_startup_lock_ready = 0; +#endif + +static void _cffi_acquire_reentrant_mutex(void) +{ + static void *volatile lock = NULL; + + while (!cffi_compare_and_swap(&lock, NULL, (void *)1)) { + /* should ideally do a spin loop instruction here, but + hard to do it portably and doesn't really matter I + think: pthread_mutex_init() should be very fast, and + this is only run at start-up anyway. */ + } + +#ifdef WITH_THREAD + if (!_cffi_embed_startup_lock_ready) { +# ifndef _MSC_VER + pthread_mutexattr_t attr; + pthread_mutexattr_init(&attr); + pthread_mutexattr_settype(&attr, PTHREAD_MUTEX_RECURSIVE); + pthread_mutex_init(&_cffi_embed_startup_lock, &attr); +# else + InitializeCriticalSection(&_cffi_embed_startup_lock); +# endif + _cffi_embed_startup_lock_ready = 1; + } +#endif + + while (!cffi_compare_and_swap(&lock, (void *)1, NULL)) + ; + +#ifndef _MSC_VER + pthread_mutex_lock(&_cffi_embed_startup_lock); +#else + EnterCriticalSection(&_cffi_embed_startup_lock); +#endif +} + +static void _cffi_release_reentrant_mutex(void) +{ +#ifndef _MSC_VER + pthread_mutex_unlock(&_cffi_embed_startup_lock); +#else + LeaveCriticalSection(&_cffi_embed_startup_lock); +#endif +} + + +/********** CPython-specific section **********/ +#ifndef PYPY_VERSION + +#include "_cffi_errors.h" + + +#define _cffi_call_python_org _cffi_exports[_CFFI_CPIDX] + +PyMODINIT_FUNC _CFFI_PYTHON_STARTUP_FUNC(void); /* forward */ + +static void _cffi_py_initialize(void) +{ + /* XXX use initsigs=0, which "skips initialization registration of + signal handlers, which might be useful when Python is + embedded" according to the Python docs. But review and think + if it should be a user-controllable setting. + + XXX we should also give a way to write errors to a buffer + instead of to stderr. + + XXX if importing 'site' fails, CPython (any version) calls + exit(). Should we try to work around this behavior here? + */ + Py_InitializeEx(0); +} + +static int _cffi_initialize_python(void) +{ + /* This initializes Python, imports _cffi_backend, and then the + present .dll/.so is set up as a CPython C extension module. + */ + int result; + PyGILState_STATE state; + PyObject *pycode=NULL, *global_dict=NULL, *x; + PyObject *builtins; + + state = PyGILState_Ensure(); + + /* Call the initxxx() function from the present module. It will + create and initialize us as a CPython extension module, instead + of letting the startup Python code do it---it might reimport + the same .dll/.so and get maybe confused on some platforms. + It might also have troubles locating the .dll/.so again for all + I know. + */ + (void)_CFFI_PYTHON_STARTUP_FUNC(); + if (PyErr_Occurred()) + goto error; + + /* Now run the Python code provided to ffi.embedding_init_code(). + */ + pycode = Py_CompileString(_CFFI_PYTHON_STARTUP_CODE, + "", + Py_file_input); + if (pycode == NULL) + goto error; + global_dict = PyDict_New(); + if (global_dict == NULL) + goto error; + builtins = PyEval_GetBuiltins(); + if (builtins == NULL) + goto error; + if (PyDict_SetItemString(global_dict, "__builtins__", builtins) < 0) + goto error; + x = PyEval_EvalCode( + pycode, global_dict, global_dict); + if (x == NULL) + goto error; + Py_DECREF(x); + + /* Done! Now if we've been called from + _cffi_start_and_call_python() in an ``extern "Python"``, we can + only hope that the Python code did correctly set up the + corresponding @ffi.def_extern() function. Otherwise, the + general logic of ``extern "Python"`` functions (inside the + _cffi_backend module) will find that the reference is still + missing and print an error. + */ + result = 0; + done: + Py_XDECREF(pycode); + Py_XDECREF(global_dict); + PyGILState_Release(state); + return result; + + error:; + { + /* Print as much information as potentially useful. + Debugging load-time failures with embedding is not fun + */ + PyObject *ecap; + PyObject *exception, *v, *tb, *f, *modules, *mod; + PyErr_Fetch(&exception, &v, &tb); + ecap = _cffi_start_error_capture(); + f = PySys_GetObject((char *)"stderr"); + if (f != NULL && f != Py_None) { + PyFile_WriteString( + "Failed to initialize the Python-CFFI embedding logic:\n\n", f); + } + + if (exception != NULL) { + PyErr_NormalizeException(&exception, &v, &tb); + PyErr_Display(exception, v, tb); + } + Py_XDECREF(exception); + Py_XDECREF(v); + Py_XDECREF(tb); + + if (f != NULL && f != Py_None) { + PyFile_WriteString("\nFrom: " _CFFI_MODULE_NAME + "\ncompiled with cffi version: 2.1.1" + "\n_cffi_backend module: ", f); + modules = PyImport_GetModuleDict(); + mod = PyDict_GetItemString(modules, "_cffi_backend"); + if (mod == NULL) { + PyFile_WriteString("not loaded", f); + } + else { + v = PyObject_GetAttrString(mod, "__file__"); + PyFile_WriteObject(v, f, 0); + Py_XDECREF(v); + } + PyFile_WriteString("\nsys.path: ", f); + PyFile_WriteObject(PySys_GetObject((char *)"path"), f, 0); + PyFile_WriteString("\n\n", f); + } + _cffi_stop_error_capture(ecap); + } + result = -1; + goto done; +} + +static int _cffi_carefully_make_gil(void) +{ + /* This does the basic initialization of Python. It can be called + completely concurrently from unrelated threads. It assumes + that we don't hold the GIL before (if it exists), and we don't + hold it afterwards. + + (What it really does used to be completely different in Python 2 + and Python 3, with the Python 2 solution avoiding the spin-lock + around the Py_InitializeEx() call. However, after recent changes + to CPython 2.7 (issue #358) it no longer works. So we use the + Python 3 solution everywhere.) + + This initializes Python by calling Py_InitializeEx(). + Important: this must not be called concurrently at all. + So we use a global variable as a simple spin lock. This global + variable must be from 'libpythonX.Y.so', not from this + cffi-based extension module, because it must be shared from + different cffi-based extension modules. + + In Python < 3.8, we choose + _PyParser_TokenNames[0] as a completely arbitrary pointer value + that is never written to. The default is to point to the + string "ENDMARKER". We change it temporarily to point to the + next character in that string. (Yes, I know it's REALLY + obscure.) + + In Python >= 3.8, this string array is no longer writable, so + instead we pick PyCapsuleType.tp_version_tag. We can't change + Python < 3.8 because someone might use a mixture of cffi + embedded modules, some of which were compiled before this file + changed. + + In Python >= 3.12, this stopped working because that particular + tp_version_tag gets modified during interpreter startup. It's + arguably a bad idea before 3.12 too, but again we can't change + that because someone might use a mixture of cffi embedded + modules, and no-one reported a bug so far. In Python >= 3.12 + we go instead for PyCapsuleType.tp_as_buffer, which is supposed + to always be NULL. We write to it temporarily a pointer to + a struct full of NULLs, which is semantically the same. + */ + +#ifdef WITH_THREAD +# if PY_VERSION_HEX < 0x030C0000 + int volatile *lock = (int volatile *)&PyCapsule_Type.tp_version_tag; + int old_value, locked_value = -42; + assert(!(PyCapsule_Type.tp_flags & Py_TPFLAGS_HAVE_VERSION_TAG)); +# else + static struct ebp_s { PyBufferProcs buf; int mark; } empty_buffer_procs; + empty_buffer_procs.mark = -42; + PyBufferProcs *volatile *lock = (PyBufferProcs *volatile *) + &PyCapsule_Type.tp_as_buffer; + PyBufferProcs *old_value, *locked_value = &empty_buffer_procs.buf; +# endif + + while (1) { /* spin loop */ + old_value = *lock; + if (old_value == 0) { + if (cffi_compare_and_swap(lock, old_value, locked_value)) + break; + } + else { +# if PY_VERSION_HEX < 0x030C0000 + assert(old_value == locked_value); +# else + /* The pointer should point to a possibly different + empty_buffer_procs from another C extension module */ + assert(((struct ebp_s *)old_value)->mark == -42); +# endif + /* should ideally do a spin loop instruction here, but + hard to do it portably and doesn't really matter I + think: PyEval_InitThreads() should be very fast, and + this is only run at start-up anyway. */ + } + } +#endif + + /* call Py_InitializeEx() */ + if (!Py_IsInitialized()) { + _cffi_py_initialize(); + PyEval_SaveThread(); /* release the GIL */ + /* the returned tstate must be the one that has been stored into the + autoTLSkey by _PyGILState_Init() called from Py_Initialize(). */ + } + else { + } + +#ifdef WITH_THREAD + /* release the lock */ + while (!cffi_compare_and_swap(lock, locked_value, old_value)) + ; +#endif + + return 0; +} + +/********** end CPython-specific section **********/ + + +#else + + +/********** PyPy-specific section **********/ + +PyMODINIT_FUNC _CFFI_PYTHON_STARTUP_FUNC(const void *[]); /* forward */ + +static struct _cffi_pypy_init_s { + const char *name; + void *func; /* function pointer */ + const char *code; +} _cffi_pypy_init = { + _CFFI_MODULE_NAME, + _CFFI_PYTHON_STARTUP_FUNC, + _CFFI_PYTHON_STARTUP_CODE, +}; + +extern int pypy_carefully_make_gil(const char *); +extern int pypy_init_embedded_cffi_module(int, struct _cffi_pypy_init_s *); + +static int _cffi_carefully_make_gil(void) +{ + return pypy_carefully_make_gil(_CFFI_MODULE_NAME); +} + +static int _cffi_initialize_python(void) +{ + return pypy_init_embedded_cffi_module(0xB011, &_cffi_pypy_init); +} + +/********** end PyPy-specific section **********/ + + +#endif + + +#ifdef __GNUC__ +__attribute__((noinline)) +#endif +static _cffi_call_python_fnptr _cffi_start_python(void) +{ + /* Delicate logic to initialize Python. This function can be + called multiple times concurrently, e.g. when the process calls + its first ``extern "Python"`` functions in multiple threads at + once. It can also be called recursively, in which case we must + ignore it. We also have to consider what occurs if several + different cffi-based extensions reach this code in parallel + threads---it is a different copy of the code, then, and we + can't have any shared global variable unless it comes from + 'libpythonX.Y.so'. + + Idea: + + * _cffi_carefully_make_gil(): "carefully" call + PyEval_InitThreads() (possibly with Py_InitializeEx() first). + + * then we use a (local) custom lock to make sure that a call to this + cffi-based extension will wait if another call to the *same* + extension is running the initialization in another thread. + It is reentrant, so that a recursive call will not block, but + only one from a different thread. + + * then we grab the GIL and (Python 2) we call Py_InitializeEx(). + At this point, concurrent calls to Py_InitializeEx() are not + possible: we have the GIL. + + * do the rest of the specific initialization, which may + temporarily release the GIL but not the custom lock. + Only release the custom lock when we are done. + */ + static char called = 0; + + if (_cffi_carefully_make_gil() != 0) + return NULL; + + _cffi_acquire_reentrant_mutex(); + + /* Here the GIL exists, but we don't have it. We're only protected + from concurrency by the reentrant mutex. */ + + /* This file only initializes the embedded module once, the first + time this is called, even if there are subinterpreters. */ + if (!called) { + called = 1; /* invoke _cffi_initialize_python() only once, + but don't set '_cffi_call_python' right now, + otherwise concurrent threads won't call + this function at all (we need them to wait) */ + if (_cffi_initialize_python() == 0) { + /* now initialization is finished. Switch to the fast-path. */ + + /* We would like nobody to see the new value of + '_cffi_call_python' without also seeing the rest of the + data initialized. However, this is not possible. But + the new value of '_cffi_call_python' is the function + 'cffi_call_python()' from _cffi_backend. So: */ + cffi_write_barrier(); + /* ^^^ we put a write barrier here, and a corresponding + read barrier at the start of cffi_call_python(). This + ensures that after that read barrier, we see everything + done here before the write barrier. + */ + + assert(_cffi_call_python_org != NULL); + _cffi_call_python = (_cffi_call_python_fnptr)_cffi_call_python_org; + } + else { + /* initialization failed. Reset this to NULL, even if it was + already set to some other value. Future calls to + _cffi_start_python() are still forced to occur, and will + always return NULL from now on. */ + _cffi_call_python_org = NULL; + } + } + + _cffi_release_reentrant_mutex(); + + return (_cffi_call_python_fnptr)_cffi_call_python_org; +} + +static +void _cffi_start_and_call_python(struct _cffi_externpy_s *externpy, char *args) +{ + _cffi_call_python_fnptr fnptr; + int current_err = errno; +#ifdef _MSC_VER + int current_lasterr = GetLastError(); +#endif + fnptr = _cffi_start_python(); + if (fnptr == NULL) { + fprintf(stderr, "function %s() called, but initialization code " + "failed. Returning 0.\n", externpy->name); + memset(args, 0, externpy->size_of_result); + } +#ifdef _MSC_VER + SetLastError(current_lasterr); +#endif + errno = current_err; + + if (fnptr != NULL) + fnptr(externpy, args); +} + + +/* The cffi_start_python() function makes sure Python is initialized + and our cffi module is set up. It can be called manually from the + user C code. The same effect is obtained automatically from any + dll-exported ``extern "Python"`` function. This function returns + -1 if initialization failed, 0 if all is OK. */ +_CFFI_UNUSED_FN +static int cffi_start_python(void) +{ + if (_cffi_call_python == &_cffi_start_and_call_python) { + if (_cffi_start_python() == NULL) + return -1; + } + cffi_read_barrier(); + return 0; +} + +#undef cffi_compare_and_swap +#undef cffi_write_barrier +#undef cffi_read_barrier + +#ifdef __cplusplus +} +#endif diff --git a/py_modules/cffi/_imp_emulation.py b/py_modules/cffi/_imp_emulation.py new file mode 100644 index 0000000..136abdd --- /dev/null +++ b/py_modules/cffi/_imp_emulation.py @@ -0,0 +1,83 @@ + +try: + # this works on Python < 3.12 + from imp import * + +except ImportError: + # this is a limited emulation for Python >= 3.12. + # Note that this is used only for tests or for the old ffi.verify(). + # This is copied from the source code of Python 3.11. + + from _imp import (acquire_lock, release_lock, + is_builtin, is_frozen) + + from importlib._bootstrap import _load + + from importlib import machinery + import os + import sys + import tokenize + + SEARCH_ERROR = 0 + PY_SOURCE = 1 + PY_COMPILED = 2 + C_EXTENSION = 3 + PY_RESOURCE = 4 + PKG_DIRECTORY = 5 + C_BUILTIN = 6 + PY_FROZEN = 7 + PY_CODERESOURCE = 8 + IMP_HOOK = 9 + + def get_suffixes(): + extensions = [(s, 'rb', C_EXTENSION) + for s in machinery.EXTENSION_SUFFIXES] + source = [(s, 'r', PY_SOURCE) for s in machinery.SOURCE_SUFFIXES] + bytecode = [(s, 'rb', PY_COMPILED) for s in machinery.BYTECODE_SUFFIXES] + return extensions + source + bytecode + + def find_module(name, path=None): + if not isinstance(name, str): + raise TypeError("'name' must be a str, not {}".format(type(name))) + elif not isinstance(path, (type(None), list)): + # Backwards-compatibility + raise RuntimeError("'path' must be None or a list, " + "not {}".format(type(path))) + + if path is None: + if is_builtin(name): + return None, None, ('', '', C_BUILTIN) + elif is_frozen(name): + return None, None, ('', '', PY_FROZEN) + else: + path = sys.path + + for entry in path: + package_directory = os.path.join(entry, name) + for suffix in ['.py', machinery.BYTECODE_SUFFIXES[0]]: + package_file_name = '__init__' + suffix + file_path = os.path.join(package_directory, package_file_name) + if os.path.isfile(file_path): + return None, package_directory, ('', '', PKG_DIRECTORY) + for suffix, mode, type_ in get_suffixes(): + file_name = name + suffix + file_path = os.path.join(entry, file_name) + if os.path.isfile(file_path): + break + else: + continue + break # Break out of outer loop when breaking out of inner loop. + else: + raise ImportError(name, name=name) + + encoding = None + if 'b' not in mode: + with open(file_path, 'rb') as file: + encoding = tokenize.detect_encoding(file.readline)[0] + file = open(file_path, mode, encoding=encoding) + return file, file_path, (suffix, mode, type_) + + def load_dynamic(name, path, file=None): + loader = machinery.ExtensionFileLoader(name, path) + spec = machinery.ModuleSpec(name=name, loader=loader, origin=path) + return _load(spec) diff --git a/py_modules/cffi/_shimmed_dist_utils.py b/py_modules/cffi/_shimmed_dist_utils.py new file mode 100644 index 0000000..c3d2312 --- /dev/null +++ b/py_modules/cffi/_shimmed_dist_utils.py @@ -0,0 +1,45 @@ +""" +Temporary shim module to indirect the bits of distutils we need from setuptools/distutils while providing useful +error messages beyond `No module named 'distutils' on Python >= 3.12, or when setuptools' vendored distutils is broken. + +This is a compromise to avoid a hard-dep on setuptools for Python >= 3.12, since many users don't need runtime compilation support from CFFI. +""" +import sys + +try: + # import setuptools first; this is the most robust way to ensure its embedded distutils is available + # (the .pth shim should usually work, but this is even more robust) + import setuptools +except Exception as ex: + if sys.version_info >= (3, 12): + # Python 3.12 has no built-in distutils to fall back on, so any import problem is fatal + raise Exception("This CFFI feature requires setuptools on Python >= 3.12. The setuptools module is missing or non-functional.") from ex + + # silently ignore on older Pythons (support fallback to stdlib distutils where available) +else: + del setuptools + +try: + # bring in just the bits of distutils we need, whether they really came from setuptools or stdlib-embedded distutils + from distutils import log, sysconfig + from distutils.ccompiler import CCompiler + from distutils.command.build_ext import build_ext + from distutils.core import Distribution, Extension + from distutils.dir_util import mkpath + from distutils.errors import DistutilsSetupError, CompileError, LinkError + from distutils.log import set_threshold, set_verbosity + + if sys.platform == 'win32': + try: + # FUTURE: msvc9compiler module was removed in setuptools 74; consider removing, as it's only used by an ancient patch in `recompiler` + from distutils.msvc9compiler import MSVCCompiler + except ImportError: + MSVCCompiler = None +except Exception as ex: + if sys.version_info >= (3, 12): + raise Exception("This CFFI feature requires setuptools on Python >= 3.12. Please install the setuptools package.") from ex + + # anything older, just let the underlying distutils import error fly + raise Exception("This CFFI feature requires distutils. Please install the distutils or setuptools package.") from ex + +del sys diff --git a/py_modules/cffi/api.py b/py_modules/cffi/api.py new file mode 100644 index 0000000..ec8ee7d --- /dev/null +++ b/py_modules/cffi/api.py @@ -0,0 +1,960 @@ +import sys, types +from .lock import allocate_lock +from .error import CDefError +from . import model + +try: + basestring +except NameError: + # Python 3.x + basestring = str + +_unspecified = object() + + + +class FFI: + r''' + The main top-level class that you instantiate once, or once per module. + + Example usage: + + ffi = FFI() + ffi.cdef(""" + int printf(const char *, ...); + """) + + C = ffi.dlopen(None) # standard library + -or- + C = ffi.verify() # use a C compiler: verify the decl above is right + + C.printf("hello, %s!\n", ffi.new("char[]", "world")) + ''' + + def __init__(self, backend=None): + """Create an FFI instance. The 'backend' argument is used to + select a non-default backend, mostly for tests. + """ + if backend is None: + # You need PyPy (>= 2.0 beta), or a CPython (>= 2.6) with + # _cffi_backend.so compiled. + import _cffi_backend as backend + from . import __version__ + if backend.__version__ != __version__: + # bad version! Try to be as explicit as possible. + if hasattr(backend, '__file__'): + # CPython + raise Exception("Version mismatch: this is the 'cffi' package version %s, located in %r. When we import the top-level '_cffi_backend' extension module, we get version %s, located in %r. The two versions should be equal; check your installation." % ( + __version__, __file__, + backend.__version__, backend.__file__)) + else: + # PyPy + raise Exception("Version mismatch: this is the 'cffi' package version %s, located in %r. This interpreter comes with a built-in '_cffi_backend' module, which is version %s. The two versions should be equal; check your installation." % ( + __version__, __file__, backend.__version__)) + # (If you insist you can also try to pass the option + # 'backend=backend_ctypes.CTypesBackend()', but don't + # rely on it! It's probably not going to work well.) + + from . import cparser + self._backend = backend + self._lock = allocate_lock() + self._parser = cparser.Parser() + self._cached_btypes = {} + self._parsed_types = types.ModuleType('parsed_types').__dict__ + self._new_types = types.ModuleType('new_types').__dict__ + self._function_caches = [] + self._libraries = [] + self._cdefsources = [] + self._included_ffis = [] + self._windows_unicode = None + self._init_once_cache = {} + self._cdef_version = None + self._embedding = None + self._typecache = model.get_typecache(backend) + if hasattr(backend, 'set_ffi'): + backend.set_ffi(self) + for name in list(backend.__dict__): + if name.startswith('RTLD_'): + setattr(self, name, getattr(backend, name)) + # + with self._lock: + self.BVoidP = self._get_cached_btype(model.voidp_type) + self.BCharA = self._get_cached_btype(model.char_array_type) + if isinstance(backend, types.ModuleType): + # _cffi_backend: attach these constants to the class + if not hasattr(FFI, 'NULL'): + FFI.NULL = self.cast(self.BVoidP, 0) + FFI.CData, FFI.CType = backend._get_types() + else: + # ctypes backend: attach these constants to the instance + self.NULL = self.cast(self.BVoidP, 0) + self.CData, self.CType = backend._get_types() + self.buffer = backend.buffer + + def cdef(self, csource, override=False, packed=False, pack=None): + """Parse the given C source. This registers all declared functions, + types, and global variables. The functions and global variables can + then be accessed via either 'ffi.dlopen()' or 'ffi.verify()'. + The types can be used in 'ffi.new()' and other functions. + If 'packed' is specified as True, all structs declared inside this + cdef are packed, i.e. laid out without any field alignment at all. + Alternatively, 'pack' can be a small integer, and requests for + alignment greater than that are ignored (pack=1 is equivalent to + packed=True). + """ + self._cdef(csource, override=override, packed=packed, pack=pack) + + def embedding_api(self, csource, packed=False, pack=None): + self._cdef(csource, packed=packed, pack=pack, dllexport=True) + if self._embedding is None: + self._embedding = '' + + def _cdef(self, csource, override=False, **options): + if not isinstance(csource, str): # unicode, on Python 2 + if not isinstance(csource, basestring): + raise TypeError("cdef() argument must be a string") + csource = csource.encode('ascii') + with self._lock: + self._cdef_version = object() + self._parser.parse(csource, override=override, **options) + self._cdefsources.append(csource) + if override: + for cache in self._function_caches: + cache.clear() + finishlist = self._parser._recomplete + if finishlist: + self._parser._recomplete = [] + for tp in finishlist: + tp.finish_backend_type(self, finishlist) + + def dlopen(self, name, flags=0): + """Load and return a dynamic library identified by 'name'. + The standard C library can be loaded by passing None. + Note that functions and types declared by 'ffi.cdef()' are not + linked to a particular library, just like C headers; in the + library we only look for the actual (untyped) symbols. + """ + if not (isinstance(name, basestring) or + name is None or + isinstance(name, self.CData)): + raise TypeError("dlopen(name): name must be a file name, None, " + "or an already-opened 'void *' handle") + with self._lock: + lib, function_cache = _make_ffi_library(self, name, flags) + self._function_caches.append(function_cache) + self._libraries.append(lib) + return lib + + def dlclose(self, lib): + """Close a library obtained with ffi.dlopen(). After this call, + access to functions or variables from the library will fail + (possibly with a segmentation fault). + """ + type(lib).__cffi_close__(lib) + + def _typeof_locked(self, cdecl): + # call me with the lock! + key = cdecl + if key in self._parsed_types: + return self._parsed_types[key] + # + if not isinstance(cdecl, str): # unicode, on Python 2 + cdecl = cdecl.encode('ascii') + # + type = self._parser.parse_type(cdecl) + really_a_function_type = type.is_raw_function + if really_a_function_type: + type = type.as_function_pointer() + btype = self._get_cached_btype(type) + result = btype, really_a_function_type + self._parsed_types[key] = result + return result + + def _typeof(self, cdecl, consider_function_as_funcptr=False): + # string -> ctype object + try: + result = self._parsed_types[cdecl] + except KeyError: + with self._lock: + result = self._typeof_locked(cdecl) + # + btype, really_a_function_type = result + if really_a_function_type and not consider_function_as_funcptr: + raise CDefError("the type %r is a function type, not a " + "pointer-to-function type" % (cdecl,)) + return btype + + def typeof(self, cdecl): + """Parse the C type given as a string and return the + corresponding object. + It can also be used on 'cdata' instance to get its C type. + """ + if isinstance(cdecl, basestring): + return self._typeof(cdecl) + if isinstance(cdecl, self.CData): + return self._backend.typeof(cdecl) + if isinstance(cdecl, types.BuiltinFunctionType): + res = _builtin_function_type(cdecl) + if res is not None: + return res + if (isinstance(cdecl, types.FunctionType) + and hasattr(cdecl, '_cffi_base_type')): + with self._lock: + return self._get_cached_btype(cdecl._cffi_base_type) + raise TypeError(type(cdecl)) + + def sizeof(self, cdecl): + """Return the size in bytes of the argument. It can be a + string naming a C type, or a 'cdata' instance. + """ + if isinstance(cdecl, basestring): + BType = self._typeof(cdecl) + return self._backend.sizeof(BType) + else: + return self._backend.sizeof(cdecl) + + def alignof(self, cdecl): + """Return the natural alignment size in bytes of the C type + given as a string. + """ + if isinstance(cdecl, basestring): + cdecl = self._typeof(cdecl) + return self._backend.alignof(cdecl) + + def offsetof(self, cdecl, *fields_or_indexes): + """Return the offset of the named field inside the given + structure or array, which must be given as a C type name. + You can give several field names in case of nested structures. + You can also give numeric values which correspond to array + items, in case of an array type. + """ + if isinstance(cdecl, basestring): + cdecl = self._typeof(cdecl) + return self._typeoffsetof(cdecl, *fields_or_indexes)[1] + + def new(self, cdecl, init=None): + """Allocate an instance according to the specified C type and + return a pointer to it. The specified C type must be either a + pointer or an array: ``new('X *')`` allocates an X and returns + a pointer to it, whereas ``new('X[n]')`` allocates an array of + n X'es and returns an array referencing it (which works + mostly like a pointer, like in C). You can also use + ``new('X[]', n)`` to allocate an array of a non-constant + length n. + + The memory is initialized following the rules of declaring a + global variable in C: by default it is zero-initialized, but + an explicit initializer can be given which can be used to + fill all or part of the memory. + + When the returned object goes out of scope, the memory + is freed. In other words the returned object has + ownership of the value of type 'cdecl' that it points to. This + means that the raw data can be used as long as this object is + kept alive, but must not be used for a longer time. Be careful + about that when copying the pointer to the memory somewhere + else, e.g. into another structure. + """ + if isinstance(cdecl, basestring): + cdecl = self._typeof(cdecl) + return self._backend.newp(cdecl, init) + + def new_allocator(self, alloc=None, free=None, + should_clear_after_alloc=True): + """Return a new allocator, i.e. a function that behaves like ffi.new() + but uses the provided low-level 'alloc' and 'free' functions. + + 'alloc' is called with the size as argument. If it returns NULL, a + MemoryError is raised. 'free' is called with the result of 'alloc' + as argument. Both can be either Python function or directly C + functions. If 'free' is None, then no free function is called. + If both 'alloc' and 'free' are None, the default is used. + + If 'should_clear_after_alloc' is set to False, then the memory + returned by 'alloc' is assumed to be already cleared (or you are + fine with garbage); otherwise CFFI will clear it. + """ + compiled_ffi = self._backend.FFI() + allocator = compiled_ffi.new_allocator(alloc, free, + should_clear_after_alloc) + def allocate(cdecl, init=None): + if isinstance(cdecl, basestring): + cdecl = self._typeof(cdecl) + return allocator(cdecl, init) + return allocate + + def cast(self, cdecl, source): + """Similar to a C cast: returns an instance of the named C + type initialized with the given 'source'. The source is + casted between integers or pointers of any type. + """ + if isinstance(cdecl, basestring): + cdecl = self._typeof(cdecl) + return self._backend.cast(cdecl, source) + + def string(self, cdata, maxlen=-1): + """Return a Python string (or unicode string) from the 'cdata'. + If 'cdata' is a pointer or array of characters or bytes, returns + the null-terminated string. The returned string extends until + the first null character, or at most 'maxlen' characters. If + 'cdata' is an array then 'maxlen' defaults to its length. + + If 'cdata' is a pointer or array of wchar_t, returns a unicode + string following the same rules. + + If 'cdata' is a single character or byte or a wchar_t, returns + it as a string or unicode string. + + If 'cdata' is an enum, returns the value of the enumerator as a + string, or 'NUMBER' if the value is out of range. + """ + return self._backend.string(cdata, maxlen) + + def unpack(self, cdata, length): + """Unpack an array of C data of the given length, + returning a Python string/unicode/list. + + If 'cdata' is a pointer to 'char', returns a byte string. + It does not stop at the first null. This is equivalent to: + ffi.buffer(cdata, length)[:] + + If 'cdata' is a pointer to 'wchar_t', returns a unicode string. + 'length' is measured in wchar_t's; it is not the size in bytes. + + If 'cdata' is a pointer to anything else, returns a list of + 'length' items. This is a faster equivalent to: + [cdata[i] for i in range(length)] + """ + return self._backend.unpack(cdata, length) + + #def buffer(self, cdata, size=-1): + # """Return a read-write buffer object that references the raw C data + # pointed to by the given 'cdata'. The 'cdata' must be a pointer or + # an array. Can be passed to functions expecting a buffer, or directly + # manipulated with: + # + # buf[:] get a copy of it in a regular string, or + # buf[idx] as a single character + # buf[:] = ... + # buf[idx] = ... change the content + # """ + # note that 'buffer' is a type, set on this instance by __init__ + + def from_buffer(self, cdecl, python_buffer=_unspecified, + require_writable=False): + """Return a cdata of the given type pointing to the data of the + given Python object, which must support the buffer interface. + Note that this is not meant to be used on the built-in types + str or unicode (you can build 'char[]' arrays explicitly) + but only on objects containing large quantities of raw data + in some other format, like 'array.array' or numpy arrays. + + The first argument is optional and default to 'char[]'. + """ + if python_buffer is _unspecified: + cdecl, python_buffer = self.BCharA, cdecl + elif isinstance(cdecl, basestring): + cdecl = self._typeof(cdecl) + return self._backend.from_buffer(cdecl, python_buffer, + require_writable) + + def memmove(self, dest, src, n): + """ffi.memmove(dest, src, n) copies n bytes of memory from src to dest. + + Like the C function memmove(), the memory areas may overlap; + apart from that it behaves like the C function memcpy(). + + 'src' can be any cdata ptr or array, or any Python buffer object. + 'dest' can be any cdata ptr or array, or a writable Python buffer + object. The size to copy, 'n', is always measured in bytes. + + Unlike other methods, this one supports all Python buffer including + byte strings and bytearrays---but it still does not support + non-contiguous buffers. + """ + return self._backend.memmove(dest, src, n) + + def callback(self, cdecl, python_callable=None, error=None, onerror=None): + """Return a callback object or a decorator making such a + callback object. 'cdecl' must name a C function pointer type. + The callback invokes the specified 'python_callable' (which may + be provided either directly or via a decorator). Important: the + callback object must be manually kept alive for as long as the + callback may be invoked from the C level. + """ + def callback_decorator_wrap(python_callable): + if not callable(python_callable): + raise TypeError("the 'python_callable' argument " + "is not callable") + return self._backend.callback(cdecl, python_callable, + error, onerror) + if isinstance(cdecl, basestring): + cdecl = self._typeof(cdecl, consider_function_as_funcptr=True) + if python_callable is None: + return callback_decorator_wrap # decorator mode + else: + return callback_decorator_wrap(python_callable) # direct mode + + def getctype(self, cdecl, replace_with=''): + """Return a string giving the C type 'cdecl', which may be itself + a string or a object. If 'replace_with' is given, it gives + extra text to append (or insert for more complicated C types), like + a variable name, or '*' to get actually the C type 'pointer-to-cdecl'. + """ + if isinstance(cdecl, basestring): + cdecl = self._typeof(cdecl) + replace_with = replace_with.strip() + if (replace_with.startswith('*') + and '&[' in self._backend.getcname(cdecl, '&')): + replace_with = '(%s)' % replace_with + elif replace_with and replace_with[0] not in '[(': + replace_with = ' ' + replace_with + return self._backend.getcname(cdecl, replace_with) + + def gc(self, cdata, destructor, size=0): + """Return a new cdata object that points to the same + data. Later, when this new cdata object is garbage-collected, + 'destructor(old_cdata_object)' will be called. + + The optional 'size' gives an estimate of the size, used to + trigger the garbage collection more eagerly. So far only used + on PyPy. It tells the GC that the returned object keeps alive + roughly 'size' bytes of external memory. + """ + return self._backend.gcp(cdata, destructor, size) + + def _get_cached_btype(self, type): + assert self._lock.acquire(False) is False + # call me with the lock! + try: + BType = self._cached_btypes[type] + except KeyError: + finishlist = [] + BType = type.get_cached_btype(self, finishlist) + for type in finishlist: + type.finish_backend_type(self, finishlist) + return BType + + def verify(self, source='', tmpdir=None, **kwargs): + """Verify that the current ffi signatures compile on this + machine, and return a dynamic library object. The dynamic + library can be used to call functions and access global + variables declared in this 'ffi'. The library is compiled + by the C compiler: it gives you C-level API compatibility + (including calling macros). This is unlike 'ffi.dlopen()', + which requires binary compatibility in the signatures. + """ + from .verifier import Verifier, _caller_dir_pycache + # + # If set_unicode(True) was called, insert the UNICODE and + # _UNICODE macro declarations + if self._windows_unicode: + self._apply_windows_unicode(kwargs) + # + # Set the tmpdir here, and not in Verifier.__init__: it picks + # up the caller's directory, which we want to be the caller of + # ffi.verify(), as opposed to the caller of Veritier(). + tmpdir = tmpdir or _caller_dir_pycache() + # + # Make a Verifier() and use it to load the library. + self.verifier = Verifier(self, source, tmpdir, **kwargs) + lib = self.verifier.load_library() + # + # Save the loaded library for keep-alive purposes, even + # if the caller doesn't keep it alive itself (it should). + self._libraries.append(lib) + return lib + + def _get_errno(self): + return self._backend.get_errno() + def _set_errno(self, errno): + self._backend.set_errno(errno) + errno = property(_get_errno, _set_errno, None, + "the value of 'errno' from/to the C calls") + + def getwinerror(self, code=-1): + return self._backend.getwinerror(code) + + def _pointer_to(self, ctype): + with self._lock: + return model.pointer_cache(self, ctype) + + def addressof(self, cdata, *fields_or_indexes): + """Return the address of a . + If 'fields_or_indexes' are given, returns the address of that + field or array item in the structure or array, recursively in + case of nested structures. + """ + try: + ctype = self._backend.typeof(cdata) + except TypeError: + if '__addressof__' in type(cdata).__dict__: + return type(cdata).__addressof__(cdata, *fields_or_indexes) + raise + if fields_or_indexes: + ctype, offset = self._typeoffsetof(ctype, *fields_or_indexes) + else: + if ctype.kind == "pointer": + raise TypeError("addressof(pointer)") + offset = 0 + ctypeptr = self._pointer_to(ctype) + return self._backend.rawaddressof(ctypeptr, cdata, offset) + + def _typeoffsetof(self, ctype, field_or_index, *fields_or_indexes): + ctype, offset = self._backend.typeoffsetof(ctype, field_or_index) + for field1 in fields_or_indexes: + ctype, offset1 = self._backend.typeoffsetof(ctype, field1, 1) + offset += offset1 + return ctype, offset + + def include(self, ffi_to_include): + """Includes the typedefs, structs, unions and enums defined + in another FFI instance. Usage is similar to a #include in C, + where a part of the program might include types defined in + another part for its own usage. Note that the include() + method has no effect on functions, constants and global + variables, which must anyway be accessed directly from the + lib object returned by the original FFI instance. + """ + if not isinstance(ffi_to_include, FFI): + raise TypeError("ffi.include() expects an argument that is also of" + " type cffi.FFI, not %r" % ( + type(ffi_to_include).__name__,)) + if ffi_to_include is self: + raise ValueError("self.include(self)") + with ffi_to_include._lock: + with self._lock: + self._parser.include(ffi_to_include._parser) + self._cdefsources.append('[') + self._cdefsources.extend(ffi_to_include._cdefsources) + self._cdefsources.append(']') + self._included_ffis.append(ffi_to_include) + + def new_handle(self, x): + return self._backend.newp_handle(self.BVoidP, x) + + def from_handle(self, x): + return self._backend.from_handle(x) + + def release(self, x): + self._backend.release(x) + + def set_unicode(self, enabled_flag): + """Windows: if 'enabled_flag' is True, enable the UNICODE and + _UNICODE defines in C, and declare the types like TCHAR and LPTCSTR + to be (pointers to) wchar_t. If 'enabled_flag' is False, + declare these types to be (pointers to) plain 8-bit characters. + This is mostly for backward compatibility; you usually want True. + """ + if self._windows_unicode is not None: + raise ValueError("set_unicode() can only be called once") + enabled_flag = bool(enabled_flag) + if enabled_flag: + self.cdef("typedef wchar_t TBYTE;" + "typedef wchar_t TCHAR;" + "typedef const wchar_t *LPCTSTR;" + "typedef const wchar_t *PCTSTR;" + "typedef wchar_t *LPTSTR;" + "typedef wchar_t *PTSTR;" + "typedef TBYTE *PTBYTE;" + "typedef TCHAR *PTCHAR;") + else: + self.cdef("typedef char TBYTE;" + "typedef char TCHAR;" + "typedef const char *LPCTSTR;" + "typedef const char *PCTSTR;" + "typedef char *LPTSTR;" + "typedef char *PTSTR;" + "typedef TBYTE *PTBYTE;" + "typedef TCHAR *PTCHAR;") + self._windows_unicode = enabled_flag + + def _apply_windows_unicode(self, kwds): + defmacros = kwds.get('define_macros', ()) + if not isinstance(defmacros, (list, tuple)): + raise TypeError("'define_macros' must be a list or tuple") + defmacros = list(defmacros) + [('UNICODE', '1'), + ('_UNICODE', '1')] + kwds['define_macros'] = defmacros + + def _apply_embedding_fix(self, kwds): + # must include an argument like "-lpython2.7" for the compiler + def ensure(key, value): + lst = kwds.setdefault(key, []) + if value not in lst: + lst.append(value) + # + if '__pypy__' in sys.builtin_module_names: + import os + if sys.platform == "win32": + # we need 'libpypy-c.lib'. Current distributions of + # pypy (>= 4.1) contain it as 'libs/python27.lib'. + pythonlib = "python{0[0]}{0[1]}".format(sys.version_info) + if hasattr(sys, 'prefix'): + ensure('library_dirs', os.path.join(sys.prefix, 'libs')) + else: + # we need 'libpypy-c.{so,dylib}', which should be by + # default located in 'sys.prefix/bin' for installed + # systems. + if sys.version_info < (3,): + pythonlib = "pypy-c" + else: + pythonlib = "pypy3-c" + if hasattr(sys, 'prefix'): + ensure('library_dirs', os.path.join(sys.prefix, 'bin')) + # On uninstalled pypy's, the libpypy-c is typically found in + # .../pypy/goal/. + if hasattr(sys, 'prefix'): + ensure('library_dirs', os.path.join(sys.prefix, 'pypy', 'goal')) + else: + if sys.platform == "win32": + template = "python%d%d" + if hasattr(sys, 'gettotalrefcount'): + template += '_d' + else: + try: + import sysconfig + except ImportError: # 2.6 + from cffi._shimmed_dist_utils import sysconfig + template = "python%d.%d" + if sysconfig.get_config_var('DEBUG_EXT'): + template += sysconfig.get_config_var('DEBUG_EXT') + pythonlib = (template % + (sys.hexversion >> 24, (sys.hexversion >> 16) & 0xff)) + if hasattr(sys, 'abiflags'): + pythonlib += sys.abiflags + ensure('libraries', pythonlib) + if sys.platform == "win32": + ensure('extra_link_args', '/MANIFEST') + + def set_source(self, module_name, source, source_extension='.c', **kwds): + import os + if hasattr(self, '_assigned_source'): + raise ValueError("set_source() cannot be called several times " + "per ffi object") + if not isinstance(module_name, basestring): + raise TypeError("'module_name' must be a string") + if os.sep in module_name or (os.altsep and os.altsep in module_name): + raise ValueError("'module_name' must not contain '/': use a dotted " + "name to make a 'package.module' location") + self._assigned_source = (str(module_name), source, + source_extension, kwds) + + def set_source_pkgconfig(self, module_name, pkgconfig_libs, source, + source_extension='.c', **kwds): + from . import pkgconfig + if not isinstance(pkgconfig_libs, list): + raise TypeError("the pkgconfig_libs argument must be a list " + "of package names") + kwds2 = pkgconfig.flags_from_pkgconfig(pkgconfig_libs) + pkgconfig.merge_flags(kwds, kwds2) + self.set_source(module_name, source, source_extension, **kwds) + + def distutils_extension(self, tmpdir='build', verbose=True): + from cffi._shimmed_dist_utils import mkpath + from .recompiler import recompile + # + if not hasattr(self, '_assigned_source'): + if hasattr(self, 'verifier'): # fallback, 'tmpdir' ignored + return self.verifier.get_extension() + raise ValueError("set_source() must be called before" + " distutils_extension()") + module_name, source, source_extension, kwds = self._assigned_source + if source is None: + raise TypeError("distutils_extension() is only for C extension " + "modules, not for dlopen()-style pure Python " + "modules") + mkpath(tmpdir) + ext, updated = recompile(self, module_name, + source, tmpdir=tmpdir, extradir=tmpdir, + source_extension=source_extension, + call_c_compiler=False, **kwds) + if verbose: + if updated: + sys.stderr.write("regenerated: %r\n" % (ext.sources[0],)) + else: + sys.stderr.write("not modified: %r\n" % (ext.sources[0],)) + return ext + + def emit_c_code(self, filename): + from .recompiler import recompile + # + if not hasattr(self, '_assigned_source'): + raise ValueError("set_source() must be called before emit_c_code()") + module_name, source, source_extension, kwds = self._assigned_source + if source is None: + raise TypeError("emit_c_code() is only for C extension modules, " + "not for dlopen()-style pure Python modules") + recompile(self, module_name, source, + c_file=filename, call_c_compiler=False, + uses_ffiplatform=False, **kwds) + + def emit_python_code(self, filename): + from .recompiler import recompile + # + if not hasattr(self, '_assigned_source'): + raise ValueError("set_source() must be called before emit_c_code()") + module_name, source, source_extension, kwds = self._assigned_source + if source is not None: + raise TypeError("emit_python_code() is only for dlopen()-style " + "pure Python modules, not for C extension modules") + recompile(self, module_name, source, + c_file=filename, call_c_compiler=False, + uses_ffiplatform=False, **kwds) + + def compile(self, tmpdir='.', verbose=0, target=None, debug=None): + """The 'target' argument gives the final file name of the + compiled DLL. Use '*' to force distutils' choice, suitable for + regular CPython C API modules. Use a file name ending in '.*' + to ask for the system's default extension for dynamic libraries + (.so/.dll/.dylib). + + The default is '*' when building a non-embedded C API extension, + and (module_name + '.*') when building an embedded library. + """ + from .recompiler import recompile + # + if not hasattr(self, '_assigned_source'): + raise ValueError("set_source() must be called before compile()") + module_name, source, source_extension, kwds = self._assigned_source + return recompile(self, module_name, source, tmpdir=tmpdir, + target=target, source_extension=source_extension, + compiler_verbose=verbose, debug=debug, **kwds) + + def init_once(self, func, tag): + # Read _init_once_cache[tag], which is either (False, lock) if + # we're calling the function now in some thread, or (True, result). + # Don't call setdefault() in most cases, to avoid allocating and + # immediately freeing a lock; but still use setdefaut() to avoid + # races. + try: + x = self._init_once_cache[tag] + except KeyError: + x = self._init_once_cache.setdefault(tag, (False, allocate_lock())) + # Common case: we got (True, result), so we return the result. + if x[0]: + return x[1] + # Else, it's a lock. Acquire it to serialize the following tests. + with x[1]: + # Read again from _init_once_cache the current status. + x = self._init_once_cache[tag] + if x[0]: + return x[1] + # Call the function and store the result back. + result = func() + self._init_once_cache[tag] = (True, result) + return result + + def embedding_init_code(self, pysource): + if self._embedding: + raise ValueError("embedding_init_code() can only be called once") + # fix 'pysource' before it gets dumped into the C file: + # - remove empty lines at the beginning, so it starts at "line 1" + # - dedent, if all non-empty lines are indented + # - check for SyntaxErrors + import re + match = re.match(r'\s*\n', pysource) + if match: + pysource = pysource[match.end():] + lines = pysource.splitlines() or [''] + prefix = re.match(r'\s*', lines[0]).group() + for i in range(1, len(lines)): + line = lines[i] + if line.rstrip(): + while not line.startswith(prefix): + prefix = prefix[:-1] + i = len(prefix) + lines = [line[i:]+'\n' for line in lines] + pysource = ''.join(lines) + # + compile(pysource, "cffi_init", "exec") + # + self._embedding = pysource + + def def_extern(self, *args, **kwds): + raise ValueError("ffi.def_extern() is only available on API-mode FFI " + "objects") + + def list_types(self): + """Returns the user type names known to this FFI instance. + This returns a tuple containing three lists of names: + (typedef_names, names_of_structs, names_of_unions) + """ + typedefs = [] + structs = [] + unions = [] + for key in self._parser._declarations: + if key.startswith('typedef '): + typedefs.append(key[8:]) + elif key.startswith('struct '): + structs.append(key[7:]) + elif key.startswith('union '): + unions.append(key[6:]) + typedefs.sort() + structs.sort() + unions.sort() + return (typedefs, structs, unions) + + +def _load_backend_lib(backend, name, flags): + import os + if not isinstance(name, basestring): + if sys.platform != "win32" or name is not None: + return backend.load_library(name, flags) + name = "c" # Windows: load_library(None) fails, but this works + # on Python 2 (backward compatibility hack only) + first_error = None + if '.' in name or '/' in name or os.sep in name: + try: + return backend.load_library(name, flags) + except OSError as e: + first_error = e + import ctypes.util + path = ctypes.util.find_library(name) + if path is None: + if name == "c" and sys.platform == "win32" and sys.version_info >= (3,): + raise OSError("dlopen(None) cannot work on Windows for Python 3 " + "(see http://bugs.python.org/issue23606)") + msg = ("ctypes.util.find_library() did not manage " + "to locate a library called %r" % (name,)) + if first_error is not None: + msg = "%s. Additionally, %s" % (first_error, msg) + raise OSError(msg) + return backend.load_library(path, flags) + +def _make_ffi_library(ffi, libname, flags): + backend = ffi._backend + backendlib = _load_backend_lib(backend, libname, flags) + # + def accessor_function(name): + key = 'function ' + name + tp, _ = ffi._parser._declarations[key] + BType = ffi._get_cached_btype(tp) + value = backendlib.load_function(BType, name) + library.__dict__[name] = value + # + def accessor_variable(name): + key = 'variable ' + name + tp, _ = ffi._parser._declarations[key] + BType = ffi._get_cached_btype(tp) + read_variable = backendlib.read_variable + write_variable = backendlib.write_variable + setattr(FFILibrary, name, property( + lambda self: read_variable(BType, name), + lambda self, value: write_variable(BType, name, value))) + # + def addressof_var(name): + try: + return addr_variables[name] + except KeyError: + with ffi._lock: + if name not in addr_variables: + key = 'variable ' + name + tp, _ = ffi._parser._declarations[key] + BType = ffi._get_cached_btype(tp) + if BType.kind != 'array': + BType = model.pointer_cache(ffi, BType) + p = backendlib.load_function(BType, name) + addr_variables[name] = p + return addr_variables[name] + # + def accessor_constant(name): + raise NotImplementedError("non-integer constant '%s' cannot be " + "accessed from a dlopen() library" % (name,)) + # + def accessor_int_constant(name): + library.__dict__[name] = ffi._parser._int_constants[name] + # + accessors = {} + accessors_version = [False] + addr_variables = {} + # + def update_accessors(): + if accessors_version[0] is ffi._cdef_version: + return + # + for key, (tp, _) in ffi._parser._declarations.items(): + if not isinstance(tp, model.EnumType): + tag, name = key.split(' ', 1) + if tag == 'function': + accessors[name] = accessor_function + elif tag == 'variable': + accessors[name] = accessor_variable + elif tag == 'constant': + accessors[name] = accessor_constant + else: + for i, enumname in enumerate(tp.enumerators): + def accessor_enum(name, tp=tp, i=i): + tp.check_not_partial() + library.__dict__[name] = tp.enumvalues[i] + accessors[enumname] = accessor_enum + for name in ffi._parser._int_constants: + accessors.setdefault(name, accessor_int_constant) + accessors_version[0] = ffi._cdef_version + # + def make_accessor(name): + with ffi._lock: + if name in library.__dict__ or name in FFILibrary.__dict__: + return # added by another thread while waiting for the lock + if name not in accessors: + update_accessors() + if name not in accessors: + raise AttributeError(name) + accessors[name](name) + # + class FFILibrary: + def __getattr__(self, name): + make_accessor(name) + return getattr(self, name) + def __setattr__(self, name, value): + try: + property = getattr(self.__class__, name) + except AttributeError: + make_accessor(name) + setattr(self, name, value) + else: + property.__set__(self, value) + def __dir__(self): + with ffi._lock: + update_accessors() + return accessors.keys() + def __addressof__(self, name): + if name in library.__dict__: + return library.__dict__[name] + if name in FFILibrary.__dict__: + return addressof_var(name) + make_accessor(name) + if name in library.__dict__: + return library.__dict__[name] + if name in FFILibrary.__dict__: + return addressof_var(name) + raise AttributeError("cffi library has no function or " + "global variable named '%s'" % (name,)) + def __cffi_close__(self): + backendlib.close_lib() + self.__dict__.clear() + # + if isinstance(libname, basestring): + try: + if not isinstance(libname, str): # unicode, on Python 2 + libname = libname.encode('utf-8') + FFILibrary.__name__ = 'FFILibrary_%s' % libname + except UnicodeError: + pass + library = FFILibrary() + return library, library.__dict__ + +def _builtin_function_type(func): + # a hack to make at least ffi.typeof(builtin_function) work, + # if the builtin function was obtained by 'vengine_cpy'. + import sys + try: + module = sys.modules[func.__module__] + ffi = module._cffi_original_ffi + types_of_builtin_funcs = module._cffi_types_of_builtin_funcs + tp = types_of_builtin_funcs[func] + except (KeyError, AttributeError, TypeError): + return None + else: + with ffi._lock: + return ffi._get_cached_btype(tp) diff --git a/py_modules/cffi/backend_ctypes.py b/py_modules/cffi/backend_ctypes.py new file mode 100644 index 0000000..12cd162 --- /dev/null +++ b/py_modules/cffi/backend_ctypes.py @@ -0,0 +1,1120 @@ +import ctypes, ctypes.util, operator, sys +from . import model + +if sys.version_info < (3,): + bytechr = chr +else: + unicode = str + long = int + xrange = range + bytechr = lambda num: bytes([num]) + +class CTypesType(type): + pass + +class CTypesData: + __metaclass__ = CTypesType + __slots__ = ['__weakref__'] + __name__ = '' + + def __init__(self, *args): + raise TypeError("cannot instantiate %r" % (self.__class__,)) + + @classmethod + def _newp(cls, init): + raise TypeError("expected a pointer or array ctype, got '%s'" + % (cls._get_c_name(),)) + + @staticmethod + def _to_ctypes(value): + raise TypeError + + @classmethod + def _arg_to_ctypes(cls, *value): + try: + ctype = cls._ctype + except AttributeError: + raise TypeError("cannot create an instance of %r" % (cls,)) + if value: + res = cls._to_ctypes(*value) + if not isinstance(res, ctype): + res = cls._ctype(res) + else: + res = cls._ctype() + return res + + @classmethod + def _create_ctype_obj(cls, init): + if init is None: + return cls._arg_to_ctypes() + else: + return cls._arg_to_ctypes(init) + + @staticmethod + def _from_ctypes(ctypes_value): + raise TypeError + + @classmethod + def _get_c_name(cls, replace_with=''): + return cls._reftypename.replace(' &', replace_with) + + @classmethod + def _fix_class(cls): + cls.__name__ = 'CData<%s>' % (cls._get_c_name(),) + cls.__qualname__ = 'CData<%s>' % (cls._get_c_name(),) + cls.__module__ = 'ffi' + + def _get_own_repr(self): + raise NotImplementedError + + def _addr_repr(self, address): + if address == 0: + return 'NULL' + else: + if address < 0: + address += 1 << (8*ctypes.sizeof(ctypes.c_void_p)) + return '0x%x' % address + + def __repr__(self, c_name=None): + own = self._get_own_repr() + return '' % (c_name or self._get_c_name(), own) + + def _convert_to_address(self, BClass): + if BClass is None: + raise TypeError("cannot convert %r to an address" % ( + self._get_c_name(),)) + else: + raise TypeError("cannot convert %r to %r" % ( + self._get_c_name(), BClass._get_c_name())) + + @classmethod + def _get_size(cls): + return ctypes.sizeof(cls._ctype) + + def _get_size_of_instance(self): + return ctypes.sizeof(self._ctype) + + @classmethod + def _cast_from(cls, source): + raise TypeError("cannot cast to %r" % (cls._get_c_name(),)) + + def _cast_to_integer(self): + return self._convert_to_address(None) + + @classmethod + def _alignment(cls): + return ctypes.alignment(cls._ctype) + + def __iter__(self): + raise TypeError("cdata %r does not support iteration" % ( + self._get_c_name()),) + + def _make_cmp(name): + cmpfunc = getattr(operator, name) + def cmp(self, other): + v_is_ptr = not isinstance(self, CTypesGenericPrimitive) + w_is_ptr = (isinstance(other, CTypesData) and + not isinstance(other, CTypesGenericPrimitive)) + if v_is_ptr and w_is_ptr: + return cmpfunc(self._convert_to_address(None), + other._convert_to_address(None)) + elif v_is_ptr or w_is_ptr: + return NotImplemented + else: + if isinstance(self, CTypesGenericPrimitive): + self = self._value + if isinstance(other, CTypesGenericPrimitive): + other = other._value + return cmpfunc(self, other) + cmp.func_name = name + return cmp + + __eq__ = _make_cmp('__eq__') + __ne__ = _make_cmp('__ne__') + __lt__ = _make_cmp('__lt__') + __le__ = _make_cmp('__le__') + __gt__ = _make_cmp('__gt__') + __ge__ = _make_cmp('__ge__') + + def __hash__(self): + return hash(self._convert_to_address(None)) + + def _to_string(self, maxlen): + raise TypeError("string(): %r" % (self,)) + + +class CTypesGenericPrimitive(CTypesData): + __slots__ = [] + + def __hash__(self): + return hash(self._value) + + def _get_own_repr(self): + return repr(self._from_ctypes(self._value)) + + +class CTypesGenericArray(CTypesData): + __slots__ = [] + + @classmethod + def _newp(cls, init): + return cls(init) + + def __iter__(self): + for i in xrange(len(self)): + yield self[i] + + def _get_own_repr(self): + return self._addr_repr(ctypes.addressof(self._blob)) + + +class CTypesGenericPtr(CTypesData): + __slots__ = ['_address', '_as_ctype_ptr'] + _automatic_casts = False + kind = "pointer" + + @classmethod + def _newp(cls, init): + return cls(init) + + @classmethod + def _cast_from(cls, source): + if source is None: + address = 0 + elif isinstance(source, CTypesData): + address = source._cast_to_integer() + elif isinstance(source, (int, long)): + address = source + else: + raise TypeError("bad type for cast to %r: %r" % + (cls, type(source).__name__)) + return cls._new_pointer_at(address) + + @classmethod + def _new_pointer_at(cls, address): + self = cls.__new__(cls) + self._address = address + self._as_ctype_ptr = ctypes.cast(address, cls._ctype) + return self + + def _get_own_repr(self): + try: + return self._addr_repr(self._address) + except AttributeError: + return '???' + + def _cast_to_integer(self): + return self._address + + def __nonzero__(self): + return bool(self._address) + __bool__ = __nonzero__ + + @classmethod + def _to_ctypes(cls, value): + if not isinstance(value, CTypesData): + raise TypeError("unexpected %s object" % type(value).__name__) + address = value._convert_to_address(cls) + return ctypes.cast(address, cls._ctype) + + @classmethod + def _from_ctypes(cls, ctypes_ptr): + address = ctypes.cast(ctypes_ptr, ctypes.c_void_p).value or 0 + return cls._new_pointer_at(address) + + @classmethod + def _initialize(cls, ctypes_ptr, value): + if value: + ctypes_ptr.contents = cls._to_ctypes(value).contents + + def _convert_to_address(self, BClass): + if (BClass in (self.__class__, None) or BClass._automatic_casts + or self._automatic_casts): + return self._address + else: + return CTypesData._convert_to_address(self, BClass) + + +class CTypesBaseStructOrUnion(CTypesData): + __slots__ = ['_blob'] + + @classmethod + def _create_ctype_obj(cls, init): + # may be overridden + raise TypeError("cannot instantiate opaque type %s" % (cls,)) + + def _get_own_repr(self): + return self._addr_repr(ctypes.addressof(self._blob)) + + @classmethod + def _offsetof(cls, fieldname): + return getattr(cls._ctype, fieldname).offset + + def _convert_to_address(self, BClass): + if getattr(BClass, '_BItem', None) is self.__class__: + return ctypes.addressof(self._blob) + else: + return CTypesData._convert_to_address(self, BClass) + + @classmethod + def _from_ctypes(cls, ctypes_struct_or_union): + self = cls.__new__(cls) + self._blob = ctypes_struct_or_union + return self + + @classmethod + def _to_ctypes(cls, value): + return value._blob + + def __repr__(self, c_name=None): + return CTypesData.__repr__(self, c_name or self._get_c_name(' &')) + + +class CTypesBackend: + + PRIMITIVE_TYPES = { + 'char': ctypes.c_char, + 'short': ctypes.c_short, + 'int': ctypes.c_int, + 'long': ctypes.c_long, + 'long long': ctypes.c_longlong, + 'signed char': ctypes.c_byte, + 'unsigned char': ctypes.c_ubyte, + 'unsigned short': ctypes.c_ushort, + 'unsigned int': ctypes.c_uint, + 'unsigned long': ctypes.c_ulong, + 'unsigned long long': ctypes.c_ulonglong, + 'float': ctypes.c_float, + 'double': ctypes.c_double, + '_Bool': ctypes.c_bool, + } + + for _name in ['unsigned long long', 'unsigned long', + 'unsigned int', 'unsigned short', 'unsigned char']: + _size = ctypes.sizeof(PRIMITIVE_TYPES[_name]) + PRIMITIVE_TYPES['uint%d_t' % (8*_size)] = PRIMITIVE_TYPES[_name] + if _size == ctypes.sizeof(ctypes.c_void_p): + PRIMITIVE_TYPES['uintptr_t'] = PRIMITIVE_TYPES[_name] + if _size == ctypes.sizeof(ctypes.c_size_t): + PRIMITIVE_TYPES['size_t'] = PRIMITIVE_TYPES[_name] + + for _name in ['long long', 'long', 'int', 'short', 'signed char']: + _size = ctypes.sizeof(PRIMITIVE_TYPES[_name]) + PRIMITIVE_TYPES['int%d_t' % (8*_size)] = PRIMITIVE_TYPES[_name] + if _size == ctypes.sizeof(ctypes.c_void_p): + PRIMITIVE_TYPES['intptr_t'] = PRIMITIVE_TYPES[_name] + PRIMITIVE_TYPES['ptrdiff_t'] = PRIMITIVE_TYPES[_name] + if _size == ctypes.sizeof(ctypes.c_size_t): + PRIMITIVE_TYPES['ssize_t'] = PRIMITIVE_TYPES[_name] + + + def __init__(self): + self.RTLD_LAZY = 0 # not supported anyway by ctypes + self.RTLD_NOW = 0 + self.RTLD_GLOBAL = ctypes.RTLD_GLOBAL + self.RTLD_LOCAL = ctypes.RTLD_LOCAL + + def set_ffi(self, ffi): + self.ffi = ffi + + def _get_types(self): + return CTypesData, CTypesType + + def load_library(self, path, flags=0): + cdll = ctypes.CDLL(path, flags) + return CTypesLibrary(self, cdll) + + def new_void_type(self): + class CTypesVoid(CTypesData): + __slots__ = [] + _reftypename = 'void &' + @staticmethod + def _from_ctypes(novalue): + return None + @staticmethod + def _to_ctypes(novalue): + if novalue is not None: + raise TypeError("None expected, got %s object" % + (type(novalue).__name__,)) + CTypesVoid._fix_class() + return CTypesVoid + + def new_primitive_type(self, name): + if name == 'wchar_t': + raise NotImplementedError(name) + ctype = self.PRIMITIVE_TYPES[name] + if name == 'char': + kind = 'char' + elif name in ('float', 'double'): + kind = 'float' + else: + if name in ('signed char', 'unsigned char'): + kind = 'byte' + elif name == '_Bool': + kind = 'bool' + else: + kind = 'int' + is_signed = (ctype(-1).value == -1) + # + def _cast_source_to_int(source): + if isinstance(source, (int, long, float)): + source = int(source) + elif isinstance(source, CTypesData): + source = source._cast_to_integer() + elif isinstance(source, bytes): + source = ord(source) + elif source is None: + source = 0 + else: + raise TypeError("bad type for cast to %r: %r" % + (CTypesPrimitive, type(source).__name__)) + return source + # + kind1 = kind + class CTypesPrimitive(CTypesGenericPrimitive): + __slots__ = ['_value'] + _ctype = ctype + _reftypename = '%s &' % name + kind = kind1 + + def __init__(self, value): + self._value = value + + @staticmethod + def _create_ctype_obj(init): + if init is None: + return ctype() + return ctype(CTypesPrimitive._to_ctypes(init)) + + if kind in {'int', 'byte'}: + @classmethod + def _cast_from(cls, source): + source = _cast_source_to_int(source) + source = ctype(source).value # cast within range + return cls(source) + def __int__(self): + return self._value + + if kind == 'bool': + @classmethod + def _cast_from(cls, source): + if not isinstance(source, (int, long, float)): + source = _cast_source_to_int(source) + return cls(bool(source)) + def __int__(self): + return int(self._value) + + if kind == 'char': + @classmethod + def _cast_from(cls, source): + source = _cast_source_to_int(source) + source = bytechr(source & 0xFF) + return cls(source) + def __int__(self): + return ord(self._value) + + if kind == 'float': + @classmethod + def _cast_from(cls, source): + if isinstance(source, float): + pass + elif isinstance(source, CTypesGenericPrimitive): + if hasattr(source, '__float__'): + source = float(source) + else: + source = int(source) + else: + source = _cast_source_to_int(source) + source = ctype(source).value # fix precision + return cls(source) + def __int__(self): + return int(self._value) + def __float__(self): + return self._value + + _cast_to_integer = __int__ + + if kind in {'int', 'byte', 'bool'}: + @staticmethod + def _to_ctypes(x): + if not isinstance(x, (int, long)): + if isinstance(x, CTypesData): + x = int(x) + else: + raise TypeError("integer expected, got %s" % + type(x).__name__) + if ctype(x).value != x: + if not is_signed and x < 0: + raise OverflowError("%s: negative integer" % name) + else: + raise OverflowError("%s: integer out of bounds" + % name) + return x + + if kind == 'char': + @staticmethod + def _to_ctypes(x): + if isinstance(x, bytes) and len(x) == 1: + return x + if isinstance(x, CTypesPrimitive): # > + return x._value + raise TypeError("character expected, got %s" % + type(x).__name__) + def __nonzero__(self): + return ord(self._value) != 0 + else: + def __nonzero__(self): + return self._value != 0 + __bool__ = __nonzero__ + + if kind == 'float': + @staticmethod + def _to_ctypes(x): + if not isinstance(x, (int, long, float, CTypesData)): + raise TypeError("float expected, got %s" % + type(x).__name__) + return ctype(x).value + + @staticmethod + def _from_ctypes(value): + return getattr(value, 'value', value) + + @staticmethod + def _initialize(blob, init): + blob.value = CTypesPrimitive._to_ctypes(init) + + if kind == 'char': + def _to_string(self, maxlen): + return self._value + if kind == 'byte': + def _to_string(self, maxlen): + return chr(self._value & 0xff) + # + CTypesPrimitive._fix_class() + return CTypesPrimitive + + def new_pointer_type(self, BItem): + getbtype = self.ffi._get_cached_btype + if BItem is getbtype(model.PrimitiveType('char')): + kind = 'charp' + elif BItem in (getbtype(model.PrimitiveType('signed char')), + getbtype(model.PrimitiveType('unsigned char'))): + kind = 'bytep' + elif BItem is getbtype(model.void_type): + kind = 'voidp' + else: + kind = 'generic' + # + class CTypesPtr(CTypesGenericPtr): + __slots__ = ['_own'] + if kind == 'charp': + __slots__ += ['__as_strbuf'] + _BItem = BItem + if hasattr(BItem, '_ctype'): + _ctype = ctypes.POINTER(BItem._ctype) + _bitem_size = ctypes.sizeof(BItem._ctype) + else: + _ctype = ctypes.c_void_p + if issubclass(BItem, CTypesGenericArray): + _reftypename = BItem._get_c_name('(* &)') + else: + _reftypename = BItem._get_c_name(' * &') + + def __init__(self, init): + ctypeobj = BItem._create_ctype_obj(init) + if kind == 'charp': + self.__as_strbuf = ctypes.create_string_buffer( + ctypeobj.value + b'\x00') + self._as_ctype_ptr = ctypes.cast( + self.__as_strbuf, self._ctype) + else: + self._as_ctype_ptr = ctypes.pointer(ctypeobj) + self._address = ctypes.cast(self._as_ctype_ptr, + ctypes.c_void_p).value + self._own = True + + def __add__(self, other): + if isinstance(other, (int, long)): + return self._new_pointer_at(self._address + + other * self._bitem_size) + else: + return NotImplemented + + def __sub__(self, other): + if isinstance(other, (int, long)): + return self._new_pointer_at(self._address - + other * self._bitem_size) + elif type(self) is type(other): + return (self._address - other._address) // self._bitem_size + else: + return NotImplemented + + def __getitem__(self, index): + if getattr(self, '_own', False) and index != 0: + raise IndexError + return BItem._from_ctypes(self._as_ctype_ptr[index]) + + def __setitem__(self, index, value): + self._as_ctype_ptr[index] = BItem._to_ctypes(value) + + if kind in {'charp', 'voidp'}: + @classmethod + def _arg_to_ctypes(cls, *value): + if value and isinstance(value[0], bytes): + return ctypes.c_char_p(value[0]) + else: + return super()._arg_to_ctypes(*value) + + if kind in {'charp', 'bytep'}: + def _to_string(self, maxlen): + if maxlen < 0: + maxlen = sys.maxsize + p = ctypes.cast(self._as_ctype_ptr, + ctypes.POINTER(ctypes.c_char)) + n = 0 + while n < maxlen and p[n] != b'\x00': + n += 1 + return b''.join([p[i] for i in range(n)]) + + def _get_own_repr(self): + if getattr(self, '_own', False): + return 'owning %d bytes' % ( + ctypes.sizeof(self._as_ctype_ptr.contents),) + return super()._get_own_repr() + # + if (BItem is self.ffi._get_cached_btype(model.void_type) or + BItem is self.ffi._get_cached_btype(model.PrimitiveType('char'))): + CTypesPtr._automatic_casts = True + # + CTypesPtr._fix_class() + return CTypesPtr + + def new_array_type(self, CTypesPtr, length): + if length is None: + brackets = ' &[]' + else: + brackets = ' &[%d]' % length + BItem = CTypesPtr._BItem + getbtype = self.ffi._get_cached_btype + if BItem is getbtype(model.PrimitiveType('char')): + kind = 'char' + elif BItem in (getbtype(model.PrimitiveType('signed char')), + getbtype(model.PrimitiveType('unsigned char'))): + kind = 'byte' + else: + kind = 'generic' + # + class CTypesArray(CTypesGenericArray): + __slots__ = ['_blob', '_own'] + if length is not None: + _ctype = BItem._ctype * length + else: + __slots__.append('_ctype') + _reftypename = BItem._get_c_name(brackets) + _declared_length = length + _CTPtr = CTypesPtr + + def __init__(self, init): + if length is None: + if isinstance(init, (int, long)): + len1 = init + init = None + elif kind == 'char' and isinstance(init, bytes): + len1 = len(init) + 1 # extra null + else: + init = tuple(init) + len1 = len(init) + self._ctype = BItem._ctype * len1 + self._blob = self._ctype() + self._own = True + if init is not None: + self._initialize(self._blob, init) + + @staticmethod + def _initialize(blob, init): + if isinstance(init, bytes): + init = [init[i:i+1] for i in range(len(init))] + else: + if isinstance(init, CTypesGenericArray): + if (len(init) != len(blob) or + not isinstance(init, CTypesArray)): + raise TypeError("length/type mismatch: %s" % (init,)) + init = tuple(init) + if len(init) > len(blob): + raise IndexError("too many initializers") + addr = ctypes.cast(blob, ctypes.c_void_p).value + PTR = ctypes.POINTER(BItem._ctype) + itemsize = ctypes.sizeof(BItem._ctype) + for i, value in enumerate(init): + p = ctypes.cast(addr + i * itemsize, PTR) + BItem._initialize(p.contents, value) + + def __len__(self): + return len(self._blob) + + def __getitem__(self, index): + if not (0 <= index < len(self._blob)): + raise IndexError + return BItem._from_ctypes(self._blob[index]) + + def __setitem__(self, index, value): + if not (0 <= index < len(self._blob)): + raise IndexError + self._blob[index] = BItem._to_ctypes(value) + + if kind in {'char', 'byte'}: + def _to_string(self, maxlen): + if maxlen < 0: + maxlen = len(self._blob) + p = ctypes.cast(self._blob, + ctypes.POINTER(ctypes.c_char)) + n = 0 + while n < maxlen and p[n] != b'\x00': + n += 1 + return b''.join([p[i] for i in range(n)]) + + def _get_own_repr(self): + if getattr(self, '_own', False): + return 'owning %d bytes' % (ctypes.sizeof(self._blob),) + return super()._get_own_repr() + + def _convert_to_address(self, BClass): + if BClass in (CTypesPtr, None) or BClass._automatic_casts: + return ctypes.addressof(self._blob) + else: + return CTypesData._convert_to_address(self, BClass) + + @staticmethod + def _from_ctypes(ctypes_array): + self = CTypesArray.__new__(CTypesArray) + self._blob = ctypes_array + return self + + @staticmethod + def _arg_to_ctypes(value): + return CTypesPtr._arg_to_ctypes(value) + + def __add__(self, other): + if isinstance(other, (int, long)): + return CTypesPtr._new_pointer_at( + ctypes.addressof(self._blob) + + other * ctypes.sizeof(BItem._ctype)) + else: + return NotImplemented + + @classmethod + def _cast_from(cls, source): + raise NotImplementedError("casting to %r" % ( + cls._get_c_name(),)) + # + CTypesArray._fix_class() + return CTypesArray + + def _new_struct_or_union(self, kind, name, base_ctypes_class): + # + class struct_or_union(base_ctypes_class): + pass + struct_or_union.__name__ = '%s_%s' % (kind, name) + kind1 = kind + # + class CTypesStructOrUnion(CTypesBaseStructOrUnion): + __slots__ = ['_blob'] + _ctype = struct_or_union + _reftypename = '%s &' % (name,) + _kind = kind = kind1 + # + CTypesStructOrUnion._fix_class() + return CTypesStructOrUnion + + def new_struct_type(self, name): + return self._new_struct_or_union('struct', name, ctypes.Structure) + + def new_union_type(self, name): + return self._new_struct_or_union('union', name, ctypes.Union) + + def complete_struct_or_union(self, CTypesStructOrUnion, fields, tp, + totalsize=-1, totalalignment=-1, sflags=0, + pack=0): + if totalsize >= 0 or totalalignment >= 0: + raise NotImplementedError("the ctypes backend of CFFI does not support " + "structures completed by verify(); please " + "compile and install the _cffi_backend module.") + struct_or_union = CTypesStructOrUnion._ctype + fnames = [fname for (fname, BField, bitsize) in fields] + btypes = [BField for (fname, BField, bitsize) in fields] + bitfields = [bitsize for (fname, BField, bitsize) in fields] + # + bfield_types = {} + cfields = [] + for (fname, BField, bitsize) in fields: + if bitsize < 0: + cfields.append((fname, BField._ctype)) + bfield_types[fname] = BField + else: + cfields.append((fname, BField._ctype, bitsize)) + bfield_types[fname] = Ellipsis + if sflags & 8: + struct_or_union._pack_ = 1 + elif pack: + struct_or_union._pack_ = pack + struct_or_union._fields_ = cfields + CTypesStructOrUnion._bfield_types = bfield_types + # + @staticmethod + def _create_ctype_obj(init): + result = struct_or_union() + if init is not None: + initialize(result, init) + return result + CTypesStructOrUnion._create_ctype_obj = _create_ctype_obj + # + def initialize(blob, init): + if is_union: + if len(init) > 1: + raise ValueError("union initializer: %d items given, but " + "only one supported (use a dict if needed)" + % (len(init),)) + if not isinstance(init, dict): + if isinstance(init, (bytes, unicode)): + raise TypeError("union initializer: got a str") + init = tuple(init) + if len(init) > len(fnames): + raise ValueError("too many values for %s initializer" % + CTypesStructOrUnion._get_c_name()) + init = dict(zip(fnames, init)) + addr = ctypes.addressof(blob) + for fname, value in init.items(): + BField, bitsize = name2fieldtype[fname] + assert bitsize < 0, \ + "not implemented: initializer with bit fields" + offset = CTypesStructOrUnion._offsetof(fname) + PTR = ctypes.POINTER(BField._ctype) + p = ctypes.cast(addr + offset, PTR) + BField._initialize(p.contents, value) + is_union = CTypesStructOrUnion._kind == 'union' + name2fieldtype = dict(zip(fnames, zip(btypes, bitfields))) + # + for fname, BField, bitsize in fields: + if fname == '': + raise NotImplementedError("nested anonymous structs/unions") + if hasattr(CTypesStructOrUnion, fname): + raise ValueError("the field name %r conflicts in " + "the ctypes backend" % fname) + if bitsize < 0: + def getter(self, fname=fname, BField=BField, + offset=CTypesStructOrUnion._offsetof(fname), + PTR=ctypes.POINTER(BField._ctype)): + addr = ctypes.addressof(self._blob) + p = ctypes.cast(addr + offset, PTR) + return BField._from_ctypes(p.contents) + def setter(self, value, fname=fname, BField=BField): + setattr(self._blob, fname, BField._to_ctypes(value)) + # + if issubclass(BField, CTypesGenericArray): + setter = None + if BField._declared_length == 0: + def getter(self, fname=fname, BFieldPtr=BField._CTPtr, + offset=CTypesStructOrUnion._offsetof(fname), + PTR=ctypes.POINTER(BField._ctype)): + addr = ctypes.addressof(self._blob) + p = ctypes.cast(addr + offset, PTR) + return BFieldPtr._from_ctypes(p) + # + else: + def getter(self, fname=fname, BField=BField): + return BField._from_ctypes(getattr(self._blob, fname)) + def setter(self, value, fname=fname, BField=BField): + # xxx obscure workaround + value = BField._to_ctypes(value) + oldvalue = getattr(self._blob, fname) + setattr(self._blob, fname, value) + if value != getattr(self._blob, fname): + setattr(self._blob, fname, oldvalue) + raise OverflowError("value too large for bitfield") + setattr(CTypesStructOrUnion, fname, property(getter, setter)) + # + CTypesPtr = self.ffi._get_cached_btype(model.PointerType(tp)) + for fname in fnames: + if hasattr(CTypesPtr, fname): + raise ValueError("the field name %r conflicts in " + "the ctypes backend" % fname) + def getter(self, fname=fname): + return getattr(self[0], fname) + def setter(self, value, fname=fname): + setattr(self[0], fname, value) + setattr(CTypesPtr, fname, property(getter, setter)) + + def new_function_type(self, BArgs, BResult, has_varargs): + nameargs = [BArg._get_c_name() for BArg in BArgs] + if has_varargs: + nameargs.append('...') + nameargs = ', '.join(nameargs) + # + class CTypesFunctionPtr(CTypesGenericPtr): + __slots__ = ['_own_callback', '_name'] + _ctype = ctypes.CFUNCTYPE(getattr(BResult, '_ctype', None), + *[BArg._ctype for BArg in BArgs], + use_errno=True) + _reftypename = BResult._get_c_name('(* &)(%s)' % (nameargs,)) + + def __init__(self, init, error=None): + # create a callback to the Python callable init() + import traceback + assert not has_varargs, "varargs not supported for callbacks" + if getattr(BResult, '_ctype', None) is not None: + error = BResult._from_ctypes( + BResult._create_ctype_obj(error)) + else: + error = None + def callback(*args): + args2 = [] + for arg, BArg in zip(args, BArgs): + args2.append(BArg._from_ctypes(arg)) + try: + res2 = init(*args2) + res2 = BResult._to_ctypes(res2) + except: + traceback.print_exc() + res2 = error + if issubclass(BResult, CTypesGenericPtr): + if res2: + res2 = ctypes.cast(res2, ctypes.c_void_p).value + # .value: http://bugs.python.org/issue1574593 + else: + res2 = None + #print repr(res2) + return res2 + if issubclass(BResult, CTypesGenericPtr): + # The only pointers callbacks can return are void*s: + # http://bugs.python.org/issue5710 + callback_ctype = ctypes.CFUNCTYPE( + ctypes.c_void_p, + *[BArg._ctype for BArg in BArgs], + use_errno=True) + else: + callback_ctype = CTypesFunctionPtr._ctype + self._as_ctype_ptr = callback_ctype(callback) + self._address = ctypes.cast(self._as_ctype_ptr, + ctypes.c_void_p).value + self._own_callback = init + + @staticmethod + def _initialize(ctypes_ptr, value): + if value: + raise NotImplementedError("ctypes backend: not supported: " + "initializers for function pointers") + + def __repr__(self): + c_name = getattr(self, '_name', None) + if c_name: + i = self._reftypename.index('(* &)') + if self._reftypename[i-1] not in ' )*': + c_name = ' ' + c_name + c_name = self._reftypename.replace('(* &)', c_name) + return CTypesData.__repr__(self, c_name) + + def _get_own_repr(self): + if getattr(self, '_own_callback', None) is not None: + return 'calling %r' % (self._own_callback,) + return super()._get_own_repr() + + def __call__(self, *args): + if has_varargs: + assert len(args) >= len(BArgs) + extraargs = args[len(BArgs):] + args = args[:len(BArgs)] + else: + assert len(args) == len(BArgs) + ctypes_args = [] + for arg, BArg in zip(args, BArgs): + ctypes_args.append(BArg._arg_to_ctypes(arg)) + if has_varargs: + for i, arg in enumerate(extraargs): + if arg is None: + ctypes_args.append(ctypes.c_void_p(0)) # NULL + continue + if not isinstance(arg, CTypesData): + raise TypeError( + "argument %d passed in the variadic part " + "needs to be a cdata object (got %s)" % + (1 + len(BArgs) + i, type(arg).__name__)) + ctypes_args.append(arg._arg_to_ctypes(arg)) + result = self._as_ctype_ptr(*ctypes_args) + return BResult._from_ctypes(result) + # + CTypesFunctionPtr._fix_class() + return CTypesFunctionPtr + + def new_enum_type(self, name, enumerators, enumvalues, CTypesInt): + assert isinstance(name, str) + reverse_mapping = dict(zip(reversed(enumvalues), + reversed(enumerators))) + # + class CTypesEnum(CTypesInt): + __slots__ = [] + _reftypename = '%s &' % name + + def _get_own_repr(self): + value = self._value + try: + return '%d: %s' % (value, reverse_mapping[value]) + except KeyError: + return str(value) + + def _to_string(self, maxlen): + value = self._value + try: + return reverse_mapping[value] + except KeyError: + return str(value) + # + CTypesEnum._fix_class() + return CTypesEnum + + def get_errno(self): + return ctypes.get_errno() + + def set_errno(self, value): + ctypes.set_errno(value) + + def string(self, b, maxlen=-1): + return b._to_string(maxlen) + + def buffer(self, bptr, size=-1): + raise NotImplementedError("buffer() with ctypes backend") + + def sizeof(self, cdata_or_BType): + if isinstance(cdata_or_BType, CTypesData): + return cdata_or_BType._get_size_of_instance() + else: + assert issubclass(cdata_or_BType, CTypesData) + return cdata_or_BType._get_size() + + def alignof(self, BType): + assert issubclass(BType, CTypesData) + return BType._alignment() + + def newp(self, BType, source): + if not issubclass(BType, CTypesData): + raise TypeError + return BType._newp(source) + + def cast(self, BType, source): + return BType._cast_from(source) + + def callback(self, BType, source, error, onerror): + assert onerror is None # XXX not implemented + return BType(source, error) + + _weakref_cache_ref = None + + def gcp(self, cdata, destructor, size=0): + if self._weakref_cache_ref is None: + import weakref + class MyRef(weakref.ref): + def __eq__(self, other): + myref = self() + return self is other or ( + myref is not None and myref is other()) + def __ne__(self, other): + return not (self == other) + def __hash__(self): + try: + return self._hash + except AttributeError: + self._hash = hash(self()) + return self._hash + self._weakref_cache_ref = {}, MyRef + weak_cache, MyRef = self._weakref_cache_ref + + if destructor is None: + try: + del weak_cache[MyRef(cdata)] + except KeyError: + raise TypeError("Can remove destructor only on a object " + "previously returned by ffi.gc()") + return None + + def remove(k): + cdata, destructor = weak_cache.pop(k, (None, None)) + if destructor is not None: + destructor(cdata) + + new_cdata = self.cast(self.typeof(cdata), cdata) + assert new_cdata is not cdata + weak_cache[MyRef(new_cdata, remove)] = (cdata, destructor) + return new_cdata + + typeof = type + + def getcname(self, BType, replace_with): + return BType._get_c_name(replace_with) + + def typeoffsetof(self, BType, fieldname, num=0): + if isinstance(fieldname, str): + if num == 0 and issubclass(BType, CTypesGenericPtr): + BType = BType._BItem + if not issubclass(BType, CTypesBaseStructOrUnion): + raise TypeError("expected a struct or union ctype") + BField = BType._bfield_types[fieldname] + if BField is Ellipsis: + raise TypeError("not supported for bitfields") + return (BField, BType._offsetof(fieldname)) + elif isinstance(fieldname, (int, long)): + if issubclass(BType, CTypesGenericArray): + BType = BType._CTPtr + if not issubclass(BType, CTypesGenericPtr): + raise TypeError("expected an array or ptr ctype") + BItem = BType._BItem + offset = BItem._get_size() * fieldname + if offset > sys.maxsize: + raise OverflowError + return (BItem, offset) + else: + raise TypeError(type(fieldname)) + + def rawaddressof(self, BTypePtr, cdata, offset=None): + if isinstance(cdata, CTypesBaseStructOrUnion): + ptr = ctypes.pointer(type(cdata)._to_ctypes(cdata)) + elif isinstance(cdata, CTypesGenericPtr): + if offset is None or not issubclass(type(cdata)._BItem, + CTypesBaseStructOrUnion): + raise TypeError("unexpected cdata type") + ptr = type(cdata)._to_ctypes(cdata) + elif isinstance(cdata, CTypesGenericArray): + ptr = type(cdata)._to_ctypes(cdata) + else: + raise TypeError("expected a ") + if offset: + ptr = ctypes.cast( + ctypes.c_void_p( + ctypes.cast(ptr, ctypes.c_void_p).value + offset), + type(ptr)) + return BTypePtr._from_ctypes(ptr) + + +class CTypesLibrary: + + def __init__(self, backend, cdll): + self.backend = backend + self.cdll = cdll + + def load_function(self, BType, name): + c_func = getattr(self.cdll, name) + funcobj = BType._from_ctypes(c_func) + funcobj._name = name + return funcobj + + def read_variable(self, BType, name): + try: + ctypes_obj = BType._ctype.in_dll(self.cdll, name) + except AttributeError as e: + raise NotImplementedError(e) + return BType._from_ctypes(ctypes_obj) + + def write_variable(self, BType, name, value): + new_ctypes_obj = BType._to_ctypes(value) + ctypes_obj = BType._ctype.in_dll(self.cdll, name) + ctypes.memmove(ctypes.addressof(ctypes_obj), + ctypes.addressof(new_ctypes_obj), + ctypes.sizeof(BType._ctype)) diff --git a/py_modules/cffi/cffi_opcode.py b/py_modules/cffi/cffi_opcode.py new file mode 100644 index 0000000..c964ba1 --- /dev/null +++ b/py_modules/cffi/cffi_opcode.py @@ -0,0 +1,187 @@ +from .error import VerificationError + +class CffiOp: + def __init__(self, op, arg): + self.op = op + self.arg = arg + + def as_c_expr(self): + if self.op is None: + assert isinstance(self.arg, str) + return '(_cffi_opcode_t)(%s)' % (self.arg,) + classname = CLASS_NAME[self.op] + return '_CFFI_OP(_CFFI_OP_%s, %s)' % (classname, self.arg) + + def as_python_bytes(self): + if self.op is None and self.arg.isdigit(): + value = int(self.arg) # non-negative: '-' not in self.arg + if value >= 2**31: + raise OverflowError("cannot emit %r: limited to 2**31-1" + % (self.arg,)) + return format_four_bytes(value) + if isinstance(self.arg, str): + raise VerificationError("cannot emit to Python: %r" % (self.arg,)) + return format_four_bytes((self.arg << 8) | self.op) + + def __str__(self): + classname = CLASS_NAME.get(self.op, self.op) + return '(%s %s)' % (classname, self.arg) + +def format_four_bytes(num): + return '\\x%02X\\x%02X\\x%02X\\x%02X' % ( + (num >> 24) & 0xFF, + (num >> 16) & 0xFF, + (num >> 8) & 0xFF, + (num ) & 0xFF) + +OP_PRIMITIVE = 1 +OP_POINTER = 3 +OP_ARRAY = 5 +OP_OPEN_ARRAY = 7 +OP_STRUCT_UNION = 9 +OP_ENUM = 11 +OP_FUNCTION = 13 +OP_FUNCTION_END = 15 +OP_NOOP = 17 +OP_BITFIELD = 19 +OP_TYPENAME = 21 +OP_CPYTHON_BLTN_V = 23 # varargs +OP_CPYTHON_BLTN_N = 25 # noargs +OP_CPYTHON_BLTN_O = 27 # O (i.e. a single arg) +OP_CONSTANT = 29 +OP_CONSTANT_INT = 31 +OP_GLOBAL_VAR = 33 +OP_DLOPEN_FUNC = 35 +OP_DLOPEN_CONST = 37 +OP_GLOBAL_VAR_F = 39 +OP_EXTERN_PYTHON = 41 + +PRIM_VOID = 0 +PRIM_BOOL = 1 +PRIM_CHAR = 2 +PRIM_SCHAR = 3 +PRIM_UCHAR = 4 +PRIM_SHORT = 5 +PRIM_USHORT = 6 +PRIM_INT = 7 +PRIM_UINT = 8 +PRIM_LONG = 9 +PRIM_ULONG = 10 +PRIM_LONGLONG = 11 +PRIM_ULONGLONG = 12 +PRIM_FLOAT = 13 +PRIM_DOUBLE = 14 +PRIM_LONGDOUBLE = 15 + +PRIM_WCHAR = 16 +PRIM_INT8 = 17 +PRIM_UINT8 = 18 +PRIM_INT16 = 19 +PRIM_UINT16 = 20 +PRIM_INT32 = 21 +PRIM_UINT32 = 22 +PRIM_INT64 = 23 +PRIM_UINT64 = 24 +PRIM_INTPTR = 25 +PRIM_UINTPTR = 26 +PRIM_PTRDIFF = 27 +PRIM_SIZE = 28 +PRIM_SSIZE = 29 +PRIM_INT_LEAST8 = 30 +PRIM_UINT_LEAST8 = 31 +PRIM_INT_LEAST16 = 32 +PRIM_UINT_LEAST16 = 33 +PRIM_INT_LEAST32 = 34 +PRIM_UINT_LEAST32 = 35 +PRIM_INT_LEAST64 = 36 +PRIM_UINT_LEAST64 = 37 +PRIM_INT_FAST8 = 38 +PRIM_UINT_FAST8 = 39 +PRIM_INT_FAST16 = 40 +PRIM_UINT_FAST16 = 41 +PRIM_INT_FAST32 = 42 +PRIM_UINT_FAST32 = 43 +PRIM_INT_FAST64 = 44 +PRIM_UINT_FAST64 = 45 +PRIM_INTMAX = 46 +PRIM_UINTMAX = 47 +PRIM_FLOATCOMPLEX = 48 +PRIM_DOUBLECOMPLEX = 49 +PRIM_CHAR16 = 50 +PRIM_CHAR32 = 51 + +_NUM_PRIM = 52 +_UNKNOWN_PRIM = -1 +_UNKNOWN_FLOAT_PRIM = -2 +_UNKNOWN_LONG_DOUBLE = -3 + +_IO_FILE_STRUCT = -1 + +PRIMITIVE_TO_INDEX = { + 'char': PRIM_CHAR, + 'short': PRIM_SHORT, + 'int': PRIM_INT, + 'long': PRIM_LONG, + 'long long': PRIM_LONGLONG, + 'signed char': PRIM_SCHAR, + 'unsigned char': PRIM_UCHAR, + 'unsigned short': PRIM_USHORT, + 'unsigned int': PRIM_UINT, + 'unsigned long': PRIM_ULONG, + 'unsigned long long': PRIM_ULONGLONG, + 'float': PRIM_FLOAT, + 'double': PRIM_DOUBLE, + 'long double': PRIM_LONGDOUBLE, + '_cffi_float_complex_t': PRIM_FLOATCOMPLEX, + '_cffi_double_complex_t': PRIM_DOUBLECOMPLEX, + '_Bool': PRIM_BOOL, + 'wchar_t': PRIM_WCHAR, + 'char16_t': PRIM_CHAR16, + 'char32_t': PRIM_CHAR32, + 'int8_t': PRIM_INT8, + 'uint8_t': PRIM_UINT8, + 'int16_t': PRIM_INT16, + 'uint16_t': PRIM_UINT16, + 'int32_t': PRIM_INT32, + 'uint32_t': PRIM_UINT32, + 'int64_t': PRIM_INT64, + 'uint64_t': PRIM_UINT64, + 'intptr_t': PRIM_INTPTR, + 'uintptr_t': PRIM_UINTPTR, + 'ptrdiff_t': PRIM_PTRDIFF, + 'size_t': PRIM_SIZE, + 'ssize_t': PRIM_SSIZE, + 'int_least8_t': PRIM_INT_LEAST8, + 'uint_least8_t': PRIM_UINT_LEAST8, + 'int_least16_t': PRIM_INT_LEAST16, + 'uint_least16_t': PRIM_UINT_LEAST16, + 'int_least32_t': PRIM_INT_LEAST32, + 'uint_least32_t': PRIM_UINT_LEAST32, + 'int_least64_t': PRIM_INT_LEAST64, + 'uint_least64_t': PRIM_UINT_LEAST64, + 'int_fast8_t': PRIM_INT_FAST8, + 'uint_fast8_t': PRIM_UINT_FAST8, + 'int_fast16_t': PRIM_INT_FAST16, + 'uint_fast16_t': PRIM_UINT_FAST16, + 'int_fast32_t': PRIM_INT_FAST32, + 'uint_fast32_t': PRIM_UINT_FAST32, + 'int_fast64_t': PRIM_INT_FAST64, + 'uint_fast64_t': PRIM_UINT_FAST64, + 'intmax_t': PRIM_INTMAX, + 'uintmax_t': PRIM_UINTMAX, + } + +F_UNION = 0x01 +F_CHECK_FIELDS = 0x02 +F_PACKED = 0x04 +F_EXTERNAL = 0x08 +F_OPAQUE = 0x10 + +G_FLAGS = dict([('_CFFI_' + _key, globals()[_key]) + for _key in ['F_UNION', 'F_CHECK_FIELDS', 'F_PACKED', + 'F_EXTERNAL', 'F_OPAQUE']]) + +CLASS_NAME = {} +for _name, _value in list(globals().items()): + if _name.startswith('OP_') and isinstance(_value, int): + CLASS_NAME[_value] = _name[3:] diff --git a/py_modules/cffi/commontypes.py b/py_modules/cffi/commontypes.py new file mode 100644 index 0000000..d4dae35 --- /dev/null +++ b/py_modules/cffi/commontypes.py @@ -0,0 +1,82 @@ +import sys +from . import model +from .error import FFIError + + +COMMON_TYPES = {} + +try: + # fetch "bool" and all simple Windows types + from _cffi_backend import _get_common_types + _get_common_types(COMMON_TYPES) +except ImportError: + pass + +COMMON_TYPES['FILE'] = model.unknown_type('FILE', '_IO_FILE') +COMMON_TYPES['bool'] = '_Bool' # in case we got ImportError above +COMMON_TYPES['float _Complex'] = '_cffi_float_complex_t' +COMMON_TYPES['double _Complex'] = '_cffi_double_complex_t' + +for _type in model.PrimitiveType.ALL_PRIMITIVE_TYPES: + if _type.endswith('_t'): + COMMON_TYPES[_type] = _type +del _type + +_CACHE = {} + +def resolve_common_type(parser, commontype): + try: + return _CACHE[commontype] + except KeyError: + cdecl = COMMON_TYPES.get(commontype, commontype) + if not isinstance(cdecl, str): + result, quals = cdecl, 0 # cdecl is already a BaseType + elif cdecl in model.PrimitiveType.ALL_PRIMITIVE_TYPES: + result, quals = model.PrimitiveType(cdecl), 0 + elif cdecl == 'set-unicode-needed': + raise FFIError("The Windows type %r is only available after " + "you call ffi.set_unicode()" % (commontype,)) + else: + if commontype == cdecl: + raise FFIError( + "Unsupported type: %r. Please look at " + "http://cffi.readthedocs.io/en/latest/cdef.html#ffi-cdef-limitations " + "and file an issue if you think this type should really " + "be supported." % (commontype,)) + result, quals = parser.parse_type_and_quals(cdecl) # recursive + + assert isinstance(result, model.BaseTypeByIdentity) + _CACHE[commontype] = result, quals + return result, quals + + +# ____________________________________________________________ +# extra types for Windows (most of them are in commontypes.c) + + +def win_common_types(): + return { + "UNICODE_STRING": model.StructType( + "_UNICODE_STRING", + ["Length", + "MaximumLength", + "Buffer"], + [model.PrimitiveType("unsigned short"), + model.PrimitiveType("unsigned short"), + model.PointerType(model.PrimitiveType("wchar_t"))], + [-1, -1, -1]), + "PUNICODE_STRING": "UNICODE_STRING *", + "PCUNICODE_STRING": "const UNICODE_STRING *", + + "TBYTE": "set-unicode-needed", + "TCHAR": "set-unicode-needed", + "LPCTSTR": "set-unicode-needed", + "PCTSTR": "set-unicode-needed", + "LPTSTR": "set-unicode-needed", + "PTSTR": "set-unicode-needed", + "PTBYTE": "set-unicode-needed", + "PTCHAR": "set-unicode-needed", + } + +if sys.platform == 'win32': + COMMON_TYPES.update(win_common_types()) diff --git a/py_modules/cffi/cparser.py b/py_modules/cffi/cparser.py new file mode 100644 index 0000000..1896b7e --- /dev/null +++ b/py_modules/cffi/cparser.py @@ -0,0 +1,1014 @@ +from . import model +from .commontypes import COMMON_TYPES, resolve_common_type +from .error import FFIError, CDefError +try: + from . import _pycparser as pycparser +except ImportError: + import pycparser +import weakref, re, sys + +try: + if sys.version_info < (3,): + import thread as _thread + else: + import _thread + lock = _thread.allocate_lock() +except ImportError: + lock = None + +def _workaround_for_static_import_finders(): + # Issue #392: packaging tools like cx_Freeze can not find these + # because pycparser uses exec dynamic import. This is an obscure + # workaround. This function is never called. + import pycparser.yacctab + import pycparser.lextab + +CDEF_SOURCE_STRING = "" +_r_comment = re.compile(r"/\*.*?\*/|//([^\n\\]|\\.)*?$", + re.DOTALL | re.MULTILINE) +_r_define = re.compile(r"^\s*#\s*define\s+([A-Za-z_][A-Za-z_0-9]*)" + r"\b((?:[^\n\\]|\\.)*?)$", + re.DOTALL | re.MULTILINE) +_r_line_directive = re.compile(r"^[ \t]*#[ \t]*(?:line|\d+)\b.*$", re.MULTILINE) +_r_partial_enum = re.compile(r"=\s*\.\.\.\s*[,}]|\.\.\.\s*\}") +_r_enum_dotdotdot = re.compile(r"__dotdotdot\d+__$") +_r_partial_array = re.compile(r"\[\s*\.\.\.\s*\]") +_r_words = re.compile(r"\w+|\S") +_parser_cache = None +_r_int_literal = re.compile(r"-?0?x?[0-9a-f]+[lu]*$", re.IGNORECASE) +_r_stdcall1 = re.compile(r"\b(__stdcall|WINAPI)\b") +_r_stdcall2 = re.compile(r"[(]\s*(__stdcall|WINAPI)\b") +_r_cdecl = re.compile(r"\b__cdecl\b") +_r_extern_python = re.compile(r'\bextern\s*"' + r'(Python|Python\s*\+\s*C|C\s*\+\s*Python)"\s*.') +_r_star_const_space = re.compile( # matches "* const " + r"[*]\s*((const|volatile|restrict)\b\s*)+") +_r_int_dotdotdot = re.compile(r"(\b(int|long|short|signed|unsigned|char)\s*)+" + r"\.\.\.") +_r_float_dotdotdot = re.compile(r"\b(double|float)\s*\.\.\.") + +def _get_parser(): + global _parser_cache + if _parser_cache is None: + _parser_cache = pycparser.CParser() + return _parser_cache + +def _workaround_for_old_pycparser(csource): + # Workaround for a pycparser issue (fixed between pycparser 2.10 and + # 2.14): "char*const***" gives us a wrong syntax tree, the same as + # for "char***(*const)". This means we can't tell the difference + # afterwards. But "char(*const(***))" gives us the right syntax + # tree. The issue only occurs if there are several stars in + # sequence with no parenthesis in between, just possibly qualifiers. + # Attempt to fix it by adding some parentheses in the source: each + # time we see "* const" or "* const *", we add an opening + # parenthesis before each star---the hard part is figuring out where + # to close them. + parts = [] + while True: + match = _r_star_const_space.search(csource) + if not match: + break + #print repr(''.join(parts)+csource), '=>', + parts.append(csource[:match.start()]) + parts.append('('); closing = ')' + parts.append(match.group()) # e.g. "* const " + endpos = match.end() + if csource.startswith('*', endpos): + parts.append('('); closing += ')' + level = 0 + i = endpos + while i < len(csource): + c = csource[i] + if c == '(': + level += 1 + elif c == ')': + if level == 0: + break + level -= 1 + elif c in ',;=': + if level == 0: + break + i += 1 + csource = csource[endpos:i] + closing + csource[i:] + #print repr(''.join(parts)+csource) + parts.append(csource) + return ''.join(parts) + +def _preprocess_extern_python(csource): + # input: `extern "Python" int foo(int);` or + # `extern "Python" { int foo(int); }` + # output: + # void __cffi_extern_python_start; + # int foo(int); + # void __cffi_extern_python_stop; + # + # input: `extern "Python+C" int foo(int);` + # output: + # void __cffi_extern_python_plus_c_start; + # int foo(int); + # void __cffi_extern_python_stop; + parts = [] + while True: + match = _r_extern_python.search(csource) + if not match: + break + endpos = match.end() - 1 + #print + #print ''.join(parts)+csource + #print '=>' + parts.append(csource[:match.start()]) + if 'C' in match.group(1): + parts.append('void __cffi_extern_python_plus_c_start; ') + else: + parts.append('void __cffi_extern_python_start; ') + if csource[endpos] == '{': + # grouping variant + closing = csource.find('}', endpos) + if closing < 0: + raise CDefError("'extern \"Python\" {': no '}' found") + if csource.find('{', endpos + 1, closing) >= 0: + raise NotImplementedError("cannot use { } inside a block " + "'extern \"Python\" { ... }'") + parts.append(csource[endpos+1:closing]) + csource = csource[closing+1:] + else: + # non-grouping variant + semicolon = csource.find(';', endpos) + if semicolon < 0: + raise CDefError("'extern \"Python\": no ';' found") + parts.append(csource[endpos:semicolon+1]) + csource = csource[semicolon+1:] + parts.append(' void __cffi_extern_python_stop;') + #print ''.join(parts)+csource + #print + parts.append(csource) + return ''.join(parts) + +def _warn_for_string_literal(csource): + if '"' not in csource: + return + for line in csource.splitlines(): + if '"' in line and not line.lstrip().startswith('#'): + import warnings + warnings.warn("String literal found in cdef() or type source. " + "String literals are ignored here, but you should " + "remove them anyway because some character sequences " + "confuse pre-parsing.") + break + +def _warn_for_non_extern_non_static_global_variable(decl): + if not decl.storage: + import warnings + warnings.warn("Global variable '%s' in cdef(): for consistency " + "with C it should have a storage class specifier " + "(usually 'extern')" % (decl.name,)) + +def _remove_line_directives(csource): + # _r_line_directive matches whole lines, without the final \n, if they + # start with '#line' with some spacing allowed, or '#NUMBER'. This + # function stores them away and replaces them with exactly the string + # '#line@N', where N is the index in the list 'line_directives'. + line_directives = [] + def replace(m): + i = len(line_directives) + line_directives.append(m.group()) + return '#line@%d' % i + csource = _r_line_directive.sub(replace, csource) + return csource, line_directives + +def _put_back_line_directives(csource, line_directives): + def replace(m): + s = m.group() + if not s.startswith('#line@'): + raise AssertionError("unexpected #line directive " + "(should have been processed and removed") + return line_directives[int(s[6:])] + return _r_line_directive.sub(replace, csource) + +def _preprocess(csource): + # First, remove the lines of the form '#line N "filename"' because + # the "filename" part could confuse the rest + csource, line_directives = _remove_line_directives(csource) + # Remove comments. NOTE: this only work because the cdef() section + # should not contain any string literals (except in line directives)! + def replace_keeping_newlines(m): + return ' ' + m.group().count('\n') * '\n' + csource = _r_comment.sub(replace_keeping_newlines, csource) + # Remove the "#define FOO x" lines + macros = {} + for match in _r_define.finditer(csource): + macroname, macrovalue = match.groups() + macrovalue = macrovalue.replace('\\\n', '').strip() + macros[macroname] = macrovalue + csource = _r_define.sub('', csource) + # + if pycparser.__version__ < '2.14': + csource = _workaround_for_old_pycparser(csource) + # + # BIG HACK: replace WINAPI or __stdcall with "volatile const". + # It doesn't make sense for the return type of a function to be + # "volatile volatile const", so we abuse it to detect __stdcall... + # Hack number 2 is that "int(volatile *fptr)();" is not valid C + # syntax, so we place the "volatile" before the opening parenthesis. + csource = _r_stdcall2.sub(' volatile volatile const(', csource) + csource = _r_stdcall1.sub(' volatile volatile const ', csource) + csource = _r_cdecl.sub(' ', csource) + # + # Replace `extern "Python"` with start/end markers + csource = _preprocess_extern_python(csource) + # + # Now there should not be any string literal left; warn if we get one + _warn_for_string_literal(csource) + # + # Replace "[...]" with "[__dotdotdotarray__]" + csource = _r_partial_array.sub('[__dotdotdotarray__]', csource) + # + # Replace "...}" with "__dotdotdotNUM__}". This construction should + # occur only at the end of enums; at the end of structs we have "...;}" + # and at the end of vararg functions "...);". Also replace "=...[,}]" + # with ",__dotdotdotNUM__[,}]": this occurs in the enums too, when + # giving an unknown value. + matches = list(_r_partial_enum.finditer(csource)) + for number, match in enumerate(reversed(matches)): + p = match.start() + if csource[p] == '=': + p2 = csource.find('...', p, match.end()) + assert p2 > p + csource = '%s,__dotdotdot%d__ %s' % (csource[:p], number, + csource[p2+3:]) + else: + assert csource[p:p+3] == '...' + csource = '%s __dotdotdot%d__ %s' % (csource[:p], number, + csource[p+3:]) + # Replace "int ..." or "unsigned long int..." with "__dotdotdotint__" + csource = _r_int_dotdotdot.sub(' __dotdotdotint__ ', csource) + # Replace "float ..." or "double..." with "__dotdotdotfloat__" + csource = _r_float_dotdotdot.sub(' __dotdotdotfloat__ ', csource) + # Replace all remaining "..." with the same name, "__dotdotdot__", + # which is declared with a typedef for the purpose of C parsing. + csource = csource.replace('...', ' __dotdotdot__ ') + # Finally, put back the line directives + csource = _put_back_line_directives(csource, line_directives) + return csource, macros + +def _common_type_names(csource): + # Look in the source for what looks like usages of types from the + # list of common types. A "usage" is approximated here as the + # appearance of the word, minus a "definition" of the type, which + # is the last word in a "typedef" statement. Approximative only + # but should be fine for all the common types. + look_for_words = set(COMMON_TYPES) + look_for_words.add(';') + look_for_words.add(',') + look_for_words.add('(') + look_for_words.add(')') + look_for_words.add('typedef') + words_used = set() + is_typedef = False + paren = 0 + previous_word = '' + for word in _r_words.findall(csource): + if word in look_for_words: + if word == ';': + if is_typedef: + words_used.discard(previous_word) + look_for_words.discard(previous_word) + is_typedef = False + elif word == 'typedef': + is_typedef = True + paren = 0 + elif word == '(': + paren += 1 + elif word == ')': + paren -= 1 + elif word == ',': + if is_typedef and paren == 0: + words_used.discard(previous_word) + look_for_words.discard(previous_word) + else: # word in COMMON_TYPES + words_used.add(word) + previous_word = word + return words_used + + +class Parser: + + def __init__(self): + self._declarations = {} + self._included_declarations = set() + self._anonymous_counter = 0 + self._structnode2type = weakref.WeakKeyDictionary() + self._options = {} + self._int_constants = {} + self._recomplete = [] + self._uses_new_feature = None + + def _parse(self, csource): + csource, macros = _preprocess(csource) + # XXX: for more efficiency we would need to poke into the + # internals of CParser... the following registers the + # typedefs, because their presence or absence influences the + # parsing itself (but what they are typedef'ed to plays no role) + ctn = _common_type_names(csource) + typenames = [] + for name in sorted(self._declarations): + if name.startswith('typedef '): + name = name[8:] + typenames.append(name) + ctn.discard(name) + typenames += sorted(ctn) + # + csourcelines = [] + csourcelines.append('# 1 ""') + for typename in typenames: + csourcelines.append('typedef int %s;' % typename) + csourcelines.append('typedef int __dotdotdotint__, __dotdotdotfloat__,' + ' __dotdotdot__;') + # this forces pycparser to consider the following in the file + # called from line 1 + csourcelines.append('# 1 "%s"' % (CDEF_SOURCE_STRING,)) + csourcelines.append(csource) + csourcelines.append('') # see test_missing_newline_bug + fullcsource = '\n'.join(csourcelines) + if lock is not None: + lock.acquire() # pycparser is not thread-safe... + try: + ast = _get_parser().parse(fullcsource) + except pycparser.c_parser.ParseError as e: + self.convert_pycparser_error(e, csource) + finally: + if lock is not None: + lock.release() + # csource will be used to find buggy source text + return ast, macros, csource + + def _convert_pycparser_error(self, e, csource): + # xxx look for ":NUM:" at the start of str(e) + # and interpret that as a line number. This will not work if + # the user gives explicit ``# NUM "FILE"`` directives. + line = None + msg = str(e) + match = re.match(r"%s:(\d+):" % (CDEF_SOURCE_STRING,), msg) + if match: + linenum = int(match.group(1), 10) + csourcelines = csource.splitlines() + if 1 <= linenum <= len(csourcelines): + line = csourcelines[linenum-1] + return line + + def convert_pycparser_error(self, e, csource): + line = self._convert_pycparser_error(e, csource) + + msg = str(e) + if line: + msg = 'cannot parse "%s"\n%s' % (line.strip(), msg) + else: + msg = 'parse error\n%s' % (msg,) + raise CDefError(msg) + + def parse(self, csource, override=False, packed=False, pack=None, + dllexport=False): + if packed: + if packed != True: + raise ValueError("'packed' should be False or True; use " + "'pack' to give another value") + if pack: + raise ValueError("cannot give both 'pack' and 'packed'") + pack = 1 + elif pack: + if pack & (pack - 1): + raise ValueError("'pack' must be a power of two, not %r" % + (pack,)) + else: + pack = 0 + prev_options = self._options + try: + self._options = {'override': override, + 'packed': pack, + 'dllexport': dllexport} + self._internal_parse(csource) + finally: + self._options = prev_options + + def _internal_parse(self, csource): + ast, macros, csource = self._parse(csource) + # add the macros + self._process_macros(macros) + # find the first "__dotdotdot__" and use that as a separator + # between the repeated typedefs and the real csource + iterator = iter(ast.ext) + for decl in iterator: + if decl.name == '__dotdotdot__': + break + else: + assert 0 + current_decl = None + # + try: + self._inside_extern_python = '__cffi_extern_python_stop' + for decl in iterator: + current_decl = decl + if isinstance(decl, pycparser.c_ast.Decl): + self._parse_decl(decl) + elif isinstance(decl, pycparser.c_ast.Typedef): + if not decl.name: + raise CDefError("typedef does not declare any name", + decl) + quals = 0 + if (isinstance(decl.type.type, pycparser.c_ast.IdentifierType) and + decl.type.type.names[-1].startswith('__dotdotdot')): + realtype = self._get_unknown_type(decl) + elif (isinstance(decl.type, pycparser.c_ast.PtrDecl) and + isinstance(decl.type.type, pycparser.c_ast.TypeDecl) and + isinstance(decl.type.type.type, + pycparser.c_ast.IdentifierType) and + decl.type.type.type.names[-1].startswith('__dotdotdot')): + realtype = self._get_unknown_ptr_type(decl) + else: + realtype, quals = self._get_type_and_quals( + decl.type, name=decl.name, partial_length_ok=True, + typedef_example="*(%s *)0" % (decl.name,)) + self._declare('typedef ' + decl.name, realtype, quals=quals) + elif decl.__class__.__name__ == 'Pragma': + # skip pragma, only in pycparser 2.15 + import warnings + warnings.warn( + "#pragma in cdef() are entirely ignored. " + "They should be removed for now, otherwise your " + "code might behave differently in a future version " + "of CFFI if #pragma support gets added. Note that " + "'#pragma pack' needs to be replaced with the " + "'packed' keyword argument to cdef().") + else: + raise CDefError("unexpected <%s>: this construct is valid " + "C but not valid in cdef()" % + decl.__class__.__name__, decl) + except CDefError as e: + if len(e.args) == 1: + e.args = e.args + (current_decl,) + raise + except FFIError as e: + msg = self._convert_pycparser_error(e, csource) + if msg: + e.args = (e.args[0] + "\n *** Err: %s" % msg,) + raise + + def _add_constants(self, key, val): + if key in self._int_constants: + if self._int_constants[key] == val: + return # ignore identical double declarations + raise FFIError( + "multiple declarations of constant: %s" % (key,)) + self._int_constants[key] = val + + def _add_integer_constant(self, name, int_str): + int_str = int_str.lower().rstrip("ul") + neg = int_str.startswith('-') + if neg: + int_str = int_str[1:] + # "010" is not valid oct in py3 + if (int_str.startswith("0") and int_str != '0' + and not int_str.startswith("0x")): + int_str = "0o" + int_str[1:] + pyvalue = int(int_str, 0) + if neg: + pyvalue = -pyvalue + self._add_constants(name, pyvalue) + self._declare('macro ' + name, pyvalue) + + def _process_macros(self, macros): + for key, value in macros.items(): + value = value.strip() + if _r_int_literal.match(value): + self._add_integer_constant(key, value) + elif value == '...': + self._declare('macro ' + key, value) + else: + raise CDefError( + 'only supports one of the following syntax:\n' + ' #define %s ... (literally dot-dot-dot)\n' + ' #define %s NUMBER (with NUMBER an integer' + ' constant, decimal/hex/octal)\n' + 'got:\n' + ' #define %s %s' + % (key, key, key, value)) + + def _declare_function(self, tp, quals, decl): + tp = self._get_type_pointer(tp, quals) + if self._options.get('dllexport'): + tag = 'dllexport_python ' + elif self._inside_extern_python == '__cffi_extern_python_start': + tag = 'extern_python ' + elif self._inside_extern_python == '__cffi_extern_python_plus_c_start': + tag = 'extern_python_plus_c ' + else: + tag = 'function ' + self._declare(tag + decl.name, tp) + + def _parse_decl(self, decl): + node = decl.type + if isinstance(node, pycparser.c_ast.FuncDecl): + tp, quals = self._get_type_and_quals(node, name=decl.name) + assert isinstance(tp, model.RawFunctionType) + self._declare_function(tp, quals, decl) + else: + if isinstance(node, pycparser.c_ast.Struct): + self._get_struct_union_enum_type('struct', node) + elif isinstance(node, pycparser.c_ast.Union): + self._get_struct_union_enum_type('union', node) + elif isinstance(node, pycparser.c_ast.Enum): + self._get_struct_union_enum_type('enum', node) + elif not decl.name: + raise CDefError("construct does not declare any variable", + decl) + # + if decl.name: + tp, quals = self._get_type_and_quals(node, + partial_length_ok=True) + if tp.is_raw_function: + self._declare_function(tp, quals, decl) + elif (tp.is_integer_type() and + hasattr(decl, 'init') and + hasattr(decl.init, 'value') and + _r_int_literal.match(decl.init.value)): + self._add_integer_constant(decl.name, decl.init.value) + elif (tp.is_integer_type() and + isinstance(decl.init, pycparser.c_ast.UnaryOp) and + decl.init.op == '-' and + hasattr(decl.init.expr, 'value') and + _r_int_literal.match(decl.init.expr.value)): + self._add_integer_constant(decl.name, + '-' + decl.init.expr.value) + elif (tp is model.void_type and + decl.name.startswith('__cffi_extern_python_')): + # hack: `extern "Python"` in the C source is replaced + # with "void __cffi_extern_python_start;" and + # "void __cffi_extern_python_stop;" + self._inside_extern_python = decl.name + else: + if self._inside_extern_python !='__cffi_extern_python_stop': + raise CDefError( + "cannot declare constants or " + "variables with 'extern \"Python\"'") + if (quals & model.Q_CONST) and not tp.is_array_type: + self._declare('constant ' + decl.name, tp, quals=quals) + else: + _warn_for_non_extern_non_static_global_variable(decl) + self._declare('variable ' + decl.name, tp, quals=quals) + + def parse_type(self, cdecl): + return self.parse_type_and_quals(cdecl)[0] + + def parse_type_and_quals(self, cdecl): + ast, macros = self._parse('void __dummy(\n%s\n);' % cdecl)[:2] + assert not macros + exprnode = ast.ext[-1].type.args.params[0] + if isinstance(exprnode, pycparser.c_ast.ID): + raise CDefError("unknown identifier '%s'" % (exprnode.name,)) + return self._get_type_and_quals(exprnode.type) + + def _declare(self, name, obj, included=False, quals=0): + if name in self._declarations: + prevobj, prevquals = self._declarations[name] + if prevobj is obj and prevquals == quals: + return + if not self._options.get('override'): + raise FFIError( + "multiple declarations of %s (for interactive usage, " + "try cdef(xx, override=True))" % (name,)) + assert '__dotdotdot__' not in name.split() + self._declarations[name] = (obj, quals) + if included: + self._included_declarations.add(obj) + + def _extract_quals(self, type): + quals = 0 + if isinstance(type, (pycparser.c_ast.TypeDecl, + pycparser.c_ast.PtrDecl)): + if 'const' in type.quals: + quals |= model.Q_CONST + if 'volatile' in type.quals: + quals |= model.Q_VOLATILE + if 'restrict' in type.quals: + quals |= model.Q_RESTRICT + return quals + + def _get_type_pointer(self, type, quals, declname=None): + if isinstance(type, model.RawFunctionType): + return type.as_function_pointer() + if (isinstance(type, model.StructOrUnionOrEnum) and + type.name.startswith('$') and type.name[1:].isdigit() and + type.forcename is None and declname is not None): + return model.NamedPointerType(type, declname, quals) + return model.PointerType(type, quals) + + def _get_type_and_quals(self, typenode, name=None, partial_length_ok=False, + typedef_example=None): + # first, dereference typedefs, if we have it already parsed, we're good + if (isinstance(typenode, pycparser.c_ast.TypeDecl) and + isinstance(typenode.type, pycparser.c_ast.IdentifierType) and + len(typenode.type.names) == 1 and + ('typedef ' + typenode.type.names[0]) in self._declarations): + tp, quals = self._declarations['typedef ' + typenode.type.names[0]] + quals |= self._extract_quals(typenode) + return tp, quals + # + if isinstance(typenode, pycparser.c_ast.ArrayDecl): + # array type + if typenode.dim is None: + length = None + else: + length = self._parse_constant( + typenode.dim, partial_length_ok=partial_length_ok) + # a hack: in 'typedef int foo_t[...][...];', don't use '...' as + # the length but use directly the C expression that would be + # generated by recompiler.py. This lets the typedef be used in + # many more places within recompiler.py + if typedef_example is not None: + if length == '...': + length = '_cffi_array_len(%s)' % (typedef_example,) + typedef_example = "*" + typedef_example + # + tp, quals = self._get_type_and_quals(typenode.type, + partial_length_ok=partial_length_ok, + typedef_example=typedef_example) + return model.ArrayType(tp, length), quals + # + if isinstance(typenode, pycparser.c_ast.PtrDecl): + # pointer type + itemtype, itemquals = self._get_type_and_quals(typenode.type) + tp = self._get_type_pointer(itemtype, itemquals, declname=name) + quals = self._extract_quals(typenode) + return tp, quals + # + if isinstance(typenode, pycparser.c_ast.TypeDecl): + quals = self._extract_quals(typenode) + type = typenode.type + if isinstance(type, pycparser.c_ast.IdentifierType): + # assume a primitive type. get it from .names, but reduce + # synonyms to a single chosen combination + names = list(type.names) + if names != ['signed', 'char']: # keep this unmodified + prefixes = {} + while names: + name = names[0] + if name in ('short', 'long', 'signed', 'unsigned'): + prefixes[name] = prefixes.get(name, 0) + 1 + del names[0] + else: + break + # ignore the 'signed' prefix below, and reorder the others + newnames = [] + for prefix in ('unsigned', 'short', 'long'): + for i in range(prefixes.get(prefix, 0)): + newnames.append(prefix) + if not names: + names = ['int'] # implicitly + if names == ['int']: # but kill it if 'short' or 'long' + if 'short' in prefixes or 'long' in prefixes: + names = [] + names = newnames + names + ident = ' '.join(names) + if ident == 'void': + return model.void_type, quals + if ident == '__dotdotdot__': + raise FFIError(':%d: bad usage of "..."' % + typenode.coord.line) + tp0, quals0 = resolve_common_type(self, ident) + return tp0, (quals | quals0) + # + if isinstance(type, pycparser.c_ast.Struct): + # 'struct foobar' + tp = self._get_struct_union_enum_type('struct', type, name) + return tp, quals + # + if isinstance(type, pycparser.c_ast.Union): + # 'union foobar' + tp = self._get_struct_union_enum_type('union', type, name) + return tp, quals + # + if isinstance(type, pycparser.c_ast.Enum): + # 'enum foobar' + tp = self._get_struct_union_enum_type('enum', type, name) + return tp, quals + # + if isinstance(typenode, pycparser.c_ast.FuncDecl): + # a function type + return self._parse_function_type(typenode, name), 0 + # + # nested anonymous structs or unions end up here + if isinstance(typenode, pycparser.c_ast.Struct): + return self._get_struct_union_enum_type('struct', typenode, name, + nested=True), 0 + if isinstance(typenode, pycparser.c_ast.Union): + return self._get_struct_union_enum_type('union', typenode, name, + nested=True), 0 + # + raise FFIError(":%d: bad or unsupported type declaration" % + typenode.coord.line) + + def _parse_function_type(self, typenode, funcname=None): + params = list(getattr(typenode.args, 'params', [])) + for i, arg in enumerate(params): + if not hasattr(arg, 'type'): + raise CDefError("%s arg %d: unknown type '%s'" + " (if you meant to use the old C syntax of giving" + " untyped arguments, it is not supported)" + % (funcname or 'in expression', i + 1, + getattr(arg, 'name', '?'))) + ellipsis = ( + len(params) > 0 and + isinstance(params[-1].type, pycparser.c_ast.TypeDecl) and + isinstance(params[-1].type.type, + pycparser.c_ast.IdentifierType) and + params[-1].type.type.names == ['__dotdotdot__']) + if ellipsis: + params.pop() + if not params: + raise CDefError( + "%s: a function with only '(...)' as argument" + " is not correct C" % (funcname or 'in expression')) + args = [self._as_func_arg(*self._get_type_and_quals(argdeclnode.type)) + for argdeclnode in params] + if not ellipsis and args == [model.void_type]: + args = [] + result, quals = self._get_type_and_quals(typenode.type) + # the 'quals' on the result type are ignored. HACK: we absure them + # to detect __stdcall functions: we textually replace "__stdcall" + # with "volatile volatile const" above. + abi = None + if hasattr(typenode.type, 'quals'): # else, probable syntax error anyway + if typenode.type.quals[-3:] == ['volatile', 'volatile', 'const']: + abi = '__stdcall' + return model.RawFunctionType(tuple(args), result, ellipsis, abi) + + def _as_func_arg(self, type, quals): + if isinstance(type, model.ArrayType): + return model.PointerType(type.item, quals) + elif isinstance(type, model.RawFunctionType): + return type.as_function_pointer() + else: + return type + + def _get_struct_union_enum_type(self, kind, type, name=None, nested=False): + # First, a level of caching on the exact 'type' node of the AST. + # This is obscure, but needed because pycparser "unrolls" declarations + # such as "typedef struct { } foo_t, *foo_p" and we end up with + # an AST that is not a tree, but a DAG, with the "type" node of the + # two branches foo_t and foo_p of the trees being the same node. + # It's a bit silly but detecting "DAG-ness" in the AST tree seems + # to be the only way to distinguish this case from two independent + # structs. See test_struct_with_two_usages. + try: + return self._structnode2type[type] + except KeyError: + pass + # + # Note that this must handle parsing "struct foo" any number of + # times and always return the same StructType object. Additionally, + # one of these times (not necessarily the first), the fields of + # the struct can be specified with "struct foo { ...fields... }". + # If no name is given, then we have to create a new anonymous struct + # with no caching; in this case, the fields are either specified + # right now or never. + # + force_name = name + name = type.name + # + # get the type or create it if needed + if name is None: + # 'force_name' is used to guess a more readable name for + # anonymous structs, for the common case "typedef struct { } foo". + if force_name is not None: + explicit_name = '$%s' % force_name + else: + self._anonymous_counter += 1 + explicit_name = '$%d' % self._anonymous_counter + tp = None + else: + explicit_name = name + key = '%s %s' % (kind, name) + tp, _ = self._declarations.get(key, (None, None)) + # + if tp is None: + if kind == 'struct': + tp = model.StructType(explicit_name, None, None, None) + elif kind == 'union': + tp = model.UnionType(explicit_name, None, None, None) + elif kind == 'enum': + if explicit_name == '__dotdotdot__': + raise CDefError("Enums cannot be declared with ...") + tp = self._build_enum_type(explicit_name, type.values) + else: + raise AssertionError("kind = %r" % (kind,)) + if name is not None: + self._declare(key, tp) + elif kind == 'enum' and type.values is not None: + raise NotImplementedError( + "enum %s: the '{}' declaration should appear on the first " + "time the enum is mentioned, not later" % explicit_name) + if not tp.forcename: + tp.force_the_name(force_name) + if tp.forcename and '$' in tp.name: + self._declare('anonymous %s' % tp.forcename, tp) + # + self._structnode2type[type] = tp + # + # enums: done here + if kind == 'enum': + return tp + # + # is there a 'type.decls'? If yes, then this is the place in the + # C sources that declare the fields. If no, then just return the + # existing type, possibly still incomplete. + if type.decls is None: + return tp + # + if tp.fldnames is not None: + raise CDefError("duplicate declaration of struct %s" % name) + fldnames = [] + fldtypes = [] + fldbitsize = [] + fldquals = [] + for decl in type.decls: + if (isinstance(decl.type, pycparser.c_ast.IdentifierType) and + ''.join(decl.type.names) == '__dotdotdot__'): + # XXX pycparser is inconsistent: 'names' should be a list + # of strings, but is sometimes just one string. Use + # str.join() as a way to cope with both. + self._make_partial(tp, nested) + continue + if decl.bitsize is None: + bitsize = -1 + else: + bitsize = self._parse_constant(decl.bitsize) + self._partial_length = False + type, fqual = self._get_type_and_quals(decl.type, + partial_length_ok=True) + if self._partial_length: + self._make_partial(tp, nested) + if isinstance(type, model.StructType) and type.partial: + self._make_partial(tp, nested) + fldnames.append(decl.name or '') + fldtypes.append(type) + fldbitsize.append(bitsize) + fldquals.append(fqual) + tp.fldnames = tuple(fldnames) + tp.fldtypes = tuple(fldtypes) + tp.fldbitsize = tuple(fldbitsize) + tp.fldquals = tuple(fldquals) + if fldbitsize != [-1] * len(fldbitsize): + if isinstance(tp, model.StructType) and tp.partial: + raise NotImplementedError("%s: using both bitfields and '...;'" + % (tp,)) + tp.packed = self._options.get('packed') + if tp.completed: # must be re-completed: it is not opaque any more + tp.completed = 0 + self._recomplete.append(tp) + return tp + + def _make_partial(self, tp, nested): + if not isinstance(tp, model.StructOrUnion): + raise CDefError("%s cannot be partial" % (tp,)) + if not tp.has_c_name() and not nested: + raise NotImplementedError("%s is partial but has no C name" %(tp,)) + tp.partial = True + + def _parse_constant(self, exprnode, partial_length_ok=False): + # for now, limited to expressions that are an immediate number + # or positive/negative number + if isinstance(exprnode, pycparser.c_ast.Constant): + s = exprnode.value + if '0' <= s[0] <= '9': + s = s.rstrip('uUlL') + try: + if s.startswith('0'): + return int(s, 8) + else: + return int(s, 10) + except ValueError: + if len(s) > 1: + if s.lower()[0:2] == '0x': + return int(s, 16) + elif s.lower()[0:2] == '0b': + return int(s, 2) + raise CDefError("invalid constant %r" % (s,)) + elif s[0] == "'" and s[-1] == "'" and ( + len(s) == 3 or (len(s) == 4 and s[1] == "\\")): + return ord(s[-2]) + else: + raise CDefError("invalid constant %r" % (s,)) + # + if (isinstance(exprnode, pycparser.c_ast.UnaryOp) and + exprnode.op == '+'): + return self._parse_constant(exprnode.expr) + # + if (isinstance(exprnode, pycparser.c_ast.UnaryOp) and + exprnode.op == '-'): + return -self._parse_constant(exprnode.expr) + # load previously defined int constant + if (isinstance(exprnode, pycparser.c_ast.ID) and + exprnode.name in self._int_constants): + return self._int_constants[exprnode.name] + # + if (isinstance(exprnode, pycparser.c_ast.ID) and + exprnode.name == '__dotdotdotarray__'): + if partial_length_ok: + self._partial_length = True + return '...' + raise FFIError(":%d: unsupported '[...]' here, cannot derive " + "the actual array length in this context" + % exprnode.coord.line) + # + if isinstance(exprnode, pycparser.c_ast.BinaryOp): + left = self._parse_constant(exprnode.left) + right = self._parse_constant(exprnode.right) + if exprnode.op == '+': + return left + right + elif exprnode.op == '-': + return left - right + elif exprnode.op == '*': + return left * right + elif exprnode.op == '/': + return self._c_div(left, right) + elif exprnode.op == '%': + return left - self._c_div(left, right) * right + elif exprnode.op == '<<': + return left << right + elif exprnode.op == '>>': + return left >> right + elif exprnode.op == '&': + return left & right + elif exprnode.op == '|': + return left | right + elif exprnode.op == '^': + return left ^ right + # + raise FFIError(":%d: unsupported expression: expected a " + "simple numeric constant" % exprnode.coord.line) + + def _c_div(self, a, b): + result = a // b + if ((a < 0) ^ (b < 0)) and (a % b) != 0: + result += 1 + return result + + def _build_enum_type(self, explicit_name, decls): + if decls is not None: + partial = False + enumerators = [] + enumvalues = [] + nextenumvalue = 0 + for enum in decls.enumerators: + if _r_enum_dotdotdot.match(enum.name): + partial = True + continue + if enum.value is not None: + nextenumvalue = self._parse_constant(enum.value) + enumerators.append(enum.name) + enumvalues.append(nextenumvalue) + self._add_constants(enum.name, nextenumvalue) + nextenumvalue += 1 + enumerators = tuple(enumerators) + enumvalues = tuple(enumvalues) + tp = model.EnumType(explicit_name, enumerators, enumvalues) + tp.partial = partial + else: # opaque enum + tp = model.EnumType(explicit_name, (), ()) + return tp + + def include(self, other): + for name, (tp, quals) in other._declarations.items(): + if name.startswith('anonymous $enum_$'): + continue # fix for test_anonymous_enum_include + kind = name.split(' ', 1)[0] + if kind in ('struct', 'union', 'enum', 'anonymous', 'typedef'): + self._declare(name, tp, included=True, quals=quals) + for k, v in other._int_constants.items(): + self._add_constants(k, v) + + def _get_unknown_type(self, decl): + typenames = decl.type.type.names + if typenames == ['__dotdotdot__']: + return model.unknown_type(decl.name) + + if typenames == ['__dotdotdotint__']: + if self._uses_new_feature is None: + self._uses_new_feature = "'typedef int... %s'" % decl.name + return model.UnknownIntegerType(decl.name) + + if typenames == ['__dotdotdotfloat__']: + # note: not for 'long double' so far + if self._uses_new_feature is None: + self._uses_new_feature = "'typedef float... %s'" % decl.name + return model.UnknownFloatType(decl.name) + + raise FFIError(':%d: unsupported usage of "..." in typedef' + % decl.coord.line) + + def _get_unknown_ptr_type(self, decl): + if decl.type.type.type.names == ['__dotdotdot__']: + return model.unknown_ptr_type(decl.name) + raise FFIError(':%d: unsupported usage of "..." in typedef' + % decl.coord.line) diff --git a/py_modules/cffi/error.py b/py_modules/cffi/error.py new file mode 100644 index 0000000..0a27247 --- /dev/null +++ b/py_modules/cffi/error.py @@ -0,0 +1,31 @@ + +class FFIError(Exception): + __module__ = 'cffi' + +class CDefError(Exception): + __module__ = 'cffi' + def __str__(self): + try: + current_decl = self.args[1] + filename = current_decl.coord.file + linenum = current_decl.coord.line + prefix = '%s:%d: ' % (filename, linenum) + except (AttributeError, TypeError, IndexError): + prefix = '' + return '%s%s' % (prefix, self.args[0]) + +class VerificationError(Exception): + """ An error raised when verification fails + """ + __module__ = 'cffi' + +class VerificationMissing(Exception): + """ An error raised when incomplete structures are passed into + cdef, but no verification has been done + """ + __module__ = 'cffi' + +class PkgConfigError(Exception): + """ An error raised for missing modules in pkg-config + """ + __module__ = 'cffi' diff --git a/py_modules/cffi/ffiplatform.py b/py_modules/cffi/ffiplatform.py new file mode 100644 index 0000000..adca28f --- /dev/null +++ b/py_modules/cffi/ffiplatform.py @@ -0,0 +1,113 @@ +import sys, os +from .error import VerificationError + + +LIST_OF_FILE_NAMES = ['sources', 'include_dirs', 'library_dirs', + 'extra_objects', 'depends'] + +def get_extension(srcfilename, modname, sources=(), **kwds): + from cffi._shimmed_dist_utils import Extension + allsources = [srcfilename] + for src in sources: + allsources.append(os.path.normpath(src)) + return Extension(name=modname, sources=allsources, **kwds) + +def compile(tmpdir, ext, compiler_verbose=0, debug=None): + """Compile a C extension module using distutils.""" + + saved_environ = os.environ.copy() + try: + outputfilename = _build(tmpdir, ext, compiler_verbose, debug) + outputfilename = os.path.abspath(outputfilename) + finally: + # workaround for a distutils bugs where some env vars can + # become longer and longer every time it is used + for key, value in saved_environ.items(): + if os.environ.get(key) != value: + os.environ[key] = value + return outputfilename + +def _build(tmpdir, ext, compiler_verbose=0, debug=None): + # XXX compact but horrible :-( + from cffi._shimmed_dist_utils import Distribution, CompileError, LinkError, set_threshold, set_verbosity + + dist = Distribution({'ext_modules': [ext]}) + dist.parse_config_files() + options = dist.get_option_dict('build_ext') + if debug is None: + debug = sys.flags.debug + options['debug'] = ('ffiplatform', debug) + options['force'] = ('ffiplatform', True) + options['build_lib'] = ('ffiplatform', tmpdir) + options['build_temp'] = ('ffiplatform', tmpdir) + # + try: + old_level = set_threshold(0) or 0 + try: + set_verbosity(compiler_verbose) + dist.run_command('build_ext') + cmd_obj = dist.get_command_obj('build_ext') + [soname] = cmd_obj.get_outputs() + finally: + set_threshold(old_level) + except (CompileError, LinkError) as e: + raise VerificationError('%s: %s' % (e.__class__.__name__, e)) + # + return soname + +try: + from os.path import samefile +except ImportError: + def samefile(f1, f2): + return os.path.abspath(f1) == os.path.abspath(f2) + +def maybe_relative_path(path): + if not os.path.isabs(path): + return path # already relative + dir = path + names = [] + while True: + prevdir = dir + dir, name = os.path.split(prevdir) + if dir == prevdir or not dir: + return path # failed to make it relative + names.append(name) + try: + if samefile(dir, os.curdir): + names.reverse() + return os.path.join(*names) + except OSError: + pass + +# ____________________________________________________________ + +try: + int_or_long = (int, long) + import cStringIO +except NameError: + int_or_long = int # Python 3 + import io as cStringIO + +def _flatten(x, f): + if isinstance(x, str): + f.write('%ds%s' % (len(x), x)) + elif isinstance(x, dict): + keys = sorted(x.keys()) + f.write('%dd' % len(keys)) + for key in keys: + _flatten(key, f) + _flatten(x[key], f) + elif isinstance(x, (list, tuple)): + f.write('%dl' % len(x)) + for value in x: + _flatten(value, f) + elif isinstance(x, int_or_long): + f.write('%di' % (x,)) + else: + raise TypeError( + "the keywords to verify() contains unsupported object %r" % (x,)) + +def flatten(x): + f = cStringIO.StringIO() + _flatten(x, f) + return f.getvalue() diff --git a/py_modules/cffi/gen_src.py b/py_modules/cffi/gen_src.py new file mode 100644 index 0000000..8eb57b1 --- /dev/null +++ b/py_modules/cffi/gen_src.py @@ -0,0 +1,5 @@ +"""Entry point so ``python -m cffi.gen_src`` works.""" + +if __name__ == '__main__': + from cffi._cffi_gen_src import run + run() diff --git a/py_modules/cffi/lock.py b/py_modules/cffi/lock.py new file mode 100644 index 0000000..db91b71 --- /dev/null +++ b/py_modules/cffi/lock.py @@ -0,0 +1,30 @@ +import sys + +if sys.version_info < (3,): + try: + from thread import allocate_lock + except ImportError: + from dummy_thread import allocate_lock +else: + try: + from _thread import allocate_lock + except ImportError: + from _dummy_thread import allocate_lock + + +##import sys +##l1 = allocate_lock + +##class allocate_lock(object): +## def __init__(self): +## self._real = l1() +## def __enter__(self): +## for i in range(4, 0, -1): +## print sys._getframe(i).f_code +## print +## return self._real.__enter__() +## def __exit__(self, *args): +## return self._real.__exit__(*args) +## def acquire(self, f): +## assert f is False +## return self._real.acquire(f) diff --git a/py_modules/cffi/model.py b/py_modules/cffi/model.py new file mode 100644 index 0000000..8bade1d --- /dev/null +++ b/py_modules/cffi/model.py @@ -0,0 +1,617 @@ +import types +import weakref + +from .lock import allocate_lock +from .error import CDefError, VerificationError, VerificationMissing + +# type qualifiers +Q_CONST = 0x01 +Q_RESTRICT = 0x02 +Q_VOLATILE = 0x04 + +def qualify(quals, replace_with): + if quals & Q_CONST: + replace_with = ' const ' + replace_with.lstrip() + if quals & Q_VOLATILE: + replace_with = ' volatile ' + replace_with.lstrip() + if quals & Q_RESTRICT: + # It seems that __restrict is supported by gcc and msvc. + # If you hit some different compiler, add a #define in + # _cffi_include.h for it (and in its copies, documented there) + replace_with = ' __restrict ' + replace_with.lstrip() + return replace_with + + +class BaseTypeByIdentity: + is_array_type = False + is_raw_function = False + + def get_c_name(self, replace_with='', context='a C file', quals=0): + result = self.c_name_with_marker + assert result.count('&') == 1 + # some logic duplication with ffi.getctype()... :-( + replace_with = replace_with.strip() + if replace_with: + if replace_with.startswith('*') and '&[' in result: + replace_with = '(%s)' % replace_with + elif replace_with[0] not in '[(': + replace_with = ' ' + replace_with + replace_with = qualify(quals, replace_with) + result = result.replace('&', replace_with) + if '$' in result: + raise VerificationError( + "cannot generate '%s' in %s: unknown type name" + % (self._get_c_name(), context)) + return result + + def _get_c_name(self): + return self.c_name_with_marker.replace('&', '') + + def has_c_name(self): + return '$' not in self._get_c_name() + + def is_integer_type(self): + return False + + def get_cached_btype(self, ffi, finishlist, can_delay=False): + try: + BType = ffi._cached_btypes[self] + except KeyError: + BType = self.build_backend_type(ffi, finishlist) + BType2 = ffi._cached_btypes.setdefault(self, BType) + assert BType2 is BType + return BType + + def __repr__(self): + return '<%s>' % (self._get_c_name(),) + + def _get_items(self): + return [(name, getattr(self, name)) for name in self._attrs_] + + +class BaseType(BaseTypeByIdentity): + + def __eq__(self, other): + return (self.__class__ == other.__class__ and + self._get_items() == other._get_items()) + + def __ne__(self, other): + return not self == other + + def __hash__(self): + return hash((self.__class__, tuple(self._get_items()))) + + +class VoidType(BaseType): + _attrs_ = () + + def __init__(self): + self.c_name_with_marker = 'void&' + + def build_backend_type(self, ffi, finishlist): + return global_cache(self, ffi, 'new_void_type') + +void_type = VoidType() + + +class BasePrimitiveType(BaseType): + def is_complex_type(self): + return False + + +class PrimitiveType(BasePrimitiveType): + _attrs_ = ('name',) + + ALL_PRIMITIVE_TYPES = { + 'char': 'c', + 'short': 'i', + 'int': 'i', + 'long': 'i', + 'long long': 'i', + 'signed char': 'i', + 'unsigned char': 'i', + 'unsigned short': 'i', + 'unsigned int': 'i', + 'unsigned long': 'i', + 'unsigned long long': 'i', + 'float': 'f', + 'double': 'f', + 'long double': 'f', + '_cffi_float_complex_t': 'j', + '_cffi_double_complex_t': 'j', + '_Bool': 'i', + # the following types are not primitive in the C sense + 'wchar_t': 'c', + 'char16_t': 'c', + 'char32_t': 'c', + 'int8_t': 'i', + 'uint8_t': 'i', + 'int16_t': 'i', + 'uint16_t': 'i', + 'int32_t': 'i', + 'uint32_t': 'i', + 'int64_t': 'i', + 'uint64_t': 'i', + 'int_least8_t': 'i', + 'uint_least8_t': 'i', + 'int_least16_t': 'i', + 'uint_least16_t': 'i', + 'int_least32_t': 'i', + 'uint_least32_t': 'i', + 'int_least64_t': 'i', + 'uint_least64_t': 'i', + 'int_fast8_t': 'i', + 'uint_fast8_t': 'i', + 'int_fast16_t': 'i', + 'uint_fast16_t': 'i', + 'int_fast32_t': 'i', + 'uint_fast32_t': 'i', + 'int_fast64_t': 'i', + 'uint_fast64_t': 'i', + 'intptr_t': 'i', + 'uintptr_t': 'i', + 'intmax_t': 'i', + 'uintmax_t': 'i', + 'ptrdiff_t': 'i', + 'size_t': 'i', + 'ssize_t': 'i', + } + + def __init__(self, name): + assert name in self.ALL_PRIMITIVE_TYPES + self.name = name + self.c_name_with_marker = name + '&' + + def is_char_type(self): + return self.ALL_PRIMITIVE_TYPES[self.name] == 'c' + def is_integer_type(self): + return self.ALL_PRIMITIVE_TYPES[self.name] == 'i' + def is_float_type(self): + return self.ALL_PRIMITIVE_TYPES[self.name] == 'f' + def is_complex_type(self): + return self.ALL_PRIMITIVE_TYPES[self.name] == 'j' + + def build_backend_type(self, ffi, finishlist): + return global_cache(self, ffi, 'new_primitive_type', self.name) + + +class UnknownIntegerType(BasePrimitiveType): + _attrs_ = ('name',) + + def __init__(self, name): + self.name = name + self.c_name_with_marker = name + '&' + + def is_integer_type(self): + return True + + def build_backend_type(self, ffi, finishlist): + raise NotImplementedError("integer type '%s' can only be used after " + "compilation" % self.name) + +class UnknownFloatType(BasePrimitiveType): + _attrs_ = ('name', ) + + def __init__(self, name): + self.name = name + self.c_name_with_marker = name + '&' + + def build_backend_type(self, ffi, finishlist): + raise NotImplementedError("float type '%s' can only be used after " + "compilation" % self.name) + + +class BaseFunctionType(BaseType): + _attrs_ = ('args', 'result', 'ellipsis', 'abi') + + def __init__(self, args, result, ellipsis, abi=None): + self.args = args + self.result = result + self.ellipsis = ellipsis + self.abi = abi + # + reprargs = [arg._get_c_name() for arg in self.args] + if self.ellipsis: + reprargs.append('...') + reprargs = reprargs or ['void'] + replace_with = self._base_pattern % (', '.join(reprargs),) + if abi is not None: + replace_with = replace_with[:1] + abi + ' ' + replace_with[1:] + self.c_name_with_marker = ( + self.result.c_name_with_marker.replace('&', replace_with)) + + +class RawFunctionType(BaseFunctionType): + # Corresponds to a C type like 'int(int)', which is the C type of + # a function, but not a pointer-to-function. The backend has no + # notion of such a type; it's used temporarily by parsing. + _base_pattern = '(&)(%s)' + is_raw_function = True + + def build_backend_type(self, ffi, finishlist): + raise CDefError("cannot render the type %r: it is a function " + "type, not a pointer-to-function type" % (self,)) + + def as_function_pointer(self): + return FunctionPtrType(self.args, self.result, self.ellipsis, self.abi) + + +class FunctionPtrType(BaseFunctionType): + _base_pattern = '(*&)(%s)' + + def build_backend_type(self, ffi, finishlist): + result = self.result.get_cached_btype(ffi, finishlist) + args = [] + for tp in self.args: + args.append(tp.get_cached_btype(ffi, finishlist)) + abi_args = () + if self.abi == "__stdcall": + if not self.ellipsis: # __stdcall ignored for variadic funcs + try: + abi_args = (ffi._backend.FFI_STDCALL,) + except AttributeError: + pass + return global_cache(self, ffi, 'new_function_type', + tuple(args), result, self.ellipsis, *abi_args) + + def as_raw_function(self): + return RawFunctionType(self.args, self.result, self.ellipsis, self.abi) + + +class PointerType(BaseType): + _attrs_ = ('totype', 'quals') + + def __init__(self, totype, quals=0): + self.totype = totype + self.quals = quals + extra = " *&" + if totype.is_array_type: + extra = "(%s)" % (extra.lstrip(),) + extra = qualify(quals, extra) + self.c_name_with_marker = totype.c_name_with_marker.replace('&', extra) + + def build_backend_type(self, ffi, finishlist): + BItem = self.totype.get_cached_btype(ffi, finishlist, can_delay=True) + return global_cache(self, ffi, 'new_pointer_type', BItem) + +voidp_type = PointerType(void_type) + +def ConstPointerType(totype): + return PointerType(totype, Q_CONST) + +const_voidp_type = ConstPointerType(void_type) + + +class NamedPointerType(PointerType): + _attrs_ = ('totype', 'name') + + def __init__(self, totype, name, quals=0): + PointerType.__init__(self, totype, quals) + self.name = name + self.c_name_with_marker = name + '&' + + +class ArrayType(BaseType): + _attrs_ = ('item', 'length') + is_array_type = True + + def __init__(self, item, length): + self.item = item + self.length = length + # + if length is None: + brackets = '&[]' + elif length == '...': + brackets = '&[/*...*/]' + else: + brackets = '&[%s]' % length + self.c_name_with_marker = ( + self.item.c_name_with_marker.replace('&', brackets)) + + def length_is_unknown(self): + return isinstance(self.length, str) + + def resolve_length(self, newlength): + return ArrayType(self.item, newlength) + + def build_backend_type(self, ffi, finishlist): + if self.length_is_unknown(): + raise CDefError("cannot render the type %r: unknown length" % + (self,)) + self.item.get_cached_btype(ffi, finishlist) # force the item BType + BPtrItem = PointerType(self.item).get_cached_btype(ffi, finishlist) + return global_cache(self, ffi, 'new_array_type', BPtrItem, self.length) + +char_array_type = ArrayType(PrimitiveType('char'), None) + + +class StructOrUnionOrEnum(BaseTypeByIdentity): + _attrs_ = ('name',) + forcename = None + + def build_c_name_with_marker(self): + name = self.forcename or '%s %s' % (self.kind, self.name) + self.c_name_with_marker = name + '&' + + def force_the_name(self, forcename): + self.forcename = forcename + self.build_c_name_with_marker() + + def get_official_name(self): + assert self.c_name_with_marker.endswith('&') + return self.c_name_with_marker[:-1] + + +class StructOrUnion(StructOrUnionOrEnum): + fixedlayout = None + completed = 0 + partial = False + packed = 0 + + def __init__(self, name, fldnames, fldtypes, fldbitsize, fldquals=None): + self.name = name + self.fldnames = fldnames + self.fldtypes = fldtypes + self.fldbitsize = fldbitsize + self.fldquals = fldquals + self.build_c_name_with_marker() + + def anonymous_struct_fields(self): + if self.fldtypes is not None: + for name, type in zip(self.fldnames, self.fldtypes): + if name == '' and isinstance(type, StructOrUnion): + yield type + + def enumfields(self, expand_anonymous_struct_union=True): + fldquals = self.fldquals + if fldquals is None: + fldquals = (0,) * len(self.fldnames) + for name, type, bitsize, quals in zip(self.fldnames, self.fldtypes, + self.fldbitsize, fldquals): + if (name == '' and isinstance(type, StructOrUnion) + and expand_anonymous_struct_union): + # nested anonymous struct/union + yield from type.enumfields() + else: + yield (name, type, bitsize, quals) + + def force_flatten(self): + # force the struct or union to have a declaration that lists + # directly all fields returned by enumfields(), flattening + # nested anonymous structs/unions. + names = [] + types = [] + bitsizes = [] + fldquals = [] + for name, type, bitsize, quals in self.enumfields(): + names.append(name) + types.append(type) + bitsizes.append(bitsize) + fldquals.append(quals) + self.fldnames = tuple(names) + self.fldtypes = tuple(types) + self.fldbitsize = tuple(bitsizes) + self.fldquals = tuple(fldquals) + + def get_cached_btype(self, ffi, finishlist, can_delay=False): + BType = StructOrUnionOrEnum.get_cached_btype(self, ffi, finishlist, + can_delay) + if not can_delay: + self.finish_backend_type(ffi, finishlist) + return BType + + def finish_backend_type(self, ffi, finishlist): + if self.completed: + if self.completed != 2: + raise NotImplementedError("recursive structure declaration " + "for '%s'" % (self.name,)) + return + BType = ffi._cached_btypes[self] + # + self.completed = 1 + # + if self.fldtypes is None: + pass # not completing it: it's an opaque struct + # + elif self.fixedlayout is None: + fldtypes = [tp.get_cached_btype(ffi, finishlist) + for tp in self.fldtypes] + lst = list(zip(self.fldnames, fldtypes, self.fldbitsize)) + extra_flags = () + if self.packed: + if self.packed == 1: + extra_flags = (8,) # SF_PACKED + else: + extra_flags = (0, self.packed) + ffi._backend.complete_struct_or_union(BType, lst, self, + -1, -1, *extra_flags) + # + else: + fldtypes = [] + fieldofs, fieldsize, totalsize, totalalignment = self.fixedlayout + for i in range(len(self.fldnames)): + fsize = fieldsize[i] + ftype = self.fldtypes[i] + # + if isinstance(ftype, ArrayType) and ftype.length_is_unknown(): + # fix the length to match the total size + BItemType = ftype.item.get_cached_btype(ffi, finishlist) + nlen, nrest = divmod(fsize, ffi.sizeof(BItemType)) + if nrest != 0: + self._verification_error( + "field '%s.%s' has a bogus size?" % ( + self.name, self.fldnames[i] or '{}')) + ftype = ftype.resolve_length(nlen) + self.fldtypes = (self.fldtypes[:i] + (ftype,) + + self.fldtypes[i+1:]) + # + BFieldType = ftype.get_cached_btype(ffi, finishlist) + if isinstance(ftype, ArrayType) and ftype.length is None: + assert fsize == 0 + else: + bitemsize = ffi.sizeof(BFieldType) + if bitemsize != fsize: + self._verification_error( + "field '%s.%s' is declared as %d bytes, but is " + "really %d bytes" % (self.name, + self.fldnames[i] or '{}', + bitemsize, fsize)) + fldtypes.append(BFieldType) + # + lst = list(zip(self.fldnames, fldtypes, self.fldbitsize, fieldofs)) + ffi._backend.complete_struct_or_union(BType, lst, self, + totalsize, totalalignment) + self.completed = 2 + + def _verification_error(self, msg): + raise VerificationError(msg) + + def check_not_partial(self): + if self.partial and self.fixedlayout is None: + raise VerificationMissing(self._get_c_name()) + + def build_backend_type(self, ffi, finishlist): + self.check_not_partial() + finishlist.append(self) + # + return global_cache(self, ffi, 'new_%s_type' % self.kind, + self.get_official_name(), key=self) + + +class StructType(StructOrUnion): + kind = 'struct' + + +class UnionType(StructOrUnion): + kind = 'union' + + +class EnumType(StructOrUnionOrEnum): + kind = 'enum' + partial = False + partial_resolved = False + + def __init__(self, name, enumerators, enumvalues, baseinttype=None): + self.name = name + self.enumerators = enumerators + self.enumvalues = enumvalues + self.baseinttype = baseinttype + self.build_c_name_with_marker() + + def force_the_name(self, forcename): + StructOrUnionOrEnum.force_the_name(self, forcename) + if self.forcename is None: + name = self.get_official_name() + self.forcename = '$' + name.replace(' ', '_') + + def check_not_partial(self): + if self.partial and not self.partial_resolved: + raise VerificationMissing(self._get_c_name()) + + def build_backend_type(self, ffi, finishlist): + self.check_not_partial() + base_btype = self.build_baseinttype(ffi, finishlist) + return global_cache(self, ffi, 'new_enum_type', + self.get_official_name(), + self.enumerators, self.enumvalues, + base_btype, key=self) + + def build_baseinttype(self, ffi, finishlist): + if self.baseinttype is not None: + return self.baseinttype.get_cached_btype(ffi, finishlist) + # + if self.enumvalues: + smallest_value = min(self.enumvalues) + largest_value = max(self.enumvalues) + else: + import warnings + try: + # XXX! The goal is to ensure that the warnings.warn() + # will not suppress the warning. We want to get it + # several times if we reach this point several times. + __warningregistry__.clear() + except NameError: + pass + warnings.warn("%r has no values explicitly defined; " + "guessing that it is equivalent to 'unsigned int'" + % self._get_c_name()) + smallest_value = largest_value = 0 + if smallest_value < 0: # needs a signed type + sign = 1 + candidate1 = PrimitiveType("int") + candidate2 = PrimitiveType("long") + else: + sign = 0 + candidate1 = PrimitiveType("unsigned int") + candidate2 = PrimitiveType("unsigned long") + btype1 = candidate1.get_cached_btype(ffi, finishlist) + btype2 = candidate2.get_cached_btype(ffi, finishlist) + size1 = ffi.sizeof(btype1) + size2 = ffi.sizeof(btype2) + if (smallest_value >= ((-1) << (8*size1-1)) and + largest_value < (1 << (8*size1-sign))): + return btype1 + if (smallest_value >= ((-1) << (8*size2-1)) and + largest_value < (1 << (8*size2-sign))): + return btype2 + raise CDefError("%s values don't all fit into either 'long' " + "or 'unsigned long'" % self._get_c_name()) + +def unknown_type(name, structname=None): + if structname is None: + structname = '$%s' % name + tp = StructType(structname, None, None, None) + tp.force_the_name(name) + tp.origin = "unknown_type" + return tp + +def unknown_ptr_type(name, structname=None): + if structname is None: + structname = '$$%s' % name + tp = StructType(structname, None, None, None) + return NamedPointerType(tp, name) + + +global_lock = allocate_lock() +_typecache_cffi_backend = weakref.WeakValueDictionary() + +def get_typecache(backend): + # returns _typecache_cffi_backend if backend is the _cffi_backend + # module, or type(backend).__typecache if backend is an instance of + # CTypesBackend (or some FakeBackend class during tests) + if isinstance(backend, types.ModuleType): + return _typecache_cffi_backend + with global_lock: + if not hasattr(type(backend), '__typecache'): + type(backend).__typecache = weakref.WeakValueDictionary() + return type(backend).__typecache + +def global_cache(srctype, ffi, funcname, *args, **kwds): + key = kwds.pop('key', (funcname, args)) + assert not kwds + try: + return ffi._typecache[key] + except KeyError: + pass + try: + res = getattr(ffi._backend, funcname)(*args) + except NotImplementedError as e: + raise NotImplementedError("%s: %r: %s" % (funcname, srctype, e)) + # note that setdefault() on WeakValueDictionary is not atomic + # and contains a rare bug (http://bugs.python.org/issue19542); + # we have to use a lock and do it ourselves + cache = ffi._typecache + with global_lock: + res1 = cache.get(key) + if res1 is None: + cache[key] = res + return res + else: + return res1 + +def pointer_cache(ffi, BType): + return global_cache('?', ffi, 'new_pointer_type', BType) + +def attach_exception_info(e, name): + if e.args and type(e.args[0]) is str: + e.args = ('%s: %s' % (name, e.args[0]),) + e.args[1:] diff --git a/py_modules/cffi/parse_c_type.h b/py_modules/cffi/parse_c_type.h new file mode 100644 index 0000000..84e4ef8 --- /dev/null +++ b/py_modules/cffi/parse_c_type.h @@ -0,0 +1,181 @@ + +/* This part is from file 'cffi/parse_c_type.h'. It is copied at the + beginning of C sources generated by CFFI's ffi.set_source(). */ + +typedef void *_cffi_opcode_t; + +#define _CFFI_OP(opcode, arg) (_cffi_opcode_t)(opcode | (((uintptr_t)(arg)) << 8)) +#define _CFFI_GETOP(cffi_opcode) ((unsigned char)(uintptr_t)cffi_opcode) +#define _CFFI_GETARG(cffi_opcode) (((intptr_t)cffi_opcode) >> 8) + +#define _CFFI_OP_PRIMITIVE 1 +#define _CFFI_OP_POINTER 3 +#define _CFFI_OP_ARRAY 5 +#define _CFFI_OP_OPEN_ARRAY 7 +#define _CFFI_OP_STRUCT_UNION 9 +#define _CFFI_OP_ENUM 11 +#define _CFFI_OP_FUNCTION 13 +#define _CFFI_OP_FUNCTION_END 15 +#define _CFFI_OP_NOOP 17 +#define _CFFI_OP_BITFIELD 19 +#define _CFFI_OP_TYPENAME 21 +#define _CFFI_OP_CPYTHON_BLTN_V 23 // varargs +#define _CFFI_OP_CPYTHON_BLTN_N 25 // noargs +#define _CFFI_OP_CPYTHON_BLTN_O 27 // O (i.e. a single arg) +#define _CFFI_OP_CONSTANT 29 +#define _CFFI_OP_CONSTANT_INT 31 +#define _CFFI_OP_GLOBAL_VAR 33 +#define _CFFI_OP_DLOPEN_FUNC 35 +#define _CFFI_OP_DLOPEN_CONST 37 +#define _CFFI_OP_GLOBAL_VAR_F 39 +#define _CFFI_OP_EXTERN_PYTHON 41 + +#define _CFFI_PRIM_VOID 0 +#define _CFFI_PRIM_BOOL 1 +#define _CFFI_PRIM_CHAR 2 +#define _CFFI_PRIM_SCHAR 3 +#define _CFFI_PRIM_UCHAR 4 +#define _CFFI_PRIM_SHORT 5 +#define _CFFI_PRIM_USHORT 6 +#define _CFFI_PRIM_INT 7 +#define _CFFI_PRIM_UINT 8 +#define _CFFI_PRIM_LONG 9 +#define _CFFI_PRIM_ULONG 10 +#define _CFFI_PRIM_LONGLONG 11 +#define _CFFI_PRIM_ULONGLONG 12 +#define _CFFI_PRIM_FLOAT 13 +#define _CFFI_PRIM_DOUBLE 14 +#define _CFFI_PRIM_LONGDOUBLE 15 + +#define _CFFI_PRIM_WCHAR 16 +#define _CFFI_PRIM_INT8 17 +#define _CFFI_PRIM_UINT8 18 +#define _CFFI_PRIM_INT16 19 +#define _CFFI_PRIM_UINT16 20 +#define _CFFI_PRIM_INT32 21 +#define _CFFI_PRIM_UINT32 22 +#define _CFFI_PRIM_INT64 23 +#define _CFFI_PRIM_UINT64 24 +#define _CFFI_PRIM_INTPTR 25 +#define _CFFI_PRIM_UINTPTR 26 +#define _CFFI_PRIM_PTRDIFF 27 +#define _CFFI_PRIM_SIZE 28 +#define _CFFI_PRIM_SSIZE 29 +#define _CFFI_PRIM_INT_LEAST8 30 +#define _CFFI_PRIM_UINT_LEAST8 31 +#define _CFFI_PRIM_INT_LEAST16 32 +#define _CFFI_PRIM_UINT_LEAST16 33 +#define _CFFI_PRIM_INT_LEAST32 34 +#define _CFFI_PRIM_UINT_LEAST32 35 +#define _CFFI_PRIM_INT_LEAST64 36 +#define _CFFI_PRIM_UINT_LEAST64 37 +#define _CFFI_PRIM_INT_FAST8 38 +#define _CFFI_PRIM_UINT_FAST8 39 +#define _CFFI_PRIM_INT_FAST16 40 +#define _CFFI_PRIM_UINT_FAST16 41 +#define _CFFI_PRIM_INT_FAST32 42 +#define _CFFI_PRIM_UINT_FAST32 43 +#define _CFFI_PRIM_INT_FAST64 44 +#define _CFFI_PRIM_UINT_FAST64 45 +#define _CFFI_PRIM_INTMAX 46 +#define _CFFI_PRIM_UINTMAX 47 +#define _CFFI_PRIM_FLOATCOMPLEX 48 +#define _CFFI_PRIM_DOUBLECOMPLEX 49 +#define _CFFI_PRIM_CHAR16 50 +#define _CFFI_PRIM_CHAR32 51 + +#define _CFFI__NUM_PRIM 52 +#define _CFFI__UNKNOWN_PRIM (-1) +#define _CFFI__UNKNOWN_FLOAT_PRIM (-2) +#define _CFFI__UNKNOWN_LONG_DOUBLE (-3) + +#define _CFFI__IO_FILE_STRUCT (-1) + + +struct _cffi_global_s { + const char *name; + void *address; + _cffi_opcode_t type_op; + void *size_or_direct_fn; // OP_GLOBAL_VAR: size, or 0 if unknown + // OP_CPYTHON_BLTN_*: addr of direct function +}; + +struct _cffi_getconst_s { + unsigned long long value; + const struct _cffi_type_context_s *ctx; + int gindex; +}; + +struct _cffi_struct_union_s { + const char *name; + int type_index; // -> _cffi_types, on a OP_STRUCT_UNION + int flags; // _CFFI_F_* flags below + size_t size; + int alignment; + int first_field_index; // -> _cffi_fields array + int num_fields; +}; +#define _CFFI_F_UNION 0x01 // is a union, not a struct +#define _CFFI_F_CHECK_FIELDS 0x02 // complain if fields are not in the + // "standard layout" or if some are missing +#define _CFFI_F_PACKED 0x04 // for CHECK_FIELDS, assume a packed struct +#define _CFFI_F_EXTERNAL 0x08 // in some other ffi.include() +#define _CFFI_F_OPAQUE 0x10 // opaque + +struct _cffi_field_s { + const char *name; + size_t field_offset; + size_t field_size; + _cffi_opcode_t field_type_op; +}; + +struct _cffi_enum_s { + const char *name; + int type_index; // -> _cffi_types, on a OP_ENUM + int type_prim; // _CFFI_PRIM_xxx + const char *enumerators; // comma-delimited string +}; + +struct _cffi_typename_s { + const char *name; + int type_index; /* if opaque, points to a possibly artificial + OP_STRUCT which is itself opaque */ +}; + +struct _cffi_type_context_s { + _cffi_opcode_t *types; + const struct _cffi_global_s *globals; + const struct _cffi_field_s *fields; + const struct _cffi_struct_union_s *struct_unions; + const struct _cffi_enum_s *enums; + const struct _cffi_typename_s *typenames; + int num_globals; + int num_struct_unions; + int num_enums; + int num_typenames; + const char *const *includes; + int num_types; + int flags; /* future extension */ +}; + +struct _cffi_parse_info_s { + const struct _cffi_type_context_s *ctx; + _cffi_opcode_t *output; + unsigned int output_size; + size_t error_location; + const char *error_message; +}; + +struct _cffi_externpy_s { + const char *name; + size_t size_of_result; + void *reserved1, *reserved2; +}; + +#ifdef _CFFI_INTERNAL +static int parse_c_type(struct _cffi_parse_info_s *info, const char *input); +static int search_in_globals(const struct _cffi_type_context_s *ctx, + const char *search, size_t search_len); +static int search_in_struct_unions(const struct _cffi_type_context_s *ctx, + const char *search, size_t search_len); +#endif diff --git a/py_modules/cffi/pkgconfig.py b/py_modules/cffi/pkgconfig.py new file mode 100644 index 0000000..e1034ad --- /dev/null +++ b/py_modules/cffi/pkgconfig.py @@ -0,0 +1,121 @@ +# pkg-config, https://www.freedesktop.org/wiki/Software/pkg-config/ integration for cffi +import sys, os, subprocess + +from .error import PkgConfigError + + +def merge_flags(cfg1, cfg2): + """Merge values from cffi config flags cfg2 to cf1 + + Example: + merge_flags({"libraries": ["one"]}, {"libraries": ["two"]}) + {"libraries": ["one", "two"]} + """ + for key, value in cfg2.items(): + if key not in cfg1: + cfg1[key] = value + else: + if not isinstance(cfg1[key], list): + raise TypeError("cfg1[%r] should be a list of strings" % (key,)) + if not isinstance(value, list): + raise TypeError("cfg2[%r] should be a list of strings" % (key,)) + cfg1[key].extend(value) + return cfg1 + + +def call(libname, flag, encoding=sys.getfilesystemencoding()): + """Calls pkg-config and returns the output if found + """ + a = ["pkg-config", "--print-errors"] + a.append(flag) + a.append(libname) + try: + pc = subprocess.Popen(a, stdout=subprocess.PIPE, stderr=subprocess.PIPE) + except OSError as e: + raise PkgConfigError("cannot run pkg-config: %s" % (str(e).strip(),)) + + bout, berr = pc.communicate() + if pc.returncode != 0: + try: + berr = berr.decode(encoding) + except Exception: + pass + raise PkgConfigError(berr.strip()) + + if sys.version_info >= (3,) and not isinstance(bout, str): # Python 3.x + try: + bout = bout.decode(encoding) + except UnicodeDecodeError: + raise PkgConfigError("pkg-config %s %s returned bytes that cannot " + "be decoded with encoding %r:\n%r" % + (flag, libname, encoding, bout)) + + if os.altsep != '\\' and '\\' in bout: + raise PkgConfigError("pkg-config %s %s returned an unsupported " + "backslash-escaped output:\n%r" % + (flag, libname, bout)) + return bout + + +def flags_from_pkgconfig(libs): + r"""Return compiler line flags for FFI.set_source based on pkg-config output + + Usage + ... + ffibuilder.set_source("_foo", pkgconfig = ["libfoo", "libbar >= 1.8.3"]) + + If pkg-config is installed on build machine, then arguments include_dirs, + library_dirs, libraries, define_macros, extra_compile_args and + extra_link_args are extended with an output of pkg-config for libfoo and + libbar. + + Raises PkgConfigError in case the pkg-config call fails. + """ + + def get_include_dirs(string): + return [x[2:] for x in string.split() if x.startswith("-I")] + + def get_library_dirs(string): + return [x[2:] for x in string.split() if x.startswith("-L")] + + def get_libraries(string): + return [x[2:] for x in string.split() if x.startswith("-l")] + + # convert -Dfoo=bar to list of tuples [("foo", "bar")] expected by distutils + def get_macros(string): + def _macro(x): + x = x[2:] # drop "-D" + if '=' in x: + return tuple(x.split("=", 1)) # "-Dfoo=bar" => ("foo", "bar") + else: + return (x, None) # "-Dfoo" => ("foo", None) + return [_macro(x) for x in string.split() if x.startswith("-D")] + + def get_other_cflags(string): + return [x for x in string.split() if not x.startswith("-I") and + not x.startswith("-D")] + + def get_other_libs(string): + return [x for x in string.split() if not x.startswith("-L") and + not x.startswith("-l")] + + # return kwargs for given libname + def kwargs(libname): + fse = sys.getfilesystemencoding() + all_cflags = call(libname, "--cflags") + all_libs = call(libname, "--libs") + return { + "include_dirs": get_include_dirs(all_cflags), + "library_dirs": get_library_dirs(all_libs), + "libraries": get_libraries(all_libs), + "define_macros": get_macros(all_cflags), + "extra_compile_args": get_other_cflags(all_cflags), + "extra_link_args": get_other_libs(all_libs), + } + + # merge all arguments together + ret = {} + for libname in libs: + lib_flags = kwargs(libname) + merge_flags(ret, lib_flags) + return ret diff --git a/py_modules/cffi/recompiler.py b/py_modules/cffi/recompiler.py new file mode 100644 index 0000000..5e71aed --- /dev/null +++ b/py_modules/cffi/recompiler.py @@ -0,0 +1,1586 @@ +import io, os, sys, sysconfig +from . import ffiplatform, model +from .error import VerificationError +from .cffi_opcode import * + +VERSION_BASE = 0x2601 +VERSION_EMBEDDED = 0x2701 +VERSION_CHAR16CHAR32 = 0x2801 + +FREE_THREADED_BUILD = sysconfig.get_config_var("Py_GIL_DISABLED") +USE_LIMITED_API = ((sys.platform != 'win32' or sys.version_info < (3, 0) or + sys.version_info >= (3, 5)) and + # free-threaded build doesn't support the stable ABI until Python 3.15 + (not FREE_THREADED_BUILD or sys.version_info >= (3, 15))) + +class GlobalExpr: + def __init__(self, name, address, type_op, size=0, check_value=0): + self.name = name + self.address = address + self.type_op = type_op + self.size = size + self.check_value = check_value + + def as_c_expr(self): + return ' { "%s", (void *)%s, %s, (void *)%s },' % ( + self.name, self.address, self.type_op.as_c_expr(), self.size) + + def as_python_expr(self): + return "b'%s%s',%d" % (self.type_op.as_python_bytes(), self.name, + self.check_value) + +class FieldExpr: + def __init__(self, name, field_offset, field_size, fbitsize, field_type_op): + self.name = name + self.field_offset = field_offset + self.field_size = field_size + self.fbitsize = fbitsize + self.field_type_op = field_type_op + + def as_c_expr(self): + spaces = " " * len(self.name) + return (' { "%s", %s,\n' % (self.name, self.field_offset) + + ' %s %s,\n' % (spaces, self.field_size) + + ' %s %s },' % (spaces, self.field_type_op.as_c_expr())) + + def as_python_expr(self): + raise NotImplementedError + + def as_field_python_expr(self): + if self.field_type_op.op == OP_NOOP: + size_expr = '' + elif self.field_type_op.op == OP_BITFIELD: + size_expr = format_four_bytes(self.fbitsize) + else: + raise NotImplementedError + return "b'%s%s%s'" % (self.field_type_op.as_python_bytes(), + size_expr, + self.name) + +class StructUnionExpr: + def __init__(self, name, type_index, flags, size, alignment, comment, + first_field_index, c_fields): + self.name = name + self.type_index = type_index + self.flags = flags + self.size = size + self.alignment = alignment + self.comment = comment + self.first_field_index = first_field_index + self.c_fields = c_fields + + def as_c_expr(self): + return (' { "%s", %d, %s,' % (self.name, self.type_index, self.flags) + + '\n %s, %s, ' % (self.size, self.alignment) + + '%d, %d ' % (self.first_field_index, len(self.c_fields)) + + ('/* %s */ ' % self.comment if self.comment else '') + + '},') + + def as_python_expr(self): + flags = eval(self.flags, G_FLAGS) + fields_expr = [c_field.as_field_python_expr() + for c_field in self.c_fields] + return "(b'%s%s%s',%s)" % ( + format_four_bytes(self.type_index), + format_four_bytes(flags), + self.name, + ','.join(fields_expr)) + +class EnumExpr: + def __init__(self, name, type_index, size, signed, allenums): + self.name = name + self.type_index = type_index + self.size = size + self.signed = signed + self.allenums = allenums + + def as_c_expr(self): + lines = [' { "%s", %d, _cffi_prim_int(%s, %s),' % (self.name, self.type_index, + self.size, self.signed,)] + pending = 0 + while len(self.allenums) > pending + 110: + j = self.allenums.find(',', pending + 100) + if j < 0: + break + j += 1 + lines.append(' "%s"' % (self.allenums[pending:j],)) + pending = j + lines.append(' "%s" },' % (self.allenums[pending:],)) + return '\n'.join(lines) + + def as_python_expr(self): + prim_index = { + (1, 0): PRIM_UINT8, (1, 1): PRIM_INT8, + (2, 0): PRIM_UINT16, (2, 1): PRIM_INT16, + (4, 0): PRIM_UINT32, (4, 1): PRIM_INT32, + (8, 0): PRIM_UINT64, (8, 1): PRIM_INT64, + }[self.size, self.signed] + return "b'%s%s%s\\x00%s'" % (format_four_bytes(self.type_index), + format_four_bytes(prim_index), + self.name, self.allenums) + +class TypenameExpr: + def __init__(self, name, type_index): + self.name = name + self.type_index = type_index + + def as_c_expr(self): + return ' { "%s", %d },' % (self.name, self.type_index) + + def as_python_expr(self): + return "b'%s%s'" % (format_four_bytes(self.type_index), self.name) + + +# ____________________________________________________________ + + +class Recompiler: + _num_externpy = 0 + + def __init__(self, ffi, module_name, target_is_python=False): + self.ffi = ffi + self.module_name = module_name + self.target_is_python = target_is_python + self._version = VERSION_BASE + + def needs_version(self, ver): + self._version = max(self._version, ver) + + def collect_type_table(self): + self._typesdict = {} + self._generate("collecttype") + # + all_decls = sorted(self._typesdict, key=str) + # + # prepare all FUNCTION bytecode sequences first + self.cffi_types = [] + for tp in all_decls: + if tp.is_raw_function: + assert self._typesdict[tp] is None + self._typesdict[tp] = len(self.cffi_types) + self.cffi_types.append(tp) # placeholder + for tp1 in tp.args: + assert isinstance(tp1, (model.VoidType, + model.BasePrimitiveType, + model.PointerType, + model.StructOrUnionOrEnum, + model.FunctionPtrType)) + if self._typesdict[tp1] is None: + self._typesdict[tp1] = len(self.cffi_types) + self.cffi_types.append(tp1) # placeholder + self.cffi_types.append('END') # placeholder + # + # prepare all OTHER bytecode sequences + for tp in all_decls: + if not tp.is_raw_function and self._typesdict[tp] is None: + self._typesdict[tp] = len(self.cffi_types) + self.cffi_types.append(tp) # placeholder + if tp.is_array_type and tp.length is not None: + self.cffi_types.append('LEN') # placeholder + assert None not in self._typesdict.values() + # + # collect all structs and unions and enums + self._struct_unions = {} + self._enums = {} + for tp in all_decls: + if isinstance(tp, model.StructOrUnion): + self._struct_unions[tp] = None + elif isinstance(tp, model.EnumType): + self._enums[tp] = None + for i, tp in enumerate(sorted(self._struct_unions, + key=lambda tp: tp.name)): + self._struct_unions[tp] = i + for i, tp in enumerate(sorted(self._enums, + key=lambda tp: tp.name)): + self._enums[tp] = i + # + # emit all bytecode sequences now + for tp in all_decls: + method = getattr(self, '_emit_bytecode_' + tp.__class__.__name__) + method(tp, self._typesdict[tp]) + # + # consistency check + for op in self.cffi_types: + assert isinstance(op, CffiOp) + self.cffi_types = tuple(self.cffi_types) # don't change any more + + def _enum_fields(self, tp): + # When producing C, expand all anonymous struct/union fields. + # That's necessary to have C code checking the offsets of the + # individual fields contained in them. When producing Python, + # don't do it and instead write it like it is, with the + # corresponding fields having an empty name. Empty names are + # recognized at runtime when we import the generated Python + # file. + expand_anonymous_struct_union = not self.target_is_python + return tp.enumfields(expand_anonymous_struct_union) + + def _do_collect_type(self, tp): + if not isinstance(tp, model.BaseTypeByIdentity): + if isinstance(tp, tuple): + for x in tp: + self._do_collect_type(x) + return + if tp not in self._typesdict: + self._typesdict[tp] = None + if isinstance(tp, model.FunctionPtrType): + self._do_collect_type(tp.as_raw_function()) + elif isinstance(tp, model.StructOrUnion): + if tp.fldtypes is not None and ( + tp not in self.ffi._parser._included_declarations): + for name1, tp1, _, _ in self._enum_fields(tp): + self._do_collect_type(self._field_type(tp, name1, tp1)) + else: + for _, x in tp._get_items(): + self._do_collect_type(x) + + def _generate(self, step_name): + lst = self.ffi._parser._declarations.items() + for name, (tp, quals) in sorted(lst): + kind, realname = name.split(' ', 1) + try: + method = getattr(self, '_generate_cpy_%s_%s' % (kind, + step_name)) + except AttributeError: + raise VerificationError( + "not implemented in recompile(): %r" % name) + try: + self._current_quals = quals + method(tp, realname) + except Exception as e: + model.attach_exception_info(e, name) + raise + + # ---------- + + ALL_STEPS = ["global", "field", "struct_union", "enum", "typename"] + + def collect_step_tables(self): + # collect the declarations for '_cffi_globals', '_cffi_typenames', etc. + self._lsts = {} + for step_name in self.ALL_STEPS: + self._lsts[step_name] = [] + self._seen_struct_unions = set() + self._generate("ctx") + self._add_missing_struct_unions() + # + for step_name in self.ALL_STEPS: + lst = self._lsts[step_name] + if step_name != "field": + lst.sort(key=lambda entry: entry.name) + self._lsts[step_name] = tuple(lst) # don't change any more + # + # check for a possible internal inconsistency: _cffi_struct_unions + # should have been generated with exactly self._struct_unions + lst = self._lsts["struct_union"] + for tp, i in self._struct_unions.items(): + assert i < len(lst) + assert lst[i].name == tp.name + assert len(lst) == len(self._struct_unions) + # same with enums + lst = self._lsts["enum"] + for tp, i in self._enums.items(): + assert i < len(lst) + assert lst[i].name == tp.name + assert len(lst) == len(self._enums) + + # ---------- + + def _prnt(self, what=''): + self._f.write(what + '\n') + + def write_source_to_f(self, f, preamble): + if self.target_is_python: + assert preamble is None + self.write_py_source_to_f(f) + else: + assert preamble is not None + self.write_c_source_to_f(f, preamble) + + def _rel_readlines(self, filename): + with open(os.path.join(os.path.dirname(__file__), filename), 'r') as g: + return g.readlines() + + def write_c_source_to_f(self, f, preamble): + self._f = f + prnt = self._prnt + if self.ffi._embedding is not None: + prnt('#define _CFFI_USE_EMBEDDING') + if not USE_LIMITED_API: + prnt('#define _CFFI_NO_LIMITED_API') + # + # first the '#include' (actually done by inlining the file's content) + lines = self._rel_readlines('_cffi_include.h') + i = lines.index('#include "parse_c_type.h"\n') + lines[i:i+1] = self._rel_readlines('parse_c_type.h') + prnt(''.join(lines)) + # + # if we have ffi._embedding != None, we give it here as a macro + # and include an extra file + base_module_name = self.module_name.split('.')[-1] + if self.ffi._embedding is not None: + prnt('#define _CFFI_MODULE_NAME "%s"' % (self.module_name,)) + prnt('static const char _CFFI_PYTHON_STARTUP_CODE[] = {') + self._print_string_literal_in_array(self.ffi._embedding) + prnt('0 };') + prnt('#ifdef PYPY_VERSION') + prnt('# define _CFFI_PYTHON_STARTUP_FUNC _cffi_pypyinit_%s' % ( + base_module_name,)) + prnt('#else') + prnt('# define _CFFI_PYTHON_STARTUP_FUNC PyInit_%s' % ( + base_module_name,)) + prnt('#endif') + lines = self._rel_readlines('_embedding.h') + i = lines.index('#include "_cffi_errors.h"\n') + lines[i:i+1] = self._rel_readlines('_cffi_errors.h') + prnt(''.join(lines)) + self.needs_version(VERSION_EMBEDDED) + # + # then paste the C source given by the user, verbatim. + prnt('/************************************************************/') + prnt() + prnt(preamble) + prnt() + prnt('/************************************************************/') + prnt() + # + # the declaration of '_cffi_types' + prnt('static void *_cffi_types[] = {') + typeindex2type = dict([(i, tp) for (tp, i) in self._typesdict.items()]) + for i, op in enumerate(self.cffi_types): + comment = '' + if i in typeindex2type: + comment = ' // ' + typeindex2type[i]._get_c_name() + prnt('/* %2d */ %s,%s' % (i, op.as_c_expr(), comment)) + if not self.cffi_types: + prnt(' 0') + prnt('};') + prnt() + # + # call generate_cpy_xxx_decl(), for every xxx found from + # ffi._parser._declarations. This generates all the functions. + self._seen_constants = set() + self._generate("decl") + # + # the declaration of '_cffi_globals' and '_cffi_typenames' + nums = {} + for step_name in self.ALL_STEPS: + lst = self._lsts[step_name] + nums[step_name] = len(lst) + if nums[step_name] > 0: + prnt('static const struct _cffi_%s_s _cffi_%ss[] = {' % ( + step_name, step_name)) + for entry in lst: + prnt(entry.as_c_expr()) + prnt('};') + prnt() + # + # the declaration of '_cffi_includes' + if self.ffi._included_ffis: + prnt('static const char * const _cffi_includes[] = {') + for ffi_to_include in self.ffi._included_ffis: + try: + included_module_name, included_source = ( + ffi_to_include._assigned_source[:2]) + except AttributeError: + raise VerificationError( + "ffi object %r includes %r, but the latter has not " + "been prepared with set_source()" % ( + self.ffi, ffi_to_include,)) + if included_source is None: + raise VerificationError( + "not implemented yet: ffi.include() of a Python-based " + "ffi inside a C-based ffi") + prnt(' "%s",' % (included_module_name,)) + prnt(' NULL') + prnt('};') + prnt() + # + # the declaration of '_cffi_type_context' + prnt('static const struct _cffi_type_context_s _cffi_type_context = {') + prnt(' _cffi_types,') + for step_name in self.ALL_STEPS: + if nums[step_name] > 0: + prnt(' _cffi_%ss,' % step_name) + else: + prnt(' NULL, /* no %ss */' % step_name) + for step_name in self.ALL_STEPS: + if step_name != "field": + prnt(' %d, /* num_%ss */' % (nums[step_name], step_name)) + if self.ffi._included_ffis: + prnt(' _cffi_includes,') + else: + prnt(' NULL, /* no includes */') + prnt(' %d, /* num_types */' % (len(self.cffi_types),)) + flags = 0 + if self._num_externpy > 0 or self.ffi._embedding is not None: + flags |= 1 # set to mean that we use extern "Python" + prnt(' %d, /* flags */' % flags) + prnt('};') + prnt() + # + # the init function + prnt('#ifdef __GNUC__') + prnt('# pragma GCC visibility push(default) /* for -fvisibility= */') + prnt('#endif') + prnt() + prnt('#ifdef PYPY_VERSION') + prnt('PyMODINIT_FUNC') + prnt('_cffi_pypyinit_%s(const void *p[])' % (base_module_name,)) + prnt('{') + if flags & 1: + prnt(' if (((intptr_t)p[0]) >= 0x0A03) {') + prnt(' _cffi_call_python_org = ' + '(void(*)(struct _cffi_externpy_s *, char *))p[1];') + prnt(' }') + prnt(' p[0] = (const void *)0x%x;' % self._version) + prnt(' p[1] = &_cffi_type_context;') + prnt(' return NULL;') + prnt('}') + # on Windows, distutils insists on putting init_cffi_xyz in + # 'export_symbols', so instead of fighting it, just give up and + # give it one + prnt('# ifdef _MSC_VER') + prnt(' PyMODINIT_FUNC') + prnt(' PyInit_%s(void) { return NULL; }' % (base_module_name,)) + prnt('# endif') + prnt('#else') + prnt('PyMODINIT_FUNC') + prnt('PyInit_%s(void)' % (base_module_name,)) + prnt('{') + prnt(' return _cffi_init("%s", 0x%x, &_cffi_type_context);' % ( + self.module_name, self._version)) + prnt('}') + prnt('#endif') + prnt() + prnt('#ifdef __GNUC__') + prnt('# pragma GCC visibility pop') + prnt('#endif') + self._version = None + + def _to_py(self, x): + if isinstance(x, str): + return "b'%s'" % (x,) + if isinstance(x, (list, tuple)): + rep = [self._to_py(item) for item in x] + if len(rep) == 1: + rep.append('') + return "(%s)" % (','.join(rep),) + return x.as_python_expr() # Py2: unicode unexpected; Py3: bytes unexp. + + def write_py_source_to_f(self, f): + self._f = f + prnt = self._prnt + # + # header + prnt("# auto-generated file") + prnt("import _cffi_backend") + # + # the 'import' of the included ffis + num_includes = len(self.ffi._included_ffis or ()) + for i in range(num_includes): + ffi_to_include = self.ffi._included_ffis[i] + try: + included_module_name, included_source = ( + ffi_to_include._assigned_source[:2]) + except AttributeError: + raise VerificationError( + "ffi object %r includes %r, but the latter has not " + "been prepared with set_source()" % ( + self.ffi, ffi_to_include,)) + if included_source is not None: + raise VerificationError( + "not implemented yet: ffi.include() of a C-based " + "ffi inside a Python-based ffi") + prnt('from %s import ffi as _ffi%d' % (included_module_name, i)) + prnt() + prnt("ffi = _cffi_backend.FFI('%s'," % (self.module_name,)) + prnt(" _version = 0x%x," % (self._version,)) + self._version = None + # + # the '_types' keyword argument + self.cffi_types = tuple(self.cffi_types) # don't change any more + types_lst = [op.as_python_bytes() for op in self.cffi_types] + prnt(' _types = %s,' % (self._to_py(''.join(types_lst)),)) + typeindex2type = dict([(i, tp) for (tp, i) in self._typesdict.items()]) + # + # the keyword arguments from ALL_STEPS + for step_name in self.ALL_STEPS: + lst = self._lsts[step_name] + if len(lst) > 0 and step_name != "field": + prnt(' _%ss = %s,' % (step_name, self._to_py(lst))) + # + # the '_includes' keyword argument + if num_includes > 0: + prnt(' _includes = (%s,),' % ( + ', '.join(['_ffi%d' % i for i in range(num_includes)]),)) + # + # the footer + prnt(')') + + # ---------- + + def _gettypenum(self, type): + # a KeyError here is a bug. please report it! :-) + return self._typesdict[type] + + def _convert_funcarg_to_c(self, tp, fromvar, tovar, errcode): + extraarg = '' + if isinstance(tp, model.BasePrimitiveType) and not tp.is_complex_type(): + if tp.is_integer_type() and tp.name != '_Bool': + converter = '_cffi_to_c_int' + extraarg = ', %s' % tp.name + elif isinstance(tp, model.UnknownFloatType): + # don't check with is_float_type(): it may be a 'long + # double' here, and _cffi_to_c_double would loose precision + converter = '(%s)_cffi_to_c_double' % (tp.get_c_name(''),) + else: + cname = tp.get_c_name('') + converter = '(%s)_cffi_to_c_%s' % (cname, + tp.name.replace(' ', '_')) + if cname in ('char16_t', 'char32_t'): + self.needs_version(VERSION_CHAR16CHAR32) + errvalue = '-1' + # + elif isinstance(tp, model.PointerType): + self._convert_funcarg_to_c_ptr_or_array(tp, fromvar, + tovar, errcode) + return + # + elif (isinstance(tp, (model.StructOrUnionOrEnum, model.BasePrimitiveType))): + # a struct (not a struct pointer) as a function argument; + # or, a complex (the same code works) + self._prnt(' if (_cffi_to_c((char *)&%s, _cffi_type(%d), %s) < 0)' + % (tovar, self._gettypenum(tp), fromvar)) + self._prnt(' %s;' % errcode) + return + # + elif isinstance(tp, model.FunctionPtrType): + converter = '(%s)_cffi_to_c_pointer' % tp.get_c_name('') + extraarg = ', _cffi_type(%d)' % self._gettypenum(tp) + errvalue = 'NULL' + # + else: + raise NotImplementedError(tp) + # + self._prnt(' %s = %s(%s%s);' % (tovar, converter, fromvar, extraarg)) + self._prnt(' if (%s == (%s)%s && PyErr_Occurred())' % ( + tovar, tp.get_c_name(''), errvalue)) + self._prnt(' %s;' % errcode) + + def _extra_local_variables(self, tp, localvars, freelines): + if isinstance(tp, model.PointerType): + localvars.add('Py_ssize_t datasize') + localvars.add('struct _cffi_freeme_s *large_args_free = NULL') + freelines.add('if (large_args_free != NULL)' + ' _cffi_free_array_arguments(large_args_free);') + + def _convert_funcarg_to_c_ptr_or_array(self, tp, fromvar, tovar, errcode): + self._prnt(' datasize = _cffi_prepare_pointer_call_argument(') + self._prnt(' _cffi_type(%d), %s, (char **)&%s);' % ( + self._gettypenum(tp), fromvar, tovar)) + self._prnt(' if (datasize != 0) {') + self._prnt(' %s = ((size_t)datasize) <= 640 ? ' + '(%s)alloca((size_t)datasize) : NULL;' % ( + tovar, tp.get_c_name(''))) + self._prnt(' if (_cffi_convert_array_argument(_cffi_type(%d), %s, ' + '(char **)&%s,' % (self._gettypenum(tp), fromvar, tovar)) + self._prnt(' datasize, &large_args_free) < 0)') + self._prnt(' %s;' % errcode) + self._prnt(' }') + + def _convert_expr_from_c(self, tp, var, context): + if isinstance(tp, model.BasePrimitiveType): + if tp.is_integer_type() and tp.name != '_Bool': + return '_cffi_from_c_int(%s, %s)' % (var, tp.name) + elif isinstance(tp, model.UnknownFloatType): + return '_cffi_from_c_double(%s)' % (var,) + elif tp.name != 'long double' and not tp.is_complex_type(): + cname = tp.name.replace(' ', '_') + if cname in ('char16_t', 'char32_t'): + self.needs_version(VERSION_CHAR16CHAR32) + return '_cffi_from_c_%s(%s)' % (cname, var) + else: + return '_cffi_from_c_deref((char *)&%s, _cffi_type(%d))' % ( + var, self._gettypenum(tp)) + elif isinstance(tp, (model.PointerType, model.FunctionPtrType)): + return '_cffi_from_c_pointer((char *)%s, _cffi_type(%d))' % ( + var, self._gettypenum(tp)) + elif isinstance(tp, model.ArrayType): + return '_cffi_from_c_pointer((char *)%s, _cffi_type(%d))' % ( + var, self._gettypenum(model.PointerType(tp.item))) + elif isinstance(tp, model.StructOrUnion): + if tp.fldnames is None: + raise TypeError("'%s' is used as %s, but is opaque" % ( + tp._get_c_name(), context)) + return '_cffi_from_c_struct((char *)&%s, _cffi_type(%d))' % ( + var, self._gettypenum(tp)) + elif isinstance(tp, model.EnumType): + return '_cffi_from_c_deref((char *)&%s, _cffi_type(%d))' % ( + var, self._gettypenum(tp)) + else: + raise NotImplementedError(tp) + + # ---------- + # typedefs + + def _typedef_type(self, tp, name): + return self._global_type(tp, "(*(%s *)0)" % (name,)) + + def _generate_cpy_typedef_collecttype(self, tp, name): + self._do_collect_type(self._typedef_type(tp, name)) + + def _generate_cpy_typedef_decl(self, tp, name): + pass + + def _typedef_ctx(self, tp, name): + type_index = self._typesdict[tp] + self._lsts["typename"].append(TypenameExpr(name, type_index)) + + def _generate_cpy_typedef_ctx(self, tp, name): + tp = self._typedef_type(tp, name) + self._typedef_ctx(tp, name) + if getattr(tp, "origin", None) == "unknown_type": + self._struct_ctx(tp, tp.name, approxname=None) + elif isinstance(tp, model.NamedPointerType): + self._struct_ctx(tp.totype, tp.totype.name, approxname=tp.name, + named_ptr=tp) + + # ---------- + # function declarations + + def _generate_cpy_function_collecttype(self, tp, name): + self._do_collect_type(tp.as_raw_function()) + if tp.ellipsis and not self.target_is_python: + self._do_collect_type(tp) + + def _generate_cpy_function_decl(self, tp, name): + assert not self.target_is_python + assert isinstance(tp, model.FunctionPtrType) + if tp.ellipsis: + # cannot support vararg functions better than this: check for its + # exact type (including the fixed arguments), and build it as a + # constant function pointer (no CPython wrapper) + self._generate_cpy_constant_decl(tp, name) + return + prnt = self._prnt + numargs = len(tp.args) + if numargs == 0: + argname = 'noarg' + elif numargs == 1: + argname = 'arg0' + else: + argname = 'args' + # + # ------------------------------ + # the 'd' version of the function, only for addressof(lib, 'func') + arguments = [] + call_arguments = [] + context = 'argument of %s' % name + for i, type in enumerate(tp.args): + arguments.append(type.get_c_name(' x%d' % i, context)) + call_arguments.append('x%d' % i) + repr_arguments = ', '.join(arguments) + repr_arguments = repr_arguments or 'void' + if tp.abi: + abi = tp.abi + ' ' + else: + abi = '' + name_and_arguments = '%s_cffi_d_%s(%s)' % (abi, name, repr_arguments) + prnt('static %s' % (tp.result.get_c_name(name_and_arguments),)) + prnt('{') + call_arguments = ', '.join(call_arguments) + result_code = 'return ' + if isinstance(tp.result, model.VoidType): + result_code = '' + prnt(' %s%s(%s);' % (result_code, name, call_arguments)) + prnt('}') + # + prnt('#ifndef PYPY_VERSION') # ------------------------------ + # + prnt('static PyObject *') + prnt('_cffi_f_%s(PyObject *self, PyObject *%s)' % (name, argname)) + prnt('{') + # + context = 'argument of %s' % name + for i, type in enumerate(tp.args): + arg = type.get_c_name(' x%d' % i, context) + prnt(' %s;' % arg) + # + localvars = set() + freelines = set() + for type in tp.args: + self._extra_local_variables(type, localvars, freelines) + for decl in sorted(localvars): + prnt(' %s;' % (decl,)) + # + if not isinstance(tp.result, model.VoidType): + result_code = 'result = ' + context = 'result of %s' % name + result_decl = ' %s;' % tp.result.get_c_name(' result', context) + prnt(result_decl) + prnt(' PyObject *pyresult;') + else: + result_decl = None + result_code = '' + # + if len(tp.args) > 1: + rng = range(len(tp.args)) + for i in rng: + prnt(' PyObject *arg%d;' % i) + prnt() + prnt(' if (!PyArg_UnpackTuple(args, "%s", %d, %d, %s))' % ( + name, len(rng), len(rng), + ', '.join(['&arg%d' % i for i in rng]))) + prnt(' return NULL;') + prnt() + # + for i, type in enumerate(tp.args): + self._convert_funcarg_to_c(type, 'arg%d' % i, 'x%d' % i, + 'return NULL') + prnt() + # + prnt(' Py_BEGIN_ALLOW_THREADS') + prnt(' _cffi_restore_errno();') + call_arguments = ['x%d' % i for i in range(len(tp.args))] + call_arguments = ', '.join(call_arguments) + prnt(' { %s%s(%s); }' % (result_code, name, call_arguments)) + prnt(' _cffi_save_errno();') + prnt(' Py_END_ALLOW_THREADS') + prnt() + # + prnt(' (void)self; /* unused */') + if numargs == 0: + prnt(' (void)noarg; /* unused */') + if result_code: + prnt(' pyresult = %s;' % + self._convert_expr_from_c(tp.result, 'result', 'result type')) + for freeline in freelines: + prnt(' ' + freeline) + prnt(' return pyresult;') + else: + for freeline in freelines: + prnt(' ' + freeline) + prnt(' Py_INCREF(Py_None);') + prnt(' return Py_None;') + prnt('}') + # + prnt('#else') # ------------------------------ + # + # the PyPy version: need to replace struct/union arguments with + # pointers, and if the result is a struct/union, insert a first + # arg that is a pointer to the result. We also do that for + # complex args and return type. + def need_indirection(type): + return (isinstance(type, model.StructOrUnion) or + (isinstance(type, model.PrimitiveType) and + type.is_complex_type())) + difference = False + arguments = [] + call_arguments = [] + context = 'argument of %s' % name + for i, type in enumerate(tp.args): + indirection = '' + if need_indirection(type): + indirection = '*' + difference = True + arg = type.get_c_name(' %sx%d' % (indirection, i), context) + arguments.append(arg) + call_arguments.append('%sx%d' % (indirection, i)) + tp_result = tp.result + if need_indirection(tp_result): + context = 'result of %s' % name + arg = tp_result.get_c_name(' *result', context) + arguments.insert(0, arg) + tp_result = model.void_type + result_decl = None + result_code = '*result = ' + difference = True + if difference: + repr_arguments = ', '.join(arguments) + repr_arguments = repr_arguments or 'void' + name_and_arguments = '%s_cffi_f_%s(%s)' % (abi, name, + repr_arguments) + prnt('static %s' % (tp_result.get_c_name(name_and_arguments),)) + prnt('{') + if result_decl: + prnt(result_decl) + call_arguments = ', '.join(call_arguments) + prnt(' { %s%s(%s); }' % (result_code, name, call_arguments)) + if result_decl: + prnt(' return result;') + prnt('}') + else: + prnt('# define _cffi_f_%s _cffi_d_%s' % (name, name)) + # + prnt('#endif') # ------------------------------ + prnt() + + def _generate_cpy_function_ctx(self, tp, name): + if tp.ellipsis and not self.target_is_python: + self._generate_cpy_constant_ctx(tp, name) + return + type_index = self._typesdict[tp.as_raw_function()] + numargs = len(tp.args) + if self.target_is_python: + meth_kind = OP_DLOPEN_FUNC + elif numargs == 0: + meth_kind = OP_CPYTHON_BLTN_N # 'METH_NOARGS' + elif numargs == 1: + meth_kind = OP_CPYTHON_BLTN_O # 'METH_O' + else: + meth_kind = OP_CPYTHON_BLTN_V # 'METH_VARARGS' + self._lsts["global"].append( + GlobalExpr(name, '_cffi_f_%s' % name, + CffiOp(meth_kind, type_index), + size='_cffi_d_%s' % name)) + + # ---------- + # named structs or unions + + def _field_type(self, tp_struct, field_name, tp_field): + if isinstance(tp_field, model.ArrayType): + actual_length = tp_field.length + if actual_length == '...': + ptr_struct_name = tp_struct.get_c_name('*') + actual_length = '_cffi_array_len(((%s)0)->%s)' % ( + ptr_struct_name, field_name) + tp_item = self._field_type(tp_struct, '%s[0]' % field_name, + tp_field.item) + tp_field = model.ArrayType(tp_item, actual_length) + return tp_field + + def _struct_collecttype(self, tp): + self._do_collect_type(tp) + if self.target_is_python: + # also requires nested anon struct/unions in ABI mode, recursively + for fldtype in tp.anonymous_struct_fields(): + self._struct_collecttype(fldtype) + + def _struct_decl(self, tp, cname, approxname): + if tp.fldtypes is None: + return + prnt = self._prnt + checkfuncname = '_cffi_checkfld_%s' % (approxname,) + prnt('_CFFI_UNUSED_FN') + prnt('static void %s(%s *p)' % (checkfuncname, cname)) + prnt('{') + prnt(' /* only to generate compile-time warnings or errors */') + prnt(' (void)p;') + for fname, ftype, fbitsize, fqual in self._enum_fields(tp): + try: + if ftype.is_integer_type() or fbitsize >= 0: + # accept all integers, but complain on float or double + if fname != '': + prnt(" (void)((p->%s) | 0); /* check that '%s.%s' is " + "an integer */" % (fname, cname, fname)) + continue + # only accept exactly the type declared, except that '[]' + # is interpreted as a '*' and so will match any array length. + # (It would also match '*', but that's harder to detect...) + while (isinstance(ftype, model.ArrayType) + and (ftype.length is None or ftype.length == '...')): + ftype = ftype.item + fname = fname + '[0]' + prnt(' { %s = &p->%s; (void)tmp; }' % ( + ftype.get_c_name('*tmp', 'field %r'%fname, quals=fqual), + fname)) + except VerificationError as e: + prnt(' /* %s */' % str(e)) # cannot verify it, ignore + prnt('}') + prnt('struct _cffi_align_%s { char x; %s y; };' % (approxname, cname)) + prnt() + + def _struct_ctx(self, tp, cname, approxname, named_ptr=None): + type_index = self._typesdict[tp] + reason_for_not_expanding = None + flags = [] + if isinstance(tp, model.UnionType): + flags.append("_CFFI_F_UNION") + if tp.fldtypes is None: + flags.append("_CFFI_F_OPAQUE") + reason_for_not_expanding = "opaque" + if (tp not in self.ffi._parser._included_declarations and + (named_ptr is None or + named_ptr not in self.ffi._parser._included_declarations)): + if tp.fldtypes is None: + pass # opaque + elif tp.partial or any(tp.anonymous_struct_fields()): + pass # field layout obtained silently from the C compiler + else: + flags.append("_CFFI_F_CHECK_FIELDS") + if tp.packed: + if tp.packed > 1: + raise NotImplementedError( + "%r is declared with 'pack=%r'; only 0 or 1 are " + "supported in API mode (try to use \"...;\", which " + "does not require a 'pack' declaration)" % + (tp, tp.packed)) + flags.append("_CFFI_F_PACKED") + else: + flags.append("_CFFI_F_EXTERNAL") + reason_for_not_expanding = "external" + flags = '|'.join(flags) or '0' + c_fields = [] + if reason_for_not_expanding is None: + enumfields = list(self._enum_fields(tp)) + for fldname, fldtype, fbitsize, fqual in enumfields: + fldtype = self._field_type(tp, fldname, fldtype) + self._check_not_opaque(fldtype, + "field '%s.%s'" % (tp.name, fldname)) + # cname is None for _add_missing_struct_unions() only + op = OP_NOOP + if fbitsize >= 0: + op = OP_BITFIELD + size = '%d /* bits */' % fbitsize + elif cname is None or ( + isinstance(fldtype, model.ArrayType) and + fldtype.length is None): + size = '(size_t)-1' + else: + size = 'sizeof(((%s)0)->%s)' % ( + tp.get_c_name('*') if named_ptr is None + else named_ptr.name, + fldname) + if cname is None or fbitsize >= 0: + offset = '(size_t)-1' + elif named_ptr is not None: + offset = '(size_t)(((char *)&((%s)4096)->%s) - (char *)4096)' % ( + named_ptr.name, fldname) + else: + offset = 'offsetof(%s, %s)' % (tp.get_c_name(''), fldname) + c_fields.append( + FieldExpr(fldname, offset, size, fbitsize, + CffiOp(op, self._typesdict[fldtype]))) + first_field_index = len(self._lsts["field"]) + self._lsts["field"].extend(c_fields) + # + if cname is None: # unknown name, for _add_missing_struct_unions + size = '(size_t)-2' + align = -2 + comment = "unnamed" + else: + if named_ptr is not None: + size = 'sizeof(*(%s)0)' % (named_ptr.name,) + align = '-1 /* unknown alignment */' + else: + size = 'sizeof(%s)' % (cname,) + align = 'offsetof(struct _cffi_align_%s, y)' % (approxname,) + comment = None + else: + size = '(size_t)-1' + align = -1 + first_field_index = -1 + comment = reason_for_not_expanding + self._lsts["struct_union"].append( + StructUnionExpr(tp.name, type_index, flags, size, align, comment, + first_field_index, c_fields)) + self._seen_struct_unions.add(tp) + + def _check_not_opaque(self, tp, location): + while isinstance(tp, model.ArrayType): + tp = tp.item + if isinstance(tp, model.StructOrUnion) and tp.fldtypes is None: + raise TypeError( + "%s is of an opaque type (not declared in cdef())" % location) + + def _add_missing_struct_unions(self): + # not very nice, but some struct declarations might be missing + # because they don't have any known C name. Check that they are + # not partial (we can't complete or verify them!) and emit them + # anonymously. + lst = list(self._struct_unions.items()) + lst.sort(key=lambda tp_order: tp_order[1]) + for tp, order in lst: + if tp not in self._seen_struct_unions: + if tp.partial: + raise NotImplementedError("internal inconsistency: %r is " + "partial but was not seen at " + "this point" % (tp,)) + if tp.name.startswith('$') and tp.name[1:].isdigit(): + approxname = tp.name[1:] + elif tp.name == '_IO_FILE' and tp.forcename == 'FILE': + approxname = 'FILE' + self._typedef_ctx(tp, 'FILE') + else: + raise NotImplementedError("internal inconsistency: %r" % + (tp,)) + self._struct_ctx(tp, None, approxname) + + def _generate_cpy_struct_collecttype(self, tp, name): + self._struct_collecttype(tp) + _generate_cpy_union_collecttype = _generate_cpy_struct_collecttype + + def _struct_names(self, tp): + cname = tp.get_c_name('') + if ' ' in cname: + return cname, cname.replace(' ', '_') + else: + return cname, '_' + cname + + def _generate_cpy_struct_decl(self, tp, name): + self._struct_decl(tp, *self._struct_names(tp)) + _generate_cpy_union_decl = _generate_cpy_struct_decl + + def _generate_cpy_struct_ctx(self, tp, name): + self._struct_ctx(tp, *self._struct_names(tp)) + _generate_cpy_union_ctx = _generate_cpy_struct_ctx + + # ---------- + # 'anonymous' declarations. These are produced for anonymous structs + # or unions; the 'name' is obtained by a typedef. + + def _generate_cpy_anonymous_collecttype(self, tp, name): + if isinstance(tp, model.EnumType): + self._generate_cpy_enum_collecttype(tp, name) + else: + self._struct_collecttype(tp) + + def _generate_cpy_anonymous_decl(self, tp, name): + if isinstance(tp, model.EnumType): + self._generate_cpy_enum_decl(tp) + else: + self._struct_decl(tp, name, 'typedef_' + name) + + def _generate_cpy_anonymous_ctx(self, tp, name): + if isinstance(tp, model.EnumType): + self._enum_ctx(tp, name) + else: + self._struct_ctx(tp, name, 'typedef_' + name) + + # ---------- + # constants, declared with "static const ..." + + def _generate_cpy_const(self, is_int, name, tp=None, category='const', + check_value=None): + if (category, name) in self._seen_constants: + raise VerificationError( + "duplicate declaration of %s '%s'" % (category, name)) + self._seen_constants.add((category, name)) + # + prnt = self._prnt + funcname = '_cffi_%s_%s' % (category, name) + if is_int: + prnt('static int %s(unsigned long long *o)' % funcname) + prnt('{') + prnt(' int n = (%s) <= 0;' % (name,)) + prnt(' *o = (unsigned long long)((%s) | 0);' + ' /* check that %s is an integer */' % (name, name)) + if check_value is not None: + if check_value > 0: + check_value = '%dU' % (check_value,) + prnt(' if (!_cffi_check_int(*o, n, %s))' % (check_value,)) + prnt(' n |= 2;') + prnt(' return n;') + prnt('}') + else: + assert check_value is None + prnt('static void %s(char *o)' % funcname) + prnt('{') + prnt(' *(%s)o = %s;' % (tp.get_c_name('*'), name)) + prnt('}') + prnt() + + def _generate_cpy_constant_collecttype(self, tp, name): + is_int = tp.is_integer_type() + if not is_int or self.target_is_python: + self._do_collect_type(tp) + + def _generate_cpy_constant_decl(self, tp, name): + is_int = tp.is_integer_type() + self._generate_cpy_const(is_int, name, tp) + + def _generate_cpy_constant_ctx(self, tp, name): + if not self.target_is_python and tp.is_integer_type(): + type_op = CffiOp(OP_CONSTANT_INT, -1) + else: + if self.target_is_python: + const_kind = OP_DLOPEN_CONST + else: + const_kind = OP_CONSTANT + type_index = self._typesdict[tp] + type_op = CffiOp(const_kind, type_index) + self._lsts["global"].append( + GlobalExpr(name, '_cffi_const_%s' % name, type_op)) + + # ---------- + # enums + + def _generate_cpy_enum_collecttype(self, tp, name): + self._do_collect_type(tp) + + def _generate_cpy_enum_decl(self, tp, name=None): + for enumerator in tp.enumerators: + self._generate_cpy_const(True, enumerator) + + def _enum_ctx(self, tp, cname): + type_index = self._typesdict[tp] + type_op = CffiOp(OP_ENUM, -1) + if self.target_is_python: + tp.check_not_partial() + for enumerator, enumvalue in zip(tp.enumerators, tp.enumvalues): + self._lsts["global"].append( + GlobalExpr(enumerator, '_cffi_const_%s' % enumerator, type_op, + check_value=enumvalue)) + # + if cname is not None and '$' not in cname and not self.target_is_python: + size = "sizeof(%s)" % cname + signed = "((%s)-1) <= 0" % cname + else: + basetp = tp.build_baseinttype(self.ffi, []) + size = self.ffi.sizeof(basetp) + signed = int(int(self.ffi.cast(basetp, -1)) < 0) + allenums = ",".join(tp.enumerators) + self._lsts["enum"].append( + EnumExpr(tp.name, type_index, size, signed, allenums)) + + def _generate_cpy_enum_ctx(self, tp, name): + self._enum_ctx(tp, tp._get_c_name()) + + # ---------- + # macros: for now only for integers + + def _generate_cpy_macro_collecttype(self, tp, name): + pass + + def _generate_cpy_macro_decl(self, tp, name): + if tp == '...': + check_value = None + else: + check_value = tp # an integer + self._generate_cpy_const(True, name, check_value=check_value) + + def _generate_cpy_macro_ctx(self, tp, name): + if tp == '...': + if self.target_is_python: + raise VerificationError( + "cannot use the syntax '...' in '#define %s ...' when " + "using the ABI mode" % (name,)) + check_value = None + else: + check_value = tp # an integer + type_op = CffiOp(OP_CONSTANT_INT, -1) + self._lsts["global"].append( + GlobalExpr(name, '_cffi_const_%s' % name, type_op, + check_value=check_value)) + + # ---------- + # global variables + + def _global_type(self, tp, global_name): + if isinstance(tp, model.ArrayType): + actual_length = tp.length + if actual_length == '...': + actual_length = '_cffi_array_len(%s)' % (global_name,) + tp_item = self._global_type(tp.item, '%s[0]' % global_name) + tp = model.ArrayType(tp_item, actual_length) + return tp + + def _generate_cpy_variable_collecttype(self, tp, name): + self._do_collect_type(self._global_type(tp, name)) + + def _generate_cpy_variable_decl(self, tp, name): + prnt = self._prnt + tp = self._global_type(tp, name) + if isinstance(tp, model.ArrayType) and tp.length is None: + tp = tp.item + ampersand = '' + else: + ampersand = '&' + # This code assumes that casts from "tp *" to "void *" is a + # no-op, i.e. a function that returns a "tp *" can be called + # as if it returned a "void *". This should be generally true + # on any modern machine. The only exception to that rule (on + # uncommon architectures, and as far as I can tell) might be + # if 'tp' were a function type, but that is not possible here. + # (If 'tp' is a function _pointer_ type, then casts from "fn_t + # **" to "void *" are again no-ops, as far as I can tell.) + decl = '*_cffi_var_%s(void)' % (name,) + prnt('static ' + tp.get_c_name(decl, quals=self._current_quals)) + prnt('{') + prnt(' return %s(%s);' % (ampersand, name)) + prnt('}') + prnt() + + def _generate_cpy_variable_ctx(self, tp, name): + tp = self._global_type(tp, name) + type_index = self._typesdict[tp] + if self.target_is_python: + op = OP_GLOBAL_VAR + else: + op = OP_GLOBAL_VAR_F + self._lsts["global"].append( + GlobalExpr(name, '_cffi_var_%s' % name, CffiOp(op, type_index))) + + # ---------- + # extern "Python" + + def _generate_cpy_extern_python_collecttype(self, tp, name): + assert isinstance(tp, model.FunctionPtrType) + self._do_collect_type(tp) + _generate_cpy_dllexport_python_collecttype = \ + _generate_cpy_extern_python_plus_c_collecttype = \ + _generate_cpy_extern_python_collecttype + + def _extern_python_decl(self, tp, name, tag_and_space): + prnt = self._prnt + if isinstance(tp.result, model.VoidType): + size_of_result = '0' + else: + context = 'result of %s' % name + size_of_result = '(int)sizeof(%s)' % ( + tp.result.get_c_name('', context),) + prnt('static struct _cffi_externpy_s _cffi_externpy__%s =' % name) + prnt(' { "%s.%s", %s, 0, 0 };' % ( + self.module_name, name, size_of_result)) + prnt() + # + arguments = [] + context = 'argument of %s' % name + for i, type in enumerate(tp.args): + arg = type.get_c_name(' a%d' % i, context) + arguments.append(arg) + # + repr_arguments = ', '.join(arguments) + repr_arguments = repr_arguments or 'void' + name_and_arguments = '%s(%s)' % (name, repr_arguments) + if tp.abi == "__stdcall": + name_and_arguments = '_cffi_stdcall ' + name_and_arguments + # + def may_need_128_bits(tp): + return (isinstance(tp, model.PrimitiveType) and + tp.name == 'long double') + # + size_of_a = max(len(tp.args)*8, 8) + if may_need_128_bits(tp.result): + size_of_a = max(size_of_a, 16) + if isinstance(tp.result, model.StructOrUnion): + size_of_a = 'sizeof(%s) > %d ? sizeof(%s) : %d' % ( + tp.result.get_c_name(''), size_of_a, + tp.result.get_c_name(''), size_of_a) + prnt('%s%s' % (tag_and_space, tp.result.get_c_name(name_and_arguments))) + prnt('{') + prnt(' char a[%s];' % size_of_a) + prnt(' char *p = a;') + for i, type in enumerate(tp.args): + arg = 'a%d' % i + if (isinstance(type, model.StructOrUnion) or + may_need_128_bits(type)): + arg = '&' + arg + type = model.PointerType(type) + prnt(' *(%s)(p + %d) = %s;' % (type.get_c_name('*'), i*8, arg)) + prnt(' _cffi_call_python(&_cffi_externpy__%s, p);' % name) + if not isinstance(tp.result, model.VoidType): + prnt(' return *(%s)p;' % (tp.result.get_c_name('*'),)) + prnt('}') + prnt() + self._num_externpy += 1 + + def _generate_cpy_extern_python_decl(self, tp, name): + self._extern_python_decl(tp, name, 'static ') + + def _generate_cpy_dllexport_python_decl(self, tp, name): + self._extern_python_decl(tp, name, 'CFFI_DLLEXPORT ') + + def _generate_cpy_extern_python_plus_c_decl(self, tp, name): + self._extern_python_decl(tp, name, '') + + def _generate_cpy_extern_python_ctx(self, tp, name): + if self.target_is_python: + raise VerificationError( + "cannot use 'extern \"Python\"' in the ABI mode") + if tp.ellipsis: + raise NotImplementedError("a vararg function is extern \"Python\"") + type_index = self._typesdict[tp] + type_op = CffiOp(OP_EXTERN_PYTHON, type_index) + self._lsts["global"].append( + GlobalExpr(name, '&_cffi_externpy__%s' % name, type_op, name)) + + _generate_cpy_dllexport_python_ctx = \ + _generate_cpy_extern_python_plus_c_ctx = \ + _generate_cpy_extern_python_ctx + + def _print_string_literal_in_array(self, s): + prnt = self._prnt + prnt('// # NB. this is not a string because of a size limit in MSVC') + if not isinstance(s, bytes): # unicode + s = s.encode('utf-8') # -> bytes + else: + s.decode('utf-8') # got bytes, check for valid utf-8 + for line in s.splitlines(True): + comment = line + if type('//') is bytes: # python2 + line = map(ord, line) # make a list of integers + else: # python3 + # type(line) is bytes, which enumerates like a list of integers + comment = ascii(comment)[1:-1] + prnt(('// ' + comment).rstrip()) + printed_line = '' + for c in line: + if len(printed_line) >= 76: + prnt(printed_line) + printed_line = '' + printed_line += '%d,' % (c,) + prnt(printed_line) + + # ---------- + # emitting the opcodes for individual types + + def _emit_bytecode_VoidType(self, tp, index): + self.cffi_types[index] = CffiOp(OP_PRIMITIVE, PRIM_VOID) + + def _emit_bytecode_PrimitiveType(self, tp, index): + prim_index = PRIMITIVE_TO_INDEX[tp.name] + self.cffi_types[index] = CffiOp(OP_PRIMITIVE, prim_index) + + def _emit_bytecode_UnknownIntegerType(self, tp, index): + s = ('_cffi_prim_int(sizeof(%s), (\n' + ' ((%s)-1) | 0 /* check that %s is an integer type */\n' + ' ) <= 0)' % (tp.name, tp.name, tp.name)) + self.cffi_types[index] = CffiOp(OP_PRIMITIVE, s) + + def _emit_bytecode_UnknownFloatType(self, tp, index): + s = ('_cffi_prim_float(sizeof(%s) *\n' + ' (((%s)1) / 2) * 2 /* integer => 0, float => 1 */\n' + ' )' % (tp.name, tp.name)) + self.cffi_types[index] = CffiOp(OP_PRIMITIVE, s) + + def _emit_bytecode_RawFunctionType(self, tp, index): + self.cffi_types[index] = CffiOp(OP_FUNCTION, self._typesdict[tp.result]) + index += 1 + for tp1 in tp.args: + realindex = self._typesdict[tp1] + if index != realindex: + if isinstance(tp1, model.PrimitiveType): + self._emit_bytecode_PrimitiveType(tp1, index) + else: + self.cffi_types[index] = CffiOp(OP_NOOP, realindex) + index += 1 + flags = int(tp.ellipsis) + if tp.abi is not None: + if tp.abi == '__stdcall': + flags |= 2 + else: + raise NotImplementedError("abi=%r" % (tp.abi,)) + self.cffi_types[index] = CffiOp(OP_FUNCTION_END, flags) + + def _emit_bytecode_PointerType(self, tp, index): + self.cffi_types[index] = CffiOp(OP_POINTER, self._typesdict[tp.totype]) + + _emit_bytecode_ConstPointerType = _emit_bytecode_PointerType + _emit_bytecode_NamedPointerType = _emit_bytecode_PointerType + + def _emit_bytecode_FunctionPtrType(self, tp, index): + raw = tp.as_raw_function() + self.cffi_types[index] = CffiOp(OP_POINTER, self._typesdict[raw]) + + def _emit_bytecode_ArrayType(self, tp, index): + item_index = self._typesdict[tp.item] + if tp.length is None: + self.cffi_types[index] = CffiOp(OP_OPEN_ARRAY, item_index) + elif tp.length == '...': + raise VerificationError( + "type %s badly placed: the '...' array length can only be " + "used on global arrays or on fields of structures" % ( + str(tp).replace('/*...*/', '...'),)) + else: + assert self.cffi_types[index + 1] == 'LEN' + self.cffi_types[index] = CffiOp(OP_ARRAY, item_index) + self.cffi_types[index + 1] = CffiOp(None, str(tp.length)) + + def _emit_bytecode_StructType(self, tp, index): + struct_index = self._struct_unions[tp] + self.cffi_types[index] = CffiOp(OP_STRUCT_UNION, struct_index) + _emit_bytecode_UnionType = _emit_bytecode_StructType + + def _emit_bytecode_EnumType(self, tp, index): + enum_index = self._enums[tp] + self.cffi_types[index] = CffiOp(OP_ENUM, enum_index) + + +if sys.version_info >= (3,): + NativeIO = io.StringIO +else: + class NativeIO(io.BytesIO): + def write(self, s): + if isinstance(s, unicode): + s = s.encode('ascii') + super().write(s) + +def _is_file_like(maybefile): + # compare to xml.etree.ElementTree._get_writer + return hasattr(maybefile, 'write') + +def _make_c_or_py_source(ffi, module_name, preamble, target_file, verbose): + if verbose: + print("generating %s" % (target_file,)) + recompiler = Recompiler(ffi, module_name, + target_is_python=(preamble is None)) + recompiler.collect_type_table() + recompiler.collect_step_tables() + if _is_file_like(target_file): + recompiler.write_source_to_f(target_file, preamble) + return True + f = NativeIO() + recompiler.write_source_to_f(f, preamble) + output = f.getvalue() + try: + with open(target_file, 'r') as f1: + if f1.read(len(output) + 1) != output: + raise OSError + if verbose: + print("(already up-to-date)") + return False # already up-to-date + except OSError: + tmp_file = '%s.~%d' % (target_file, os.getpid()) + with open(tmp_file, 'w') as f1: + f1.write(output) + try: + os.rename(tmp_file, target_file) + except OSError: + os.unlink(target_file) + os.rename(tmp_file, target_file) + return True + +def make_c_source(ffi, module_name, preamble, target_c_file, verbose=False): + assert preamble is not None + return _make_c_or_py_source(ffi, module_name, preamble, target_c_file, + verbose) + +def make_py_source(ffi, module_name, target_py_file, verbose=False): + return _make_c_or_py_source(ffi, module_name, None, target_py_file, + verbose) + +def _modname_to_file(outputdir, modname, extension): + parts = modname.split('.') + try: + os.makedirs(os.path.join(outputdir, *parts[:-1])) + except OSError: + pass + parts[-1] += extension + return os.path.join(outputdir, *parts), parts + + +# Aaargh. Distutils is not tested at all for the purpose of compiling +# DLLs that are not extension modules. Here are some hacks to work +# around that, in the _patch_for_*() functions... + +def _patch_meth(patchlist, cls, name, new_meth): + old = getattr(cls, name) + patchlist.append((cls, name, old)) + setattr(cls, name, new_meth) + return old + +def _unpatch_meths(patchlist): + for cls, name, old_meth in reversed(patchlist): + setattr(cls, name, old_meth) + +def _patch_for_embedding(patchlist): + if sys.platform == 'win32': + # we must not remove the manifest when building for embedding! + # FUTURE: this module was removed in setuptools 74; this is likely dead code and should be removed, + # since the toolchain it supports (VS2005-2008) is also long dead. + from cffi._shimmed_dist_utils import MSVCCompiler + if MSVCCompiler is not None: + _patch_meth(patchlist, MSVCCompiler, '_remove_visual_c_ref', + lambda self, manifest_file: manifest_file) + + if sys.platform == 'darwin': + # we must not make a '-bundle', but a '-dynamiclib' instead + from cffi._shimmed_dist_utils import CCompiler + def my_link_shared_object(self, *args, **kwds): + if '-bundle' in self.linker_so: + self.linker_so = list(self.linker_so) + i = self.linker_so.index('-bundle') + self.linker_so[i] = '-dynamiclib' + return old_link_shared_object(self, *args, **kwds) + old_link_shared_object = _patch_meth(patchlist, CCompiler, + 'link_shared_object', + my_link_shared_object) + +def _patch_for_target(patchlist, target): + from cffi._shimmed_dist_utils import build_ext + # if 'target' is different from '*', we need to patch some internal + # method to just return this 'target' value, instead of having it + # built from module_name + if target.endswith('.*'): + target = target[:-2] + if sys.platform == 'win32': + target += '.dll' + elif sys.platform == 'darwin': + target += '.dylib' + else: + target += '.so' + _patch_meth(patchlist, build_ext, 'get_ext_filename', + lambda self, ext_name: target) + + +def recompile(ffi, module_name, preamble, tmpdir='.', call_c_compiler=True, + c_file=None, source_extension='.c', extradir=None, + compiler_verbose=1, target=None, debug=None, + uses_ffiplatform=True, **kwds): + if not isinstance(module_name, str): + module_name = module_name.encode('ascii') + if ffi._windows_unicode: + ffi._apply_windows_unicode(kwds) + if preamble is not None: + if call_c_compiler and _is_file_like(c_file): + raise TypeError("Writing to file-like objects is not supported " + "with call_c_compiler=True") + embedding = (ffi._embedding is not None) + if embedding: + ffi._apply_embedding_fix(kwds) + if c_file is None: + c_file, parts = _modname_to_file(tmpdir, module_name, + source_extension) + if extradir: + parts = [extradir] + parts + ext_c_file = os.path.join(*parts) + else: + ext_c_file = c_file + # + if target is None: + if embedding: + target = '%s.*' % module_name + else: + target = '*' + # + if uses_ffiplatform: + ext = ffiplatform.get_extension(ext_c_file, module_name, **kwds) + else: + ext = None + updated = make_c_source(ffi, module_name, preamble, c_file, + verbose=compiler_verbose) + if call_c_compiler: + patchlist = [] + cwd = os.getcwd() + try: + if embedding: + _patch_for_embedding(patchlist) + if target != '*': + _patch_for_target(patchlist, target) + if compiler_verbose: + if tmpdir == '.': + msg = 'the current directory is' + else: + msg = 'setting the current directory to' + print('%s %r' % (msg, os.path.abspath(tmpdir))) + os.chdir(tmpdir) + outputfilename = ffiplatform.compile('.', ext, + compiler_verbose, debug) + finally: + os.chdir(cwd) + _unpatch_meths(patchlist) + return outputfilename + else: + return ext, updated + else: + if c_file is None: + c_file, _ = _modname_to_file(tmpdir, module_name, '.py') + updated = make_py_source(ffi, module_name, c_file, + verbose=compiler_verbose) + if call_c_compiler: + return c_file + else: + return None, updated + diff --git a/py_modules/cffi/setuptools_ext.py b/py_modules/cffi/setuptools_ext.py new file mode 100644 index 0000000..946afb4 --- /dev/null +++ b/py_modules/cffi/setuptools_ext.py @@ -0,0 +1,231 @@ +import os +import sys +import sysconfig + +try: + basestring +except NameError: + # Python 3.x + basestring = str + +def error(msg): + from cffi._shimmed_dist_utils import DistutilsSetupError + raise DistutilsSetupError(msg) + + +def execfile(filename, glob): + # We use execfile() (here rewritten for Python 3) instead of + # __import__() to load the build script. The problem with + # a normal import is that in some packages, the intermediate + # __init__.py files may already try to import the file that + # we are generating. + with open(filename) as f: + src = f.read() + src += '\n' # Python 2.6 compatibility + code = compile(src, filename, 'exec') + exec(code, glob, glob) + + +def add_cffi_module(dist, mod_spec): + from cffi.api import FFI + + if not isinstance(mod_spec, basestring): + error("argument to 'cffi_modules=...' must be a str or a list of str," + " not %r" % (type(mod_spec).__name__,)) + mod_spec = str(mod_spec) + try: + build_file_name, ffi_var_name = mod_spec.split(':') + except ValueError: + error("%r must be of the form 'path/build.py:ffi_variable'" % + (mod_spec,)) + if not os.path.exists(build_file_name): + ext = '' + rewritten = build_file_name.replace('.', '/') + '.py' + if os.path.exists(rewritten): + ext = ' (rewrite cffi_modules to [%r])' % ( + rewritten + ':' + ffi_var_name,) + error("%r does not name an existing file%s" % (build_file_name, ext)) + + mod_vars = {'__name__': '__cffi__', '__file__': build_file_name} + execfile(build_file_name, mod_vars) + + try: + ffi = mod_vars[ffi_var_name] + except KeyError: + error("%r: object %r not found in module" % (mod_spec, + ffi_var_name)) + if not isinstance(ffi, FFI): + ffi = ffi() # maybe it's a function instead of directly an ffi + if not isinstance(ffi, FFI): + error("%r is not an FFI instance (got %r)" % (mod_spec, + type(ffi).__name__)) + if not hasattr(ffi, '_assigned_source'): + error("%r: the set_source() method was not called" % (mod_spec,)) + module_name, source, source_extension, kwds = ffi._assigned_source + if ffi._windows_unicode: + kwds = kwds.copy() + ffi._apply_windows_unicode(kwds) + + if source is None: + _add_py_module(dist, ffi, module_name) + else: + _add_c_module(dist, ffi, module_name, source, source_extension, kwds) + +def _set_py_limited_api(Extension, kwds): + """ + Add py_limited_api to kwds if setuptools >= 26 is in use. + Do not alter the setting if it already exists. + Setuptools takes care of ignoring the flag on Python 2 and PyPy. + + CPython itself should ignore the flag in a debugging version + (by not listing .abi3.so in the extensions it supports), but + it doesn't so far, creating troubles. That's why we check + for "not hasattr(sys, 'gettotalrefcount')" (the 2.7 compatible equivalent + of 'd' not in sys.abiflags). (http://bugs.python.org/issue28401) + + On Windows, with CPython <= 3.4, it's better not to use py_limited_api + because virtualenv *still* doesn't copy PYTHON3.DLL on these versions. + Recently (2020) we started shipping only >= 3.5 wheels, though. So + we'll give it another try and set py_limited_api on Windows >= 3.5. + """ + from cffi._shimmed_dist_utils import log + from cffi import recompiler + + if ('py_limited_api' not in kwds and not hasattr(sys, 'gettotalrefcount') + and recompiler.USE_LIMITED_API): + import setuptools + try: + setuptools_major_version = int(setuptools.__version__.partition('.')[0]) + if setuptools_major_version >= 26: + kwds['py_limited_api'] = True + except ValueError: # certain development versions of setuptools + # If we don't know the version number of setuptools, we + # try to set 'py_limited_api' anyway. At worst, we get a + # warning. + kwds['py_limited_api'] = True + + if sysconfig.get_config_var("Py_GIL_DISABLED"): + if sys.version_info < (3, 15): + if kwds.get('py_limited_api'): + log.info("Ignoring py_limited_api=True for free-threaded build.") + kwds['py_limited_api'] = False + else: + kwds.setdefault("define_macros", []).append(("Py_TARGET_ABI3T", "0x030f0000")) + + if kwds.get('py_limited_api') is False: + # avoid setting Py_LIMITED_API if py_limited_api=False + # which _cffi_include.h does unless _CFFI_NO_LIMITED_API is defined + kwds.setdefault("define_macros", []).append(("_CFFI_NO_LIMITED_API", None)) + return kwds + +def _add_c_module(dist, ffi, module_name, source, source_extension, kwds): + # We are a setuptools extension. Need this build_ext for py_limited_api. + from setuptools.command.build_ext import build_ext + from cffi._shimmed_dist_utils import Extension, log, mkpath + from cffi import recompiler + + allsources = ['$PLACEHOLDER'] + allsources.extend(kwds.pop('sources', [])) + kwds = _set_py_limited_api(Extension, kwds) + ext = Extension(name=module_name, sources=allsources, **kwds) + + def make_mod(tmpdir, pre_run=None): + c_file = os.path.join(tmpdir, module_name + source_extension) + log.info("generating cffi module %r" % c_file) + mkpath(tmpdir) + # a setuptools-only, API-only hook: called with the "ext" and "ffi" + # arguments just before we turn the ffi into C code. To use it, + # subclass the 'distutils.command.build_ext.build_ext' class and + # add a method 'def pre_run(self, ext, ffi)'. + if pre_run is not None: + pre_run(ext, ffi) + updated = recompiler.make_c_source(ffi, module_name, source, c_file) + if not updated: + log.info("already up-to-date") + return c_file + + if dist.ext_modules is None: + dist.ext_modules = [] + dist.ext_modules.append(ext) + + base_class = dist.cmdclass.get('build_ext', build_ext) + class build_ext_make_mod(base_class): + def run(self): + if ext.sources[0] == '$PLACEHOLDER': + pre_run = getattr(self, 'pre_run', None) + ext.sources[0] = make_mod(self.build_temp, pre_run) + base_class.run(self) + dist.cmdclass['build_ext'] = build_ext_make_mod + # NB. multiple runs here will create multiple 'build_ext_make_mod' + # classes. Even in this case the 'build_ext' command should be + # run once; but just in case, the logic above does nothing if + # called again. + + +def _add_py_module(dist, ffi, module_name): + from setuptools.command.build_py import build_py + from setuptools.command.build_ext import build_ext + from cffi._shimmed_dist_utils import log, mkpath + from cffi import recompiler + + def generate_mod(py_file): + log.info("generating cffi module %r" % py_file) + mkpath(os.path.dirname(py_file)) + updated = recompiler.make_py_source(ffi, module_name, py_file) + if not updated: + log.info("already up-to-date") + + base_class = dist.cmdclass.get('build_py', build_py) + class build_py_make_mod(base_class): + def run(self): + base_class.run(self) + module_path = module_name.split('.') + module_path[-1] += '.py' + generate_mod(os.path.join(self.build_lib, *module_path)) + def get_source_files(self): + # This is called from 'setup.py sdist' only. Exclude + # the generate .py module in this case. + saved_py_modules = self.py_modules + try: + if saved_py_modules: + self.py_modules = [m for m in saved_py_modules + if m != module_name] + return base_class.get_source_files(self) + finally: + self.py_modules = saved_py_modules + dist.cmdclass['build_py'] = build_py_make_mod + + # distutils and setuptools have no notion I could find of a + # generated python module. If we don't add module_name to + # dist.py_modules, then things mostly work but there are some + # combination of options (--root and --record) that will miss + # the module. So we add it here, which gives a few apparently + # harmless warnings about not finding the file outside the + # build directory. + # Then we need to hack more in get_source_files(); see above. + if dist.py_modules is None: + dist.py_modules = [] + dist.py_modules.append(module_name) + + # the following is only for "build_ext -i" + base_class_2 = dist.cmdclass.get('build_ext', build_ext) + class build_ext_make_mod(base_class_2): + def run(self): + base_class_2.run(self) + if self.inplace: + # from get_ext_fullpath() in distutils/command/build_ext.py + module_path = module_name.split('.') + package = '.'.join(module_path[:-1]) + build_py = self.get_finalized_command('build_py') + package_dir = build_py.get_package_dir(package) + file_name = module_path[-1] + '.py' + generate_mod(os.path.join(package_dir, file_name)) + dist.cmdclass['build_ext'] = build_ext_make_mod + +def cffi_modules(dist, attr, value): + assert attr == 'cffi_modules' + if isinstance(value, basestring): + value = [value] + + for cffi_module in value: + add_cffi_module(dist, cffi_module) diff --git a/py_modules/cffi/vengine_cpy.py b/py_modules/cffi/vengine_cpy.py new file mode 100644 index 0000000..4d2ea43 --- /dev/null +++ b/py_modules/cffi/vengine_cpy.py @@ -0,0 +1,1058 @@ +# +# DEPRECATED: implementation for ffi.verify() +# +import sys +from . import model +from .error import VerificationError +from . import _imp_emulation as imp + + +class VCPythonEngine: + _class_key = 'x' + _gen_python_module = True + + def __init__(self, verifier): + self.verifier = verifier + self.ffi = verifier.ffi + self._struct_pending_verification = {} + self._types_of_builtin_functions = {} + + def patch_extension_kwds(self, kwds): + pass + + def find_module(self, module_name, path, so_suffixes): + try: + f, filename, descr = imp.find_module(module_name, path) + except ImportError: + return None + if f is not None: + f.close() + # Note that after a setuptools installation, there are both .py + # and .so files with the same basename. The code here relies on + # imp.find_module() locating the .so in priority. + if descr[0] not in so_suffixes: + return None + return filename + + def collect_types(self): + self._typesdict = {} + self._generate("collecttype") + + def _prnt(self, what=''): + self._f.write(what + '\n') + + def _gettypenum(self, type): + # a KeyError here is a bug. please report it! :-) + return self._typesdict[type] + + def _do_collect_type(self, tp): + if ((not isinstance(tp, model.PrimitiveType) + or tp.name == 'long double') + and tp not in self._typesdict): + num = len(self._typesdict) + self._typesdict[tp] = num + + def write_source_to_f(self): + self.collect_types() + # + # The new module will have a _cffi_setup() function that receives + # objects from the ffi world, and that calls some setup code in + # the module. This setup code is split in several independent + # functions, e.g. one per constant. The functions are "chained" + # by ending in a tail call to each other. + # + # This is further split in two chained lists, depending on if we + # can do it at import-time or if we must wait for _cffi_setup() to + # provide us with the objects. This is needed because we + # need the values of the enum constants in order to build the + # that we may have to pass to _cffi_setup(). + # + # The following two 'chained_list_constants' items contains + # the head of these two chained lists, as a string that gives the + # call to do, if any. + self._chained_list_constants = ['((void)lib,0)', '((void)lib,0)'] + # + prnt = self._prnt + # first paste some standard set of lines that are mostly '#define' + prnt(cffimod_header) + prnt() + # then paste the C source given by the user, verbatim. + prnt(self.verifier.preamble) + prnt() + # + # call generate_cpy_xxx_decl(), for every xxx found from + # ffi._parser._declarations. This generates all the functions. + self._generate("decl") + # + # implement the function _cffi_setup_custom() as calling the + # head of the chained list. + self._generate_setup_custom() + prnt() + # + # produce the method table, including the entries for the + # generated Python->C function wrappers, which are done + # by generate_cpy_function_method(). + prnt('static PyMethodDef _cffi_methods[] = {') + self._generate("method") + prnt(' {"_cffi_setup", _cffi_setup, METH_VARARGS, NULL},') + prnt(' {NULL, NULL, 0, NULL} /* Sentinel */') + prnt('};') + prnt() + # + # standard init. + modname = self.verifier.get_module_name() + constants = self._chained_list_constants[False] + prnt('static struct PyModuleDef _cffi_module_def = {') + prnt(' PyModuleDef_HEAD_INIT,') + prnt(' "%s",' % modname) + prnt(' NULL,') + prnt(' -1,') + prnt(' _cffi_methods,') + prnt(' NULL, NULL, NULL, NULL') + prnt('};') + prnt() + prnt('PyMODINIT_FUNC') + prnt('PyInit_%s(void)' % modname) + prnt('{') + prnt(' PyObject *lib;') + prnt(' lib = PyModule_Create(&_cffi_module_def);') + prnt(' if (lib == NULL)') + prnt(' return NULL;') + prnt(' if (%s < 0 || _cffi_init() < 0) {' % (constants,)) + prnt(' Py_DECREF(lib);') + prnt(' return NULL;') + prnt(' }') + prnt('#if Py_GIL_DISABLED') + prnt(' PyUnstable_Module_SetGIL(lib, Py_MOD_GIL_NOT_USED);') + prnt('#endif') + prnt(' return lib;') + prnt('}') + prnt() + + def load_library(self, flags=None): + # XXX review all usages of 'self' here! + # import it as a new extension module + imp.acquire_lock() + try: + if hasattr(sys, "getdlopenflags"): + previous_flags = sys.getdlopenflags() + try: + if hasattr(sys, "setdlopenflags") and flags is not None: + sys.setdlopenflags(flags) + module = imp.load_dynamic(self.verifier.get_module_name(), + self.verifier.modulefilename) + except ImportError as e: + error = "importing %r: %s" % (self.verifier.modulefilename, e) + raise VerificationError(error) + finally: + if hasattr(sys, "setdlopenflags"): + sys.setdlopenflags(previous_flags) + finally: + imp.release_lock() + # + # call loading_cpy_struct() to get the struct layout inferred by + # the C compiler + self._load(module, 'loading') + # + # the C code will need the objects. Collect them in + # order in a list. + revmapping = dict([(value, key) + for (key, value) in self._typesdict.items()]) + lst = [revmapping[i] for i in range(len(revmapping))] + lst = list(map(self.ffi._get_cached_btype, lst)) + # + # build the FFILibrary class and instance and call _cffi_setup(). + # this will set up some fields like '_cffi_types', and only then + # it will invoke the chained list of functions that will really + # build (notably) the constant objects, as if they are + # pointers, and store them as attributes on the 'library' object. + class FFILibrary: + _cffi_python_module = module + _cffi_ffi = self.ffi + _cffi_dir = [] + def __dir__(self): + return FFILibrary._cffi_dir + list(self.__dict__) + library = FFILibrary() + if module._cffi_setup(lst, VerificationError, library): + import warnings + warnings.warn("reimporting %r might overwrite older definitions" + % (self.verifier.get_module_name())) + # + # finally, call the loaded_cpy_xxx() functions. This will perform + # the final adjustments, like copying the Python->C wrapper + # functions from the module to the 'library' object, and setting + # up the FFILibrary class with properties for the global C variables. + self._load(module, 'loaded', library=library) + module._cffi_original_ffi = self.ffi + module._cffi_types_of_builtin_funcs = self._types_of_builtin_functions + return library + + def _get_declarations(self): + lst = [(key, tp) for (key, (tp, qual)) in + self.ffi._parser._declarations.items()] + lst.sort() + return lst + + def _generate(self, step_name): + for name, tp in self._get_declarations(): + kind, realname = name.split(' ', 1) + try: + method = getattr(self, '_generate_cpy_%s_%s' % (kind, + step_name)) + except AttributeError: + raise VerificationError( + "not implemented in verify(): %r" % name) + try: + method(tp, realname) + except Exception as e: + model.attach_exception_info(e, name) + raise + + def _load(self, module, step_name, **kwds): + for name, tp in self._get_declarations(): + kind, realname = name.split(' ', 1) + method = getattr(self, '_%s_cpy_%s' % (step_name, kind)) + try: + method(tp, realname, module, **kwds) + except Exception as e: + model.attach_exception_info(e, name) + raise + + def _generate_nothing(self, tp, name): + pass + + def _loaded_noop(self, tp, name, module, **kwds): + pass + + # ---------- + + def _convert_funcarg_to_c(self, tp, fromvar, tovar, errcode): + extraarg = '' + if isinstance(tp, model.PrimitiveType): + if tp.is_integer_type() and tp.name != '_Bool': + converter = '_cffi_to_c_int' + extraarg = ', %s' % tp.name + elif tp.is_complex_type(): + raise VerificationError( + "not implemented in verify(): complex types") + else: + converter = '(%s)_cffi_to_c_%s' % (tp.get_c_name(''), + tp.name.replace(' ', '_')) + errvalue = '-1' + # + elif isinstance(tp, model.PointerType): + self._convert_funcarg_to_c_ptr_or_array(tp, fromvar, + tovar, errcode) + return + # + elif isinstance(tp, (model.StructOrUnion, model.EnumType)): + # a struct (not a struct pointer) as a function argument + self._prnt(' if (_cffi_to_c((char *)&%s, _cffi_type(%d), %s) < 0)' + % (tovar, self._gettypenum(tp), fromvar)) + self._prnt(' %s;' % errcode) + return + # + elif isinstance(tp, model.FunctionPtrType): + converter = '(%s)_cffi_to_c_pointer' % tp.get_c_name('') + extraarg = ', _cffi_type(%d)' % self._gettypenum(tp) + errvalue = 'NULL' + # + else: + raise NotImplementedError(tp) + # + self._prnt(' %s = %s(%s%s);' % (tovar, converter, fromvar, extraarg)) + self._prnt(' if (%s == (%s)%s && PyErr_Occurred())' % ( + tovar, tp.get_c_name(''), errvalue)) + self._prnt(' %s;' % errcode) + + def _extra_local_variables(self, tp, localvars, freelines): + if isinstance(tp, model.PointerType): + localvars.add('Py_ssize_t datasize') + localvars.add('struct _cffi_freeme_s *large_args_free = NULL') + freelines.add('if (large_args_free != NULL)' + ' _cffi_free_array_arguments(large_args_free);') + + def _convert_funcarg_to_c_ptr_or_array(self, tp, fromvar, tovar, errcode): + self._prnt(' datasize = _cffi_prepare_pointer_call_argument(') + self._prnt(' _cffi_type(%d), %s, (char **)&%s);' % ( + self._gettypenum(tp), fromvar, tovar)) + self._prnt(' if (datasize != 0) {') + self._prnt(' %s = ((size_t)datasize) <= 640 ? ' + 'alloca((size_t)datasize) : NULL;' % (tovar,)) + self._prnt(' if (_cffi_convert_array_argument(_cffi_type(%d), %s, ' + '(char **)&%s,' % (self._gettypenum(tp), fromvar, tovar)) + self._prnt(' datasize, &large_args_free) < 0)') + self._prnt(' %s;' % errcode) + self._prnt(' }') + + def _convert_expr_from_c(self, tp, var, context): + if isinstance(tp, model.PrimitiveType): + if tp.is_integer_type() and tp.name != '_Bool': + return '_cffi_from_c_int(%s, %s)' % (var, tp.name) + elif tp.name != 'long double': + return '_cffi_from_c_%s(%s)' % (tp.name.replace(' ', '_'), var) + else: + return '_cffi_from_c_deref((char *)&%s, _cffi_type(%d))' % ( + var, self._gettypenum(tp)) + elif isinstance(tp, (model.PointerType, model.FunctionPtrType)): + return '_cffi_from_c_pointer((char *)%s, _cffi_type(%d))' % ( + var, self._gettypenum(tp)) + elif isinstance(tp, model.ArrayType): + return '_cffi_from_c_pointer((char *)%s, _cffi_type(%d))' % ( + var, self._gettypenum(model.PointerType(tp.item))) + elif isinstance(tp, model.StructOrUnion): + if tp.fldnames is None: + raise TypeError("'%s' is used as %s, but is opaque" % ( + tp._get_c_name(), context)) + return '_cffi_from_c_struct((char *)&%s, _cffi_type(%d))' % ( + var, self._gettypenum(tp)) + elif isinstance(tp, model.EnumType): + return '_cffi_from_c_deref((char *)&%s, _cffi_type(%d))' % ( + var, self._gettypenum(tp)) + else: + raise NotImplementedError(tp) + + # ---------- + # typedefs: generates no code so far + + _generate_cpy_typedef_collecttype = _generate_nothing + _generate_cpy_typedef_decl = _generate_nothing + _generate_cpy_typedef_method = _generate_nothing + _loading_cpy_typedef = _loaded_noop + _loaded_cpy_typedef = _loaded_noop + + # ---------- + # function declarations + + def _generate_cpy_function_collecttype(self, tp, name): + assert isinstance(tp, model.FunctionPtrType) + if tp.ellipsis: + self._do_collect_type(tp) + else: + # don't call _do_collect_type(tp) in this common case, + # otherwise test_autofilled_struct_as_argument fails + for type in tp.args: + self._do_collect_type(type) + self._do_collect_type(tp.result) + + def _generate_cpy_function_decl(self, tp, name): + assert isinstance(tp, model.FunctionPtrType) + if tp.ellipsis: + # cannot support vararg functions better than this: check for its + # exact type (including the fixed arguments), and build it as a + # constant function pointer (no CPython wrapper) + self._generate_cpy_const(False, name, tp) + return + prnt = self._prnt + numargs = len(tp.args) + if numargs == 0: + argname = 'noarg' + elif numargs == 1: + argname = 'arg0' + else: + argname = 'args' + prnt('static PyObject *') + prnt('_cffi_f_%s(PyObject *self, PyObject *%s)' % (name, argname)) + prnt('{') + # + context = 'argument of %s' % name + for i, type in enumerate(tp.args): + prnt(' %s;' % type.get_c_name(' x%d' % i, context)) + # + localvars = set() + freelines = set() + for type in tp.args: + self._extra_local_variables(type, localvars, freelines) + for decl in sorted(localvars): + prnt(' %s;' % (decl,)) + # + if not isinstance(tp.result, model.VoidType): + result_code = 'result = ' + context = 'result of %s' % name + prnt(' %s;' % tp.result.get_c_name(' result', context)) + prnt(' PyObject *pyresult;') + else: + result_code = '' + # + if len(tp.args) > 1: + rng = range(len(tp.args)) + for i in rng: + prnt(' PyObject *arg%d;' % i) + prnt() + prnt(' if (!PyArg_ParseTuple(args, "%s:%s", %s))' % ( + 'O' * numargs, name, ', '.join(['&arg%d' % i for i in rng]))) + prnt(' return NULL;') + prnt() + # + for i, type in enumerate(tp.args): + self._convert_funcarg_to_c(type, 'arg%d' % i, 'x%d' % i, + 'return NULL') + prnt() + # + prnt(' Py_BEGIN_ALLOW_THREADS') + prnt(' _cffi_restore_errno();') + prnt(' { %s%s(%s); }' % ( + result_code, name, + ', '.join(['x%d' % i for i in range(len(tp.args))]))) + prnt(' _cffi_save_errno();') + prnt(' Py_END_ALLOW_THREADS') + prnt() + # + prnt(' (void)self; /* unused */') + if numargs == 0: + prnt(' (void)noarg; /* unused */') + if result_code: + prnt(' pyresult = %s;' % + self._convert_expr_from_c(tp.result, 'result', 'result type')) + for freeline in freelines: + prnt(' ' + freeline) + prnt(' return pyresult;') + else: + for freeline in freelines: + prnt(' ' + freeline) + prnt(' Py_INCREF(Py_None);') + prnt(' return Py_None;') + prnt('}') + prnt() + + def _generate_cpy_function_method(self, tp, name): + if tp.ellipsis: + return + numargs = len(tp.args) + if numargs == 0: + meth = 'METH_NOARGS' + elif numargs == 1: + meth = 'METH_O' + else: + meth = 'METH_VARARGS' + self._prnt(' {"%s", _cffi_f_%s, %s, NULL},' % (name, name, meth)) + + _loading_cpy_function = _loaded_noop + + def _loaded_cpy_function(self, tp, name, module, library): + if tp.ellipsis: + return + func = getattr(module, name) + setattr(library, name, func) + self._types_of_builtin_functions[func] = tp + + # ---------- + # named structs + + _generate_cpy_struct_collecttype = _generate_nothing + def _generate_cpy_struct_decl(self, tp, name): + assert name == tp.name + self._generate_struct_or_union_decl(tp, 'struct', name) + def _generate_cpy_struct_method(self, tp, name): + self._generate_struct_or_union_method(tp, 'struct', name) + def _loading_cpy_struct(self, tp, name, module): + self._loading_struct_or_union(tp, 'struct', name, module) + def _loaded_cpy_struct(self, tp, name, module, **kwds): + self._loaded_struct_or_union(tp) + + _generate_cpy_union_collecttype = _generate_nothing + def _generate_cpy_union_decl(self, tp, name): + assert name == tp.name + self._generate_struct_or_union_decl(tp, 'union', name) + def _generate_cpy_union_method(self, tp, name): + self._generate_struct_or_union_method(tp, 'union', name) + def _loading_cpy_union(self, tp, name, module): + self._loading_struct_or_union(tp, 'union', name, module) + def _loaded_cpy_union(self, tp, name, module, **kwds): + self._loaded_struct_or_union(tp) + + def _generate_struct_or_union_decl(self, tp, prefix, name): + if tp.fldnames is None: + return # nothing to do with opaque structs + checkfuncname = '_cffi_check_%s_%s' % (prefix, name) + layoutfuncname = '_cffi_layout_%s_%s' % (prefix, name) + cname = ('%s %s' % (prefix, name)).strip() + # + prnt = self._prnt + prnt('static void %s(%s *p)' % (checkfuncname, cname)) + prnt('{') + prnt(' /* only to generate compile-time warnings or errors */') + prnt(' (void)p;') + for fname, ftype, fbitsize, fqual in tp.enumfields(): + if (isinstance(ftype, model.PrimitiveType) + and ftype.is_integer_type()) or fbitsize >= 0: + # accept all integers, but complain on float or double + prnt(' (void)((p->%s) << 1);' % fname) + else: + # only accept exactly the type declared. + try: + prnt(' { %s = &p->%s; (void)tmp; }' % ( + ftype.get_c_name('*tmp', 'field %r'%fname, quals=fqual), + fname)) + except VerificationError as e: + prnt(' /* %s */' % str(e)) # cannot verify it, ignore + prnt('}') + prnt('static PyObject *') + prnt('%s(PyObject *self, PyObject *noarg)' % (layoutfuncname,)) + prnt('{') + prnt(' struct _cffi_aligncheck { char x; %s y; };' % cname) + prnt(' static Py_ssize_t nums[] = {') + prnt(' sizeof(%s),' % cname) + prnt(' offsetof(struct _cffi_aligncheck, y),') + for fname, ftype, fbitsize, fqual in tp.enumfields(): + if fbitsize >= 0: + continue # xxx ignore fbitsize for now + prnt(' offsetof(%s, %s),' % (cname, fname)) + if isinstance(ftype, model.ArrayType) and ftype.length is None: + prnt(' 0, /* %s */' % ftype._get_c_name()) + else: + prnt(' sizeof(((%s *)0)->%s),' % (cname, fname)) + prnt(' -1') + prnt(' };') + prnt(' (void)self; /* unused */') + prnt(' (void)noarg; /* unused */') + prnt(' return _cffi_get_struct_layout(nums);') + prnt(' /* the next line is not executed, but compiled */') + prnt(' %s(0);' % (checkfuncname,)) + prnt('}') + prnt() + + def _generate_struct_or_union_method(self, tp, prefix, name): + if tp.fldnames is None: + return # nothing to do with opaque structs + layoutfuncname = '_cffi_layout_%s_%s' % (prefix, name) + self._prnt(' {"%s", %s, METH_NOARGS, NULL},' % (layoutfuncname, + layoutfuncname)) + + def _loading_struct_or_union(self, tp, prefix, name, module): + if tp.fldnames is None: + return # nothing to do with opaque structs + layoutfuncname = '_cffi_layout_%s_%s' % (prefix, name) + # + function = getattr(module, layoutfuncname) + layout = function() + if isinstance(tp, model.StructOrUnion) and tp.partial: + # use the function()'s sizes and offsets to guide the + # layout of the struct + totalsize = layout[0] + totalalignment = layout[1] + fieldofs = layout[2::2] + fieldsize = layout[3::2] + tp.force_flatten() + assert len(fieldofs) == len(fieldsize) == len(tp.fldnames) + tp.fixedlayout = fieldofs, fieldsize, totalsize, totalalignment + else: + cname = ('%s %s' % (prefix, name)).strip() + self._struct_pending_verification[tp] = layout, cname + + def _loaded_struct_or_union(self, tp): + if tp.fldnames is None: + return # nothing to do with opaque structs + self.ffi._get_cached_btype(tp) # force 'fixedlayout' to be considered + + if tp in self._struct_pending_verification: + # check that the layout sizes and offsets match the real ones + def check(realvalue, expectedvalue, msg): + if realvalue != expectedvalue: + raise VerificationError( + "%s (we have %d, but C compiler says %d)" + % (msg, expectedvalue, realvalue)) + ffi = self.ffi + BStruct = ffi._get_cached_btype(tp) + layout, cname = self._struct_pending_verification.pop(tp) + check(layout[0], ffi.sizeof(BStruct), "wrong total size") + check(layout[1], ffi.alignof(BStruct), "wrong total alignment") + i = 2 + for fname, ftype, fbitsize, fqual in tp.enumfields(): + if fbitsize >= 0: + continue # xxx ignore fbitsize for now + check(layout[i], ffi.offsetof(BStruct, fname), + "wrong offset for field %r" % (fname,)) + if layout[i+1] != 0: + BField = ffi._get_cached_btype(ftype) + check(layout[i+1], ffi.sizeof(BField), + "wrong size for field %r" % (fname,)) + i += 2 + assert i == len(layout) + + # ---------- + # 'anonymous' declarations. These are produced for anonymous structs + # or unions; the 'name' is obtained by a typedef. + + _generate_cpy_anonymous_collecttype = _generate_nothing + + def _generate_cpy_anonymous_decl(self, tp, name): + if isinstance(tp, model.EnumType): + self._generate_cpy_enum_decl(tp, name, '') + else: + self._generate_struct_or_union_decl(tp, '', name) + + def _generate_cpy_anonymous_method(self, tp, name): + if not isinstance(tp, model.EnumType): + self._generate_struct_or_union_method(tp, '', name) + + def _loading_cpy_anonymous(self, tp, name, module): + if isinstance(tp, model.EnumType): + self._loading_cpy_enum(tp, name, module) + else: + self._loading_struct_or_union(tp, '', name, module) + + def _loaded_cpy_anonymous(self, tp, name, module, **kwds): + if isinstance(tp, model.EnumType): + self._loaded_cpy_enum(tp, name, module, **kwds) + else: + self._loaded_struct_or_union(tp) + + # ---------- + # constants, likely declared with '#define' + + def _generate_cpy_const(self, is_int, name, tp=None, category='const', + vartp=None, delayed=True, size_too=False, + check_value=None): + prnt = self._prnt + funcname = '_cffi_%s_%s' % (category, name) + prnt('static int %s(PyObject *lib)' % funcname) + prnt('{') + prnt(' PyObject *o;') + prnt(' int res;') + if not is_int: + prnt(' %s;' % (vartp or tp).get_c_name(' i', name)) + else: + assert category == 'const' + # + if check_value is not None: + self._check_int_constant_value(name, check_value) + # + if not is_int: + if category == 'var': + realexpr = '&' + name + else: + realexpr = name + prnt(' i = (%s);' % (realexpr,)) + prnt(' o = %s;' % (self._convert_expr_from_c(tp, 'i', + 'variable type'),)) + assert delayed + else: + prnt(' o = _cffi_from_c_int_const(%s);' % name) + prnt(' if (o == NULL)') + prnt(' return -1;') + if size_too: + prnt(' {') + prnt(' PyObject *o1 = o;') + prnt(' o = Py_BuildValue("On", o1, (Py_ssize_t)sizeof(%s));' + % (name,)) + prnt(' Py_DECREF(o1);') + prnt(' if (o == NULL)') + prnt(' return -1;') + prnt(' }') + prnt(' res = PyObject_SetAttrString(lib, "%s", o);' % name) + prnt(' Py_DECREF(o);') + prnt(' if (res < 0)') + prnt(' return -1;') + prnt(' return %s;' % self._chained_list_constants[delayed]) + self._chained_list_constants[delayed] = funcname + '(lib)' + prnt('}') + prnt() + + def _generate_cpy_constant_collecttype(self, tp, name): + is_int = isinstance(tp, model.PrimitiveType) and tp.is_integer_type() + if not is_int: + self._do_collect_type(tp) + + def _generate_cpy_constant_decl(self, tp, name): + is_int = isinstance(tp, model.PrimitiveType) and tp.is_integer_type() + self._generate_cpy_const(is_int, name, tp) + + _generate_cpy_constant_method = _generate_nothing + _loading_cpy_constant = _loaded_noop + _loaded_cpy_constant = _loaded_noop + + # ---------- + # enums + + def _check_int_constant_value(self, name, value, err_prefix=''): + prnt = self._prnt + if value <= 0: + prnt(' if ((%s) > 0 || (long)(%s) != %dL) {' % ( + name, name, value)) + else: + prnt(' if ((%s) <= 0 || (unsigned long)(%s) != %dUL) {' % ( + name, name, value)) + prnt(' char buf[64];') + prnt(' if ((%s) <= 0)' % name) + prnt(' snprintf(buf, 63, "%%ld", (long)(%s));' % name) + prnt(' else') + prnt(' snprintf(buf, 63, "%%lu", (unsigned long)(%s));' % + name) + prnt(' PyErr_Format(_cffi_VerificationError,') + prnt(' "%s%s has the real value %s, not %s",') + prnt(' "%s", "%s", buf, "%d");' % ( + err_prefix, name, value)) + prnt(' return -1;') + prnt(' }') + + def _enum_funcname(self, prefix, name): + # "$enum_$1" => "___D_enum____D_1" + name = name.replace('$', '___D_') + return '_cffi_e_%s_%s' % (prefix, name) + + def _generate_cpy_enum_decl(self, tp, name, prefix='enum'): + if tp.partial: + for enumerator in tp.enumerators: + self._generate_cpy_const(True, enumerator, delayed=False) + return + # + funcname = self._enum_funcname(prefix, name) + prnt = self._prnt + prnt('static int %s(PyObject *lib)' % funcname) + prnt('{') + for enumerator, enumvalue in zip(tp.enumerators, tp.enumvalues): + self._check_int_constant_value(enumerator, enumvalue, + "enum %s: " % name) + prnt(' return %s;' % self._chained_list_constants[True]) + self._chained_list_constants[True] = funcname + '(lib)' + prnt('}') + prnt() + + _generate_cpy_enum_collecttype = _generate_nothing + _generate_cpy_enum_method = _generate_nothing + + def _loading_cpy_enum(self, tp, name, module): + if tp.partial: + enumvalues = [getattr(module, enumerator) + for enumerator in tp.enumerators] + tp.enumvalues = tuple(enumvalues) + tp.partial_resolved = True + + def _loaded_cpy_enum(self, tp, name, module, library): + for enumerator, enumvalue in zip(tp.enumerators, tp.enumvalues): + setattr(library, enumerator, enumvalue) + + # ---------- + # macros: for now only for integers + + def _generate_cpy_macro_decl(self, tp, name): + if tp == '...': + check_value = None + else: + check_value = tp # an integer + self._generate_cpy_const(True, name, check_value=check_value) + + _generate_cpy_macro_collecttype = _generate_nothing + _generate_cpy_macro_method = _generate_nothing + _loading_cpy_macro = _loaded_noop + _loaded_cpy_macro = _loaded_noop + + # ---------- + # global variables + + def _generate_cpy_variable_collecttype(self, tp, name): + if isinstance(tp, model.ArrayType): + tp_ptr = model.PointerType(tp.item) + else: + tp_ptr = model.PointerType(tp) + self._do_collect_type(tp_ptr) + + def _generate_cpy_variable_decl(self, tp, name): + if isinstance(tp, model.ArrayType): + tp_ptr = model.PointerType(tp.item) + self._generate_cpy_const(False, name, tp, vartp=tp_ptr, + size_too = tp.length_is_unknown()) + else: + tp_ptr = model.PointerType(tp) + self._generate_cpy_const(False, name, tp_ptr, category='var') + + _generate_cpy_variable_method = _generate_nothing + _loading_cpy_variable = _loaded_noop + + def _loaded_cpy_variable(self, tp, name, module, library): + value = getattr(library, name) + if isinstance(tp, model.ArrayType): # int a[5] is "constant" in the + # sense that "a=..." is forbidden + if tp.length_is_unknown(): + assert isinstance(value, tuple) + (value, size) = value + BItemType = self.ffi._get_cached_btype(tp.item) + length, rest = divmod(size, self.ffi.sizeof(BItemType)) + if rest != 0: + raise VerificationError( + "bad size: %r does not seem to be an array of %s" % + (name, tp.item)) + tp = tp.resolve_length(length) + # 'value' is a which we have to replace with + # a if the N is actually known + if tp.length is not None: + BArray = self.ffi._get_cached_btype(tp) + value = self.ffi.cast(BArray, value) + setattr(library, name, value) + return + # remove ptr= from the library instance, and replace + # it by a property on the class, which reads/writes into ptr[0]. + ptr = value + delattr(library, name) + def getter(library): + return ptr[0] + def setter(library, value): + ptr[0] = value + setattr(type(library), name, property(getter, setter)) + type(library)._cffi_dir.append(name) + + # ---------- + + def _generate_setup_custom(self): + prnt = self._prnt + prnt('static int _cffi_setup_custom(PyObject *lib)') + prnt('{') + prnt(' return %s;' % self._chained_list_constants[True]) + prnt('}') + +cffimod_header = r''' +#include +#include + +/* this block of #ifs should be kept exactly identical between + c/_cffi_backend.c, cffi/vengine_cpy.py, cffi/vengine_gen.py + and cffi/_cffi_include.h */ +#if defined(_MSC_VER) +# include /* for alloca() */ +# if _MSC_VER < 1600 /* MSVC < 2010 */ + typedef __int8 int8_t; + typedef __int16 int16_t; + typedef __int32 int32_t; + typedef __int64 int64_t; + typedef unsigned __int8 uint8_t; + typedef unsigned __int16 uint16_t; + typedef unsigned __int32 uint32_t; + typedef unsigned __int64 uint64_t; + typedef __int8 int_least8_t; + typedef __int16 int_least16_t; + typedef __int32 int_least32_t; + typedef __int64 int_least64_t; + typedef unsigned __int8 uint_least8_t; + typedef unsigned __int16 uint_least16_t; + typedef unsigned __int32 uint_least32_t; + typedef unsigned __int64 uint_least64_t; + typedef __int8 int_fast8_t; + typedef __int16 int_fast16_t; + typedef __int32 int_fast32_t; + typedef __int64 int_fast64_t; + typedef unsigned __int8 uint_fast8_t; + typedef unsigned __int16 uint_fast16_t; + typedef unsigned __int32 uint_fast32_t; + typedef unsigned __int64 uint_fast64_t; + typedef __int64 intmax_t; + typedef unsigned __int64 uintmax_t; +# else +# include +# endif +# if _MSC_VER < 1800 /* MSVC < 2013 */ +# ifndef __cplusplus + typedef unsigned char _Bool; +# endif +# endif +# define _cffi_float_complex_t _Fcomplex /* include for it */ +# define _cffi_double_complex_t _Dcomplex /* include for it */ +#else +# include +# if (defined (__SVR4) && defined (__sun)) || defined(_AIX) || defined(__hpux) +# include +# endif +# define _cffi_float_complex_t float _Complex +# define _cffi_double_complex_t double _Complex +#endif + +#define _cffi_from_c_double PyFloat_FromDouble +#define _cffi_from_c_float PyFloat_FromDouble +#define _cffi_from_c_long PyLong_FromLong +#define _cffi_from_c_ulong PyLong_FromUnsignedLong +#define _cffi_from_c_longlong PyLong_FromLongLong +#define _cffi_from_c_ulonglong PyLong_FromUnsignedLongLong +#define _cffi_from_c__Bool PyBool_FromLong + +#define _cffi_to_c_double PyFloat_AsDouble +#define _cffi_to_c_float PyFloat_AsDouble + +#define _cffi_from_c_int_const(x) \ + (((x) > 0) ? \ + ((unsigned long long)(x) <= (unsigned long long)LONG_MAX) ? \ + PyLong_FromLong((long)(x)) : \ + PyLong_FromUnsignedLongLong((unsigned long long)(x)) : \ + ((long long)(x) >= (long long)LONG_MIN) ? \ + PyLong_FromLong((long)(x)) : \ + PyLong_FromLongLong((long long)(x))) + +#define _cffi_from_c_int(x, type) \ + (((type)-1) > 0 ? /* unsigned */ \ + (sizeof(type) < sizeof(long) ? \ + PyLong_FromLong((long)x) : \ + sizeof(type) == sizeof(long) ? \ + PyLong_FromUnsignedLong((unsigned long)x) : \ + PyLong_FromUnsignedLongLong((unsigned long long)x)) : \ + (sizeof(type) <= sizeof(long) ? \ + PyLong_FromLong((long)x) : \ + PyLong_FromLongLong((long long)x))) + +#define _cffi_to_c_int(o, type) \ + ((type)( \ + sizeof(type) == 1 ? (((type)-1) > 0 ? (type)_cffi_to_c_u8(o) \ + : (type)_cffi_to_c_i8(o)) : \ + sizeof(type) == 2 ? (((type)-1) > 0 ? (type)_cffi_to_c_u16(o) \ + : (type)_cffi_to_c_i16(o)) : \ + sizeof(type) == 4 ? (((type)-1) > 0 ? (type)_cffi_to_c_u32(o) \ + : (type)_cffi_to_c_i32(o)) : \ + sizeof(type) == 8 ? (((type)-1) > 0 ? (type)_cffi_to_c_u64(o) \ + : (type)_cffi_to_c_i64(o)) : \ + (Py_FatalError("unsupported size for type " #type), (type)0))) + +#define _cffi_to_c_i8 \ + ((int(*)(PyObject *))_cffi_exports[1]) +#define _cffi_to_c_u8 \ + ((int(*)(PyObject *))_cffi_exports[2]) +#define _cffi_to_c_i16 \ + ((int(*)(PyObject *))_cffi_exports[3]) +#define _cffi_to_c_u16 \ + ((int(*)(PyObject *))_cffi_exports[4]) +#define _cffi_to_c_i32 \ + ((int(*)(PyObject *))_cffi_exports[5]) +#define _cffi_to_c_u32 \ + ((unsigned int(*)(PyObject *))_cffi_exports[6]) +#define _cffi_to_c_i64 \ + ((long long(*)(PyObject *))_cffi_exports[7]) +#define _cffi_to_c_u64 \ + ((unsigned long long(*)(PyObject *))_cffi_exports[8]) +#define _cffi_to_c_char \ + ((int(*)(PyObject *))_cffi_exports[9]) +#define _cffi_from_c_pointer \ + ((PyObject *(*)(char *, CTypeDescrObject *))_cffi_exports[10]) +#define _cffi_to_c_pointer \ + ((char *(*)(PyObject *, CTypeDescrObject *))_cffi_exports[11]) +#define _cffi_get_struct_layout \ + ((PyObject *(*)(Py_ssize_t[]))_cffi_exports[12]) +#define _cffi_restore_errno \ + ((void(*)(void))_cffi_exports[13]) +#define _cffi_save_errno \ + ((void(*)(void))_cffi_exports[14]) +#define _cffi_from_c_char \ + ((PyObject *(*)(char))_cffi_exports[15]) +#define _cffi_from_c_deref \ + ((PyObject *(*)(char *, CTypeDescrObject *))_cffi_exports[16]) +#define _cffi_to_c \ + ((int(*)(char *, CTypeDescrObject *, PyObject *))_cffi_exports[17]) +#define _cffi_from_c_struct \ + ((PyObject *(*)(char *, CTypeDescrObject *))_cffi_exports[18]) +#define _cffi_to_c_wchar_t \ + ((wchar_t(*)(PyObject *))_cffi_exports[19]) +#define _cffi_from_c_wchar_t \ + ((PyObject *(*)(wchar_t))_cffi_exports[20]) +#define _cffi_to_c_long_double \ + ((long double(*)(PyObject *))_cffi_exports[21]) +#define _cffi_to_c__Bool \ + ((_Bool(*)(PyObject *))_cffi_exports[22]) +#define _cffi_prepare_pointer_call_argument \ + ((Py_ssize_t(*)(CTypeDescrObject *, PyObject *, char **))_cffi_exports[23]) +#define _cffi_convert_array_from_object \ + ((int(*)(char *, CTypeDescrObject *, PyObject *))_cffi_exports[24]) +#define _CFFI_NUM_EXPORTS 25 + +typedef struct _ctypedescr CTypeDescrObject; + +static void *_cffi_exports[_CFFI_NUM_EXPORTS]; +static PyObject *_cffi_types, *_cffi_VerificationError; + +static int _cffi_setup_custom(PyObject *lib); /* forward */ + +static PyObject *_cffi_setup(PyObject *self, PyObject *args) +{ + PyObject *library; + int was_alive = (_cffi_types != NULL); + (void)self; /* unused */ + if (!PyArg_ParseTuple(args, "OOO", &_cffi_types, &_cffi_VerificationError, + &library)) + return NULL; + Py_INCREF(_cffi_types); + Py_INCREF(_cffi_VerificationError); + if (_cffi_setup_custom(library) < 0) + return NULL; + return PyBool_FromLong(was_alive); +} + +union _cffi_union_alignment_u { + unsigned char m_char; + unsigned short m_short; + unsigned int m_int; + unsigned long m_long; + unsigned long long m_longlong; + float m_float; + double m_double; + long double m_longdouble; +}; + +struct _cffi_freeme_s { + struct _cffi_freeme_s *next; + union _cffi_union_alignment_u alignment; +}; + +#ifdef __GNUC__ + __attribute__((unused)) +#endif +static int _cffi_convert_array_argument(CTypeDescrObject *ctptr, PyObject *arg, + char **output_data, Py_ssize_t datasize, + struct _cffi_freeme_s **freeme) +{ + char *p; + if (datasize < 0) + return -1; + + p = *output_data; + if (p == NULL) { + struct _cffi_freeme_s *fp = (struct _cffi_freeme_s *)PyObject_Malloc( + offsetof(struct _cffi_freeme_s, alignment) + (size_t)datasize); + if (fp == NULL) + return -1; + fp->next = *freeme; + *freeme = fp; + p = *output_data = (char *)&fp->alignment; + } + memset((void *)p, 0, (size_t)datasize); + return _cffi_convert_array_from_object(p, ctptr, arg); +} + +#ifdef __GNUC__ + __attribute__((unused)) +#endif +static void _cffi_free_array_arguments(struct _cffi_freeme_s *freeme) +{ + do { + void *p = (void *)freeme; + freeme = freeme->next; + PyObject_Free(p); + } while (freeme != NULL); +} + +static int _cffi_init(void) +{ + PyObject *module, *c_api_object = NULL; + + module = PyImport_ImportModule("_cffi_backend"); + if (module == NULL) + goto failure; + + c_api_object = PyObject_GetAttrString(module, "_C_API"); + if (c_api_object == NULL) + goto failure; + if (!PyCapsule_CheckExact(c_api_object)) { + PyErr_SetNone(PyExc_ImportError); + goto failure; + } + memcpy(_cffi_exports, PyCapsule_GetPointer(c_api_object, "cffi"), + _CFFI_NUM_EXPORTS * sizeof(void *)); + + Py_DECREF(module); + Py_DECREF(c_api_object); + return 0; + + failure: + Py_XDECREF(module); + Py_XDECREF(c_api_object); + return -1; +} + +#define _cffi_type(num) ((CTypeDescrObject *)PyList_GET_ITEM(_cffi_types, num)) + +/**********/ +''' diff --git a/py_modules/cffi/vengine_gen.py b/py_modules/cffi/vengine_gen.py new file mode 100644 index 0000000..0209c79 --- /dev/null +++ b/py_modules/cffi/vengine_gen.py @@ -0,0 +1,679 @@ +# +# DEPRECATED: implementation for ffi.verify() +# +import sys, os +import types + +from . import model +from .error import VerificationError + + +class VGenericEngine: + _class_key = 'g' + _gen_python_module = False + + def __init__(self, verifier): + self.verifier = verifier + self.ffi = verifier.ffi + self.export_symbols = [] + self._struct_pending_verification = {} + + def patch_extension_kwds(self, kwds): + # add 'export_symbols' to the dictionary. Note that we add the + # list before filling it. When we fill it, it will thus also show + # up in kwds['export_symbols']. + kwds.setdefault('export_symbols', self.export_symbols) + + def find_module(self, module_name, path, so_suffixes): + for so_suffix in so_suffixes: + basename = module_name + so_suffix + if path is None: + path = sys.path + for dirname in path: + filename = os.path.join(dirname, basename) + if os.path.isfile(filename): + return filename + + def collect_types(self): + pass # not needed in the generic engine + + def _prnt(self, what=''): + self._f.write(what + '\n') + + def write_source_to_f(self): + prnt = self._prnt + # first paste some standard set of lines that are mostly '#include' + prnt(cffimod_header) + # then paste the C source given by the user, verbatim. + prnt(self.verifier.preamble) + # + # call generate_gen_xxx_decl(), for every xxx found from + # ffi._parser._declarations. This generates all the functions. + self._generate('decl') + # + # on Windows, distutils insists on putting init_cffi_xyz in + # 'export_symbols', so instead of fighting it, just give up and + # give it one + if sys.platform == 'win32': + if sys.version_info >= (3,): + prefix = 'PyInit_' + else: + prefix = 'init' + modname = self.verifier.get_module_name() + prnt("void %s%s(void) { }\n" % (prefix, modname)) + + def load_library(self, flags=0): + # import it with the CFFI backend + backend = self.ffi._backend + # needs to make a path that contains '/', on Posix + filename = os.path.join(os.curdir, self.verifier.modulefilename) + module = backend.load_library(filename, flags) + # + # call loading_gen_struct() to get the struct layout inferred by + # the C compiler + self._load(module, 'loading') + + # build the FFILibrary class and instance, this is a module subclass + # because modules are expected to have usually-constant-attributes and + # in PyPy this means the JIT is able to treat attributes as constant, + # which we want. + class FFILibrary(types.ModuleType): + _cffi_generic_module = module + _cffi_ffi = self.ffi + _cffi_dir = [] + def __dir__(self): + return FFILibrary._cffi_dir + library = FFILibrary("") + # + # finally, call the loaded_gen_xxx() functions. This will set + # up the 'library' object. + self._load(module, 'loaded', library=library) + return library + + def _get_declarations(self): + lst = [(key, tp) for (key, (tp, qual)) in + self.ffi._parser._declarations.items()] + lst.sort() + return lst + + def _generate(self, step_name): + for name, tp in self._get_declarations(): + kind, realname = name.split(' ', 1) + try: + method = getattr(self, '_generate_gen_%s_%s' % (kind, + step_name)) + except AttributeError: + raise VerificationError( + "not implemented in verify(): %r" % name) + try: + method(tp, realname) + except Exception as e: + model.attach_exception_info(e, name) + raise + + def _load(self, module, step_name, **kwds): + for name, tp in self._get_declarations(): + kind, realname = name.split(' ', 1) + method = getattr(self, '_%s_gen_%s' % (step_name, kind)) + try: + method(tp, realname, module, **kwds) + except Exception as e: + model.attach_exception_info(e, name) + raise + + def _generate_nothing(self, tp, name): + pass + + def _loaded_noop(self, tp, name, module, **kwds): + pass + + # ---------- + # typedefs: generates no code so far + + _generate_gen_typedef_decl = _generate_nothing + _loading_gen_typedef = _loaded_noop + _loaded_gen_typedef = _loaded_noop + + # ---------- + # function declarations + + def _generate_gen_function_decl(self, tp, name): + assert isinstance(tp, model.FunctionPtrType) + if tp.ellipsis: + # cannot support vararg functions better than this: check for its + # exact type (including the fixed arguments), and build it as a + # constant function pointer (no _cffi_f_%s wrapper) + self._generate_gen_const(False, name, tp) + return + prnt = self._prnt + numargs = len(tp.args) + argnames = [] + for i, type in enumerate(tp.args): + indirection = '' + if isinstance(type, model.StructOrUnion): + indirection = '*' + argnames.append('%sx%d' % (indirection, i)) + context = 'argument of %s' % name + arglist = [type.get_c_name(' %s' % arg, context) + for type, arg in zip(tp.args, argnames)] + tpresult = tp.result + if isinstance(tpresult, model.StructOrUnion): + arglist.insert(0, tpresult.get_c_name(' *r', context)) + tpresult = model.void_type + arglist = ', '.join(arglist) or 'void' + wrappername = '_cffi_f_%s' % name + self.export_symbols.append(wrappername) + if tp.abi: + abi = tp.abi + ' ' + else: + abi = '' + funcdecl = ' %s%s(%s)' % (abi, wrappername, arglist) + context = 'result of %s' % name + prnt(tpresult.get_c_name(funcdecl, context)) + prnt('{') + # + if isinstance(tp.result, model.StructOrUnion): + result_code = '*r = ' + elif not isinstance(tp.result, model.VoidType): + result_code = 'return ' + else: + result_code = '' + prnt(' %s%s(%s);' % (result_code, name, ', '.join(argnames))) + prnt('}') + prnt() + + _loading_gen_function = _loaded_noop + + def _loaded_gen_function(self, tp, name, module, library): + assert isinstance(tp, model.FunctionPtrType) + if tp.ellipsis: + newfunction = self._load_constant(False, tp, name, module) + else: + indirections = [] + base_tp = tp + if (any(isinstance(typ, model.StructOrUnion) for typ in tp.args) + or isinstance(tp.result, model.StructOrUnion)): + indirect_args = [] + for i, typ in enumerate(tp.args): + if isinstance(typ, model.StructOrUnion): + typ = model.PointerType(typ) + indirections.append((i, typ)) + indirect_args.append(typ) + indirect_result = tp.result + if isinstance(indirect_result, model.StructOrUnion): + if indirect_result.fldtypes is None: + raise TypeError("'%s' is used as result type, " + "but is opaque" % ( + indirect_result._get_c_name(),)) + indirect_result = model.PointerType(indirect_result) + indirect_args.insert(0, indirect_result) + indirections.insert(0, ("result", indirect_result)) + indirect_result = model.void_type + tp = model.FunctionPtrType(tuple(indirect_args), + indirect_result, tp.ellipsis) + BFunc = self.ffi._get_cached_btype(tp) + wrappername = '_cffi_f_%s' % name + newfunction = module.load_function(BFunc, wrappername) + for i, typ in indirections: + newfunction = self._make_struct_wrapper(newfunction, i, typ, + base_tp) + setattr(library, name, newfunction) + type(library)._cffi_dir.append(name) + + def _make_struct_wrapper(self, oldfunc, i, tp, base_tp): + backend = self.ffi._backend + BType = self.ffi._get_cached_btype(tp) + if i == "result": + ffi = self.ffi + def newfunc(*args): + res = ffi.new(BType) + oldfunc(res, *args) + return res[0] + else: + def newfunc(*args): + args = args[:i] + (backend.newp(BType, args[i]),) + args[i+1:] + return oldfunc(*args) + newfunc._cffi_base_type = base_tp + return newfunc + + # ---------- + # named structs + + def _generate_gen_struct_decl(self, tp, name): + assert name == tp.name + self._generate_struct_or_union_decl(tp, 'struct', name) + + def _loading_gen_struct(self, tp, name, module): + self._loading_struct_or_union(tp, 'struct', name, module) + + def _loaded_gen_struct(self, tp, name, module, **kwds): + self._loaded_struct_or_union(tp) + + def _generate_gen_union_decl(self, tp, name): + assert name == tp.name + self._generate_struct_or_union_decl(tp, 'union', name) + + def _loading_gen_union(self, tp, name, module): + self._loading_struct_or_union(tp, 'union', name, module) + + def _loaded_gen_union(self, tp, name, module, **kwds): + self._loaded_struct_or_union(tp) + + def _generate_struct_or_union_decl(self, tp, prefix, name): + if tp.fldnames is None: + return # nothing to do with opaque structs + checkfuncname = '_cffi_check_%s_%s' % (prefix, name) + layoutfuncname = '_cffi_layout_%s_%s' % (prefix, name) + cname = ('%s %s' % (prefix, name)).strip() + # + prnt = self._prnt + prnt('static void %s(%s *p)' % (checkfuncname, cname)) + prnt('{') + prnt(' /* only to generate compile-time warnings or errors */') + prnt(' (void)p;') + for fname, ftype, fbitsize, fqual in tp.enumfields(): + if (isinstance(ftype, model.PrimitiveType) + and ftype.is_integer_type()) or fbitsize >= 0: + # accept all integers, but complain on float or double + prnt(' (void)((p->%s) << 1);' % fname) + else: + # only accept exactly the type declared. + try: + prnt(' { %s = &p->%s; (void)tmp; }' % ( + ftype.get_c_name('*tmp', 'field %r'%fname, quals=fqual), + fname)) + except VerificationError as e: + prnt(' /* %s */' % str(e)) # cannot verify it, ignore + prnt('}') + self.export_symbols.append(layoutfuncname) + prnt('intptr_t %s(intptr_t i)' % (layoutfuncname,)) + prnt('{') + prnt(' struct _cffi_aligncheck { char x; %s y; };' % cname) + prnt(' static intptr_t nums[] = {') + prnt(' sizeof(%s),' % cname) + prnt(' offsetof(struct _cffi_aligncheck, y),') + for fname, ftype, fbitsize, fqual in tp.enumfields(): + if fbitsize >= 0: + continue # xxx ignore fbitsize for now + prnt(' offsetof(%s, %s),' % (cname, fname)) + if isinstance(ftype, model.ArrayType) and ftype.length is None: + prnt(' 0, /* %s */' % ftype._get_c_name()) + else: + prnt(' sizeof(((%s *)0)->%s),' % (cname, fname)) + prnt(' -1') + prnt(' };') + prnt(' return nums[i];') + prnt(' /* the next line is not executed, but compiled */') + prnt(' %s(0);' % (checkfuncname,)) + prnt('}') + prnt() + + def _loading_struct_or_union(self, tp, prefix, name, module): + if tp.fldnames is None: + return # nothing to do with opaque structs + layoutfuncname = '_cffi_layout_%s_%s' % (prefix, name) + # + BFunc = self.ffi._typeof_locked("intptr_t(*)(intptr_t)")[0] + function = module.load_function(BFunc, layoutfuncname) + layout = [] + num = 0 + while True: + x = function(num) + if x < 0: break + layout.append(x) + num += 1 + if isinstance(tp, model.StructOrUnion) and tp.partial: + # use the function()'s sizes and offsets to guide the + # layout of the struct + totalsize = layout[0] + totalalignment = layout[1] + fieldofs = layout[2::2] + fieldsize = layout[3::2] + tp.force_flatten() + assert len(fieldofs) == len(fieldsize) == len(tp.fldnames) + tp.fixedlayout = fieldofs, fieldsize, totalsize, totalalignment + else: + cname = ('%s %s' % (prefix, name)).strip() + self._struct_pending_verification[tp] = layout, cname + + def _loaded_struct_or_union(self, tp): + if tp.fldnames is None: + return # nothing to do with opaque structs + self.ffi._get_cached_btype(tp) # force 'fixedlayout' to be considered + + if tp in self._struct_pending_verification: + # check that the layout sizes and offsets match the real ones + def check(realvalue, expectedvalue, msg): + if realvalue != expectedvalue: + raise VerificationError( + "%s (we have %d, but C compiler says %d)" + % (msg, expectedvalue, realvalue)) + ffi = self.ffi + BStruct = ffi._get_cached_btype(tp) + layout, cname = self._struct_pending_verification.pop(tp) + check(layout[0], ffi.sizeof(BStruct), "wrong total size") + check(layout[1], ffi.alignof(BStruct), "wrong total alignment") + i = 2 + for fname, ftype, fbitsize, fqual in tp.enumfields(): + if fbitsize >= 0: + continue # xxx ignore fbitsize for now + check(layout[i], ffi.offsetof(BStruct, fname), + "wrong offset for field %r" % (fname,)) + if layout[i+1] != 0: + BField = ffi._get_cached_btype(ftype) + check(layout[i+1], ffi.sizeof(BField), + "wrong size for field %r" % (fname,)) + i += 2 + assert i == len(layout) + + # ---------- + # 'anonymous' declarations. These are produced for anonymous structs + # or unions; the 'name' is obtained by a typedef. + + def _generate_gen_anonymous_decl(self, tp, name): + if isinstance(tp, model.EnumType): + self._generate_gen_enum_decl(tp, name, '') + else: + self._generate_struct_or_union_decl(tp, '', name) + + def _loading_gen_anonymous(self, tp, name, module): + if isinstance(tp, model.EnumType): + self._loading_gen_enum(tp, name, module, '') + else: + self._loading_struct_or_union(tp, '', name, module) + + def _loaded_gen_anonymous(self, tp, name, module, **kwds): + if isinstance(tp, model.EnumType): + self._loaded_gen_enum(tp, name, module, **kwds) + else: + self._loaded_struct_or_union(tp) + + # ---------- + # constants, likely declared with '#define' + + def _generate_gen_const(self, is_int, name, tp=None, category='const', + check_value=None): + prnt = self._prnt + funcname = '_cffi_%s_%s' % (category, name) + self.export_symbols.append(funcname) + if check_value is not None: + assert is_int + assert category == 'const' + prnt('int %s(char *out_error)' % funcname) + prnt('{') + self._check_int_constant_value(name, check_value) + prnt(' return 0;') + prnt('}') + elif is_int: + assert category == 'const' + prnt('int %s(long long *out_value)' % funcname) + prnt('{') + prnt(' *out_value = (long long)(%s);' % (name,)) + prnt(' return (%s) <= 0;' % (name,)) + prnt('}') + else: + assert tp is not None + assert check_value is None + if category == 'var': + ampersand = '&' + else: + ampersand = '' + extra = '' + if category == 'const' and isinstance(tp, model.StructOrUnion): + extra = 'const *' + ampersand = '&' + prnt(tp.get_c_name(' %s%s(void)' % (extra, funcname), name)) + prnt('{') + prnt(' return (%s%s);' % (ampersand, name)) + prnt('}') + prnt() + + def _generate_gen_constant_decl(self, tp, name): + is_int = isinstance(tp, model.PrimitiveType) and tp.is_integer_type() + self._generate_gen_const(is_int, name, tp) + + _loading_gen_constant = _loaded_noop + + def _load_constant(self, is_int, tp, name, module, check_value=None): + funcname = '_cffi_const_%s' % name + if check_value is not None: + assert is_int + self._load_known_int_constant(module, funcname) + value = check_value + elif is_int: + BType = self.ffi._typeof_locked("long long*")[0] + BFunc = self.ffi._typeof_locked("int(*)(long long*)")[0] + function = module.load_function(BFunc, funcname) + p = self.ffi.new(BType) + negative = function(p) + value = int(p[0]) + if value < 0 and not negative: + BLongLong = self.ffi._typeof_locked("long long")[0] + value += (1 << (8*self.ffi.sizeof(BLongLong))) + else: + assert check_value is None + fntypeextra = '(*)(void)' + if isinstance(tp, model.StructOrUnion): + fntypeextra = '*' + fntypeextra + BFunc = self.ffi._typeof_locked(tp.get_c_name(fntypeextra, name))[0] + function = module.load_function(BFunc, funcname) + value = function() + if isinstance(tp, model.StructOrUnion): + value = value[0] + return value + + def _loaded_gen_constant(self, tp, name, module, library): + is_int = isinstance(tp, model.PrimitiveType) and tp.is_integer_type() + value = self._load_constant(is_int, tp, name, module) + setattr(library, name, value) + type(library)._cffi_dir.append(name) + + # ---------- + # enums + + def _check_int_constant_value(self, name, value): + prnt = self._prnt + if value <= 0: + prnt(' if ((%s) > 0 || (long)(%s) != %dL) {' % ( + name, name, value)) + else: + prnt(' if ((%s) <= 0 || (unsigned long)(%s) != %dUL) {' % ( + name, name, value)) + prnt(' char buf[64];') + prnt(' if ((%s) <= 0)' % name) + prnt(' sprintf(buf, "%%ld", (long)(%s));' % name) + prnt(' else') + prnt(' sprintf(buf, "%%lu", (unsigned long)(%s));' % + name) + prnt(' sprintf(out_error, "%s has the real value %s, not %s",') + prnt(' "%s", buf, "%d");' % (name[:100], value)) + prnt(' return -1;') + prnt(' }') + + def _load_known_int_constant(self, module, funcname): + BType = self.ffi._typeof_locked("char[]")[0] + BFunc = self.ffi._typeof_locked("int(*)(char*)")[0] + function = module.load_function(BFunc, funcname) + p = self.ffi.new(BType, 256) + if function(p) < 0: + error = self.ffi.string(p) + if sys.version_info >= (3,): + error = str(error, 'utf-8') + raise VerificationError(error) + + def _enum_funcname(self, prefix, name): + # "$enum_$1" => "___D_enum____D_1" + name = name.replace('$', '___D_') + return '_cffi_e_%s_%s' % (prefix, name) + + def _generate_gen_enum_decl(self, tp, name, prefix='enum'): + if tp.partial: + for enumerator in tp.enumerators: + self._generate_gen_const(True, enumerator) + return + # + funcname = self._enum_funcname(prefix, name) + self.export_symbols.append(funcname) + prnt = self._prnt + prnt('int %s(char *out_error)' % funcname) + prnt('{') + for enumerator, enumvalue in zip(tp.enumerators, tp.enumvalues): + self._check_int_constant_value(enumerator, enumvalue) + prnt(' return 0;') + prnt('}') + prnt() + + def _loading_gen_enum(self, tp, name, module, prefix='enum'): + if tp.partial: + enumvalues = [self._load_constant(True, tp, enumerator, module) + for enumerator in tp.enumerators] + tp.enumvalues = tuple(enumvalues) + tp.partial_resolved = True + else: + funcname = self._enum_funcname(prefix, name) + self._load_known_int_constant(module, funcname) + + def _loaded_gen_enum(self, tp, name, module, library): + for enumerator, enumvalue in zip(tp.enumerators, tp.enumvalues): + setattr(library, enumerator, enumvalue) + type(library)._cffi_dir.append(enumerator) + + # ---------- + # macros: for now only for integers + + def _generate_gen_macro_decl(self, tp, name): + if tp == '...': + check_value = None + else: + check_value = tp # an integer + self._generate_gen_const(True, name, check_value=check_value) + + _loading_gen_macro = _loaded_noop + + def _loaded_gen_macro(self, tp, name, module, library): + if tp == '...': + check_value = None + else: + check_value = tp # an integer + value = self._load_constant(True, tp, name, module, + check_value=check_value) + setattr(library, name, value) + type(library)._cffi_dir.append(name) + + # ---------- + # global variables + + def _generate_gen_variable_decl(self, tp, name): + if isinstance(tp, model.ArrayType): + if tp.length_is_unknown(): + prnt = self._prnt + funcname = '_cffi_sizeof_%s' % (name,) + self.export_symbols.append(funcname) + prnt("size_t %s(void)" % funcname) + prnt("{") + prnt(" return sizeof(%s);" % (name,)) + prnt("}") + tp_ptr = model.PointerType(tp.item) + self._generate_gen_const(False, name, tp_ptr) + else: + tp_ptr = model.PointerType(tp) + self._generate_gen_const(False, name, tp_ptr, category='var') + + _loading_gen_variable = _loaded_noop + + def _loaded_gen_variable(self, tp, name, module, library): + if isinstance(tp, model.ArrayType): # int a[5] is "constant" in the + # sense that "a=..." is forbidden + if tp.length_is_unknown(): + funcname = '_cffi_sizeof_%s' % (name,) + BFunc = self.ffi._typeof_locked('size_t(*)(void)')[0] + function = module.load_function(BFunc, funcname) + size = function() + BItemType = self.ffi._get_cached_btype(tp.item) + length, rest = divmod(size, self.ffi.sizeof(BItemType)) + if rest != 0: + raise VerificationError( + "bad size: %r does not seem to be an array of %s" % + (name, tp.item)) + tp = tp.resolve_length(length) + tp_ptr = model.PointerType(tp.item) + value = self._load_constant(False, tp_ptr, name, module) + # 'value' is a which we have to replace with + # a if the N is actually known + if tp.length is not None: + BArray = self.ffi._get_cached_btype(tp) + value = self.ffi.cast(BArray, value) + setattr(library, name, value) + type(library)._cffi_dir.append(name) + return + # remove ptr= from the library instance, and replace + # it by a property on the class, which reads/writes into ptr[0]. + funcname = '_cffi_var_%s' % name + BFunc = self.ffi._typeof_locked(tp.get_c_name('*(*)(void)', name))[0] + function = module.load_function(BFunc, funcname) + ptr = function() + def getter(library): + return ptr[0] + def setter(library, value): + ptr[0] = value + setattr(type(library), name, property(getter, setter)) + type(library)._cffi_dir.append(name) + +cffimod_header = r''' +#include +#include +#include +#include +#include /* XXX for ssize_t on some platforms */ + +/* this block of #ifs should be kept exactly identical between + c/_cffi_backend.c, cffi/vengine_cpy.py, cffi/vengine_gen.py + and cffi/_cffi_include.h */ +#if defined(_MSC_VER) +# include /* for alloca() */ +# if _MSC_VER < 1600 /* MSVC < 2010 */ + typedef __int8 int8_t; + typedef __int16 int16_t; + typedef __int32 int32_t; + typedef __int64 int64_t; + typedef unsigned __int8 uint8_t; + typedef unsigned __int16 uint16_t; + typedef unsigned __int32 uint32_t; + typedef unsigned __int64 uint64_t; + typedef __int8 int_least8_t; + typedef __int16 int_least16_t; + typedef __int32 int_least32_t; + typedef __int64 int_least64_t; + typedef unsigned __int8 uint_least8_t; + typedef unsigned __int16 uint_least16_t; + typedef unsigned __int32 uint_least32_t; + typedef unsigned __int64 uint_least64_t; + typedef __int8 int_fast8_t; + typedef __int16 int_fast16_t; + typedef __int32 int_fast32_t; + typedef __int64 int_fast64_t; + typedef unsigned __int8 uint_fast8_t; + typedef unsigned __int16 uint_fast16_t; + typedef unsigned __int32 uint_fast32_t; + typedef unsigned __int64 uint_fast64_t; + typedef __int64 intmax_t; + typedef unsigned __int64 uintmax_t; +# else +# include +# endif +# if _MSC_VER < 1800 /* MSVC < 2013 */ +# ifndef __cplusplus + typedef unsigned char _Bool; +# endif +# endif +# define _cffi_float_complex_t _Fcomplex /* include for it */ +# define _cffi_double_complex_t _Dcomplex /* include for it */ +#else +# include +# if (defined (__SVR4) && defined (__sun)) || defined(_AIX) || defined(__hpux) +# include +# endif +# define _cffi_float_complex_t float _Complex +# define _cffi_double_complex_t double _Complex +#endif +''' diff --git a/py_modules/cffi/verifier.py b/py_modules/cffi/verifier.py new file mode 100644 index 0000000..96baadb --- /dev/null +++ b/py_modules/cffi/verifier.py @@ -0,0 +1,306 @@ +# +# DEPRECATED: implementation for ffi.verify() +# +import sys, os, binascii, shutil, io +from . import __version_verifier_modules__ +from . import ffiplatform +from .error import VerificationError + +if sys.version_info >= (3, 3): + import importlib.machinery + def _extension_suffixes(): + return importlib.machinery.EXTENSION_SUFFIXES[:] +else: + import imp + def _extension_suffixes(): + return [suffix for suffix, _, type in imp.get_suffixes() + if type == imp.C_EXTENSION] + + +if sys.version_info >= (3,): + NativeIO = io.StringIO +else: + class NativeIO(io.BytesIO): + def write(self, s): + if isinstance(s, unicode): + s = s.encode('ascii') + super().write(s) + + +class Verifier: + + def __init__(self, ffi, preamble, tmpdir=None, modulename=None, + ext_package=None, tag='', force_generic_engine=False, + source_extension='.c', flags=None, relative_to=None, **kwds): + if ffi._parser._uses_new_feature: + raise VerificationError( + "feature not supported with ffi.verify(), but only " + "with ffi.set_source(): %s" % (ffi._parser._uses_new_feature,)) + self.ffi = ffi + self.preamble = preamble + if not modulename: + flattened_kwds = ffiplatform.flatten(kwds) + vengine_class = _locate_engine_class(ffi, force_generic_engine) + self._vengine = vengine_class(self) + self._vengine.patch_extension_kwds(kwds) + self.flags = flags + self.kwds = self.make_relative_to(kwds, relative_to) + # + if modulename: + if tag: + raise TypeError("can't specify both 'modulename' and 'tag'") + else: + key = '\x00'.join(['%d.%d' % sys.version_info[:2], + __version_verifier_modules__, + preamble, flattened_kwds] + + ffi._cdefsources) + if sys.version_info >= (3,): + key = key.encode('utf-8') + k1 = hex(binascii.crc32(key[0::2]) & 0xffffffff) + k1 = k1.lstrip('0x').rstrip('L') + k2 = hex(binascii.crc32(key[1::2]) & 0xffffffff) + k2 = k2.lstrip('0').rstrip('L') + modulename = '_cffi_%s_%s%s%s' % (tag, self._vengine._class_key, + k1, k2) + suffix = _get_so_suffixes()[0] + self.tmpdir = tmpdir or _caller_dir_pycache() + self.sourcefilename = os.path.join(self.tmpdir, modulename + source_extension) + self.modulefilename = os.path.join(self.tmpdir, modulename + suffix) + self.ext_package = ext_package + self._has_source = False + self._has_module = False + + def write_source(self, file=None): + """Write the C source code. It is produced in 'self.sourcefilename', + which can be tweaked beforehand.""" + with self.ffi._lock: + if self._has_source and file is None: + raise VerificationError( + "source code already written") + self._write_source(file) + + def compile_module(self): + """Write the C source code (if not done already) and compile it. + This produces a dynamic link library in 'self.modulefilename'.""" + with self.ffi._lock: + if self._has_module: + raise VerificationError("module already compiled") + if not self._has_source: + self._write_source() + self._compile_module() + + def load_library(self): + """Get a C module from this Verifier instance. + Returns an instance of a FFILibrary class that behaves like the + objects returned by ffi.dlopen(), but that delegates all + operations to the C module. If necessary, the C code is written + and compiled first. + """ + with self.ffi._lock: + if not self._has_module: + self._locate_module() + if not self._has_module: + if not self._has_source: + self._write_source() + self._compile_module() + return self._load_library() + + def get_module_name(self): + basename = os.path.basename(self.modulefilename) + # kill both the .so extension and the other .'s, as introduced + # by Python 3: 'basename.cpython-33m.so' + basename = basename.split('.', 1)[0] + # and the _d added in Python 2 debug builds --- but try to be + # conservative and not kill a legitimate _d + if basename.endswith('_d') and hasattr(sys, 'gettotalrefcount'): + basename = basename[:-2] + return basename + + def get_extension(self): + if not self._has_source: + with self.ffi._lock: + if not self._has_source: + self._write_source() + sourcename = ffiplatform.maybe_relative_path(self.sourcefilename) + modname = self.get_module_name() + return ffiplatform.get_extension(sourcename, modname, **self.kwds) + + def generates_python_module(self): + return self._vengine._gen_python_module + + def make_relative_to(self, kwds, relative_to): + if relative_to and os.path.dirname(relative_to): + dirname = os.path.dirname(relative_to) + kwds = kwds.copy() + for key in ffiplatform.LIST_OF_FILE_NAMES: + if key in kwds: + lst = kwds[key] + if not isinstance(lst, (list, tuple)): + raise TypeError("keyword '%s' should be a list or tuple" + % (key,)) + lst = [os.path.join(dirname, fn) for fn in lst] + kwds[key] = lst + return kwds + + # ---------- + + def _locate_module(self): + if not os.path.isfile(self.modulefilename): + if self.ext_package: + try: + pkg = __import__(self.ext_package, None, None, ['__doc__']) + except ImportError: + return # cannot import the package itself, give up + # (e.g. it might be called differently before installation) + path = pkg.__path__ + else: + path = None + filename = self._vengine.find_module(self.get_module_name(), path, + _get_so_suffixes()) + if filename is None: + return + self.modulefilename = filename + self._vengine.collect_types() + self._has_module = True + + def _write_source_to(self, file): + self._vengine._f = file + try: + self._vengine.write_source_to_f() + finally: + del self._vengine._f + + def _write_source(self, file=None): + if file is not None: + self._write_source_to(file) + else: + # Write our source file to an in memory file. + f = NativeIO() + self._write_source_to(f) + source_data = f.getvalue() + + # Determine if this matches the current file + if os.path.exists(self.sourcefilename): + with open(self.sourcefilename, "r") as fp: + needs_written = fp.read() != source_data + else: + needs_written = True + + # Actually write the file out if it doesn't match + if needs_written: + _ensure_dir(self.sourcefilename) + with open(self.sourcefilename, "w") as fp: + fp.write(source_data) + + # Set this flag + self._has_source = True + + def _compile_module(self): + # compile this C source + tmpdir = os.path.dirname(self.sourcefilename) + outputfilename = ffiplatform.compile(tmpdir, self.get_extension()) + try: + same = ffiplatform.samefile(outputfilename, self.modulefilename) + except OSError: + same = False + if not same: + _ensure_dir(self.modulefilename) + shutil.move(outputfilename, self.modulefilename) + self._has_module = True + + def _load_library(self): + assert self._has_module + if self.flags is not None: + return self._vengine.load_library(self.flags) + else: + return self._vengine.load_library() + +# ____________________________________________________________ + +_FORCE_GENERIC_ENGINE = False # for tests + +def _locate_engine_class(ffi, force_generic_engine): + if _FORCE_GENERIC_ENGINE: + force_generic_engine = True + if not force_generic_engine: + if '__pypy__' in sys.builtin_module_names: + force_generic_engine = True + else: + try: + import _cffi_backend + except ImportError: + _cffi_backend = '?' + if ffi._backend is not _cffi_backend: + force_generic_engine = True + if force_generic_engine: + from . import vengine_gen + return vengine_gen.VGenericEngine + else: + from . import vengine_cpy + return vengine_cpy.VCPythonEngine + +# ____________________________________________________________ + +_TMPDIR = None + +def _caller_dir_pycache(): + if _TMPDIR: + return _TMPDIR + result = os.environ.get('CFFI_TMPDIR') + if result: + return result + filename = sys._getframe(2).f_code.co_filename + return os.path.abspath(os.path.join(os.path.dirname(filename), + '__pycache__')) + +def set_tmpdir(dirname): + """Set the temporary directory to use instead of __pycache__.""" + global _TMPDIR + _TMPDIR = dirname + +def cleanup_tmpdir(tmpdir=None, keep_so=False): + """Clean up the temporary directory by removing all files in it + called `_cffi_*.{c,so}` as well as the `build` subdirectory.""" + tmpdir = tmpdir or _caller_dir_pycache() + try: + filelist = os.listdir(tmpdir) + except OSError: + return + if keep_so: + suffix = '.c' # only remove .c files + else: + suffix = _get_so_suffixes()[0].lower() + for fn in filelist: + if fn.lower().startswith('_cffi_') and ( + fn.lower().endswith(suffix) or fn.lower().endswith('.c')): + try: + os.unlink(os.path.join(tmpdir, fn)) + except OSError: + pass + clean_dir = [os.path.join(tmpdir, 'build')] + for dir in clean_dir: + try: + for fn in os.listdir(dir): + fn = os.path.join(dir, fn) + if os.path.isdir(fn): + clean_dir.append(fn) + else: + os.unlink(fn) + except OSError: + pass + +def _get_so_suffixes(): + suffixes = _extension_suffixes() + if not suffixes: + # bah, no C_EXTENSION available. Occurs on pypy without cpyext + if sys.platform == 'win32': + suffixes = [".pyd"] + else: + suffixes = [".so"] + + return suffixes + +def _ensure_dir(filename): + dirname = os.path.dirname(filename) + if dirname and not os.path.isdir(dirname): + os.makedirs(dirname) diff --git a/py_modules/cryptography/__about__.py b/py_modules/cryptography/__about__.py new file mode 100644 index 0000000..d6fc1ae --- /dev/null +++ b/py_modules/cryptography/__about__.py @@ -0,0 +1,17 @@ +# This file is dual licensed under the terms of the Apache License, Version +# 2.0, and the BSD License. See the LICENSE file in the root of this repository +# for complete details. + +from __future__ import annotations + +__all__ = [ + "__author__", + "__copyright__", + "__version__", +] + +__version__ = "50.0.1" + + +__author__ = "The Python Cryptographic Authority and individual contributors" +__copyright__ = f"Copyright 2013-2026 {__author__}" diff --git a/py_modules/cryptography/__init__.py b/py_modules/cryptography/__init__.py new file mode 100644 index 0000000..d374f75 --- /dev/null +++ b/py_modules/cryptography/__init__.py @@ -0,0 +1,13 @@ +# This file is dual licensed under the terms of the Apache License, Version +# 2.0, and the BSD License. See the LICENSE file in the root of this repository +# for complete details. + +from __future__ import annotations + +from cryptography.__about__ import __author__, __copyright__, __version__ + +__all__ = [ + "__author__", + "__copyright__", + "__version__", +] diff --git a/py_modules/cryptography/cobblestone.py b/py_modules/cryptography/cobblestone.py new file mode 100644 index 0000000..09f67af --- /dev/null +++ b/py_modules/cryptography/cobblestone.py @@ -0,0 +1,21 @@ +# This file is dual licensed under the terms of the Apache License, Version +# 2.0, and the BSD License. See the LICENSE file in the root of this repository +# for complete details. + +from __future__ import annotations + +from cryptography.hazmat.bindings._rust import ( + cobblestone as _cobblestone, +) + +Cobblestone128Decryptor = _cobblestone.Cobblestone128Decryptor +Cobblestone128Encryptor = _cobblestone.Cobblestone128Encryptor +Cobblestone256Decryptor = _cobblestone.Cobblestone256Decryptor +Cobblestone256Encryptor = _cobblestone.Cobblestone256Encryptor + +__all__ = [ + "Cobblestone128Decryptor", + "Cobblestone128Encryptor", + "Cobblestone256Decryptor", + "Cobblestone256Encryptor", +] diff --git a/py_modules/cryptography/exceptions.py b/py_modules/cryptography/exceptions.py new file mode 100644 index 0000000..fe125ea --- /dev/null +++ b/py_modules/cryptography/exceptions.py @@ -0,0 +1,52 @@ +# This file is dual licensed under the terms of the Apache License, Version +# 2.0, and the BSD License. See the LICENSE file in the root of this repository +# for complete details. + +from __future__ import annotations + +import typing + +from cryptography.hazmat.bindings._rust import exceptions as rust_exceptions + +if typing.TYPE_CHECKING: + from cryptography.hazmat.bindings._rust import openssl as rust_openssl + +_Reasons = rust_exceptions._Reasons + + +class UnsupportedAlgorithm(Exception): + def __init__(self, message: str, reason: _Reasons | None = None) -> None: + super().__init__(message) + self._reason = reason + + +class AlreadyFinalized(Exception): + pass + + +class AlreadyUpdated(Exception): + pass + + +class NotYetFinalized(Exception): + pass + + +class InvalidTag(Exception): + pass + + +class InvalidSignature(Exception): + pass + + +class InternalError(Exception): + def __init__( + self, msg: str, err_code: list[rust_openssl.OpenSSLError] + ) -> None: + super().__init__(msg) + self.err_code = err_code + + +class InvalidKey(Exception): + pass diff --git a/py_modules/cryptography/fernet.py b/py_modules/cryptography/fernet.py new file mode 100644 index 0000000..582e594 --- /dev/null +++ b/py_modules/cryptography/fernet.py @@ -0,0 +1,240 @@ +# This file is dual licensed under the terms of the Apache License, Version +# 2.0, and the BSD License. See the LICENSE file in the root of this repository +# for complete details. + +from __future__ import annotations + +import base64 +import binascii +import os +import time +import typing +from collections.abc import Iterable + +from cryptography import utils +from cryptography.exceptions import InvalidSignature +from cryptography.hazmat.primitives import hashes, padding +from cryptography.hazmat.primitives.ciphers import Cipher, algorithms, modes +from cryptography.hazmat.primitives.hmac import HMAC + + +class InvalidToken(Exception): + pass + + +_MAX_CLOCK_SKEW = 60 + +# Hoisted to module level so each operation doesn't reconstruct them. +_PKCS7_128 = padding.PKCS7(128) +_SHA256 = hashes.SHA256() + + +class Fernet: + def __init__( + self, + key: bytes | str, + backend: typing.Any = None, + ) -> None: + try: + key = base64.urlsafe_b64decode(key) + except binascii.Error as exc: + raise ValueError( + "Fernet key must be 32 url-safe base64-encoded bytes." + ) from exc + if len(key) != 32: + raise ValueError( + "Fernet key must be 32 url-safe base64-encoded bytes." + ) + + self._signing_key = key[:16] + self._encryption_key = key[16:] + self._aes = algorithms.AES(self._encryption_key) + + @classmethod + def generate_key(cls) -> bytes: + return base64.urlsafe_b64encode(os.urandom(32)) + + def encrypt(self, data: bytes) -> bytes: + return self.encrypt_at_time(data, int(time.time())) + + def encrypt_at_time(self, data: bytes, current_time: int) -> bytes: + iv = os.urandom(16) + return self._encrypt_from_parts(data, current_time, iv) + + def _encrypt_from_parts( + self, data: bytes, current_time: int, iv: bytes + ) -> bytes: + utils._check_bytes("data", data) + + padder = _PKCS7_128.padder() + padded_data = padder.update(data) + padder.finalize() + encryptor = Cipher(self._aes, modes.CBC(iv)).encryptor() + ciphertext = encryptor.update(padded_data) + encryptor.finalize() + + basic_parts = ( + b"\x80" + + current_time.to_bytes(length=8, byteorder="big") + + iv + + ciphertext + ) + + h = HMAC(self._signing_key, _SHA256) + h.update(basic_parts) + hmac = h.finalize() + return base64.urlsafe_b64encode(basic_parts + hmac) + + def decrypt(self, token: bytes | str, ttl: int | None = None) -> bytes: + timestamp, data = Fernet._get_unverified_token_data(token) + if ttl is None: + time_info = None + else: + time_info = (ttl, int(time.time())) + return self._decrypt_data(data, timestamp, time_info) + + def decrypt_at_time( + self, token: bytes | str, ttl: int, current_time: int + ) -> bytes: + if ttl is None: + raise ValueError( + "decrypt_at_time() can only be used with a non-None ttl" + ) + timestamp, data = Fernet._get_unverified_token_data(token) + return self._decrypt_data(data, timestamp, (ttl, current_time)) + + def extract_timestamp(self, token: bytes | str) -> int: + timestamp, data = Fernet._get_unverified_token_data(token) + # Verify the token was not tampered with. + self._verify_signature(data) + return timestamp + + @staticmethod + def _get_unverified_token_data(token: bytes | str) -> tuple[int, bytes]: + if not isinstance(token, (str, bytes)): + raise TypeError("token must be bytes or str") + + try: + data = base64.urlsafe_b64decode(token) + except (TypeError, binascii.Error): + raise InvalidToken + + if not data or data[0] != 0x80: + raise InvalidToken + + if len(data) < 9: + raise InvalidToken + + timestamp = int.from_bytes(data[1:9], byteorder="big") + return timestamp, data + + def _verify_signature(self, data: bytes) -> None: + h = HMAC(self._signing_key, _SHA256) + h.update(memoryview(data)[:-32]) + try: + h.verify(data[-32:]) + except InvalidSignature: + raise InvalidToken + + def _decrypt_data( + self, + data: bytes, + timestamp: int, + time_info: tuple[int, int] | None, + ) -> bytes: + if time_info is not None: + ttl, current_time = time_info + if timestamp + ttl < current_time: + raise InvalidToken + + if current_time + _MAX_CLOCK_SKEW < timestamp: + raise InvalidToken + + self._verify_signature(data) + + mv = memoryview(data) + iv = data[9:25] + ciphertext = mv[25:-32] + decryptor = Cipher(self._aes, modes.CBC(iv)).decryptor() + plaintext_padded = decryptor.update(ciphertext) + try: + plaintext_padded += decryptor.finalize() + except ValueError: + raise InvalidToken + unpadder = _PKCS7_128.unpadder() + + unpadded = unpadder.update(plaintext_padded) + try: + unpadded += unpadder.finalize() + except ValueError: + raise InvalidToken + return unpadded + + +class MultiFernet: + def __init__(self, fernets: Iterable[Fernet]): + fernets = list(fernets) + if not fernets: + raise ValueError( + "MultiFernet requires at least one Fernet instance" + ) + self._fernets = fernets + + def encrypt(self, msg: bytes) -> bytes: + return self.encrypt_at_time(msg, int(time.time())) + + def encrypt_at_time(self, msg: bytes, current_time: int) -> bytes: + return self._fernets[0].encrypt_at_time(msg, current_time) + + def rotate(self, msg: bytes | str) -> bytes: + timestamp, data = Fernet._get_unverified_token_data(msg) + for f in self._fernets: + try: + p = f._decrypt_data(data, timestamp, None) + break + except InvalidToken: + pass + else: + raise InvalidToken + + iv = os.urandom(16) + return self._fernets[0]._encrypt_from_parts(p, timestamp, iv) + + def decrypt(self, msg: bytes | str, ttl: int | None = None) -> bytes: + if ttl is None: + time_info = None + else: + time_info = (ttl, int(time.time())) + # Parse the token once rather than once per key. + timestamp, data = Fernet._get_unverified_token_data(msg) + for f in self._fernets: + try: + return f._decrypt_data(data, timestamp, time_info) + except InvalidToken: + pass + raise InvalidToken + + def decrypt_at_time( + self, msg: bytes | str, ttl: int, current_time: int + ) -> bytes: + if ttl is None: + raise ValueError( + "decrypt_at_time() can only be used with a non-None ttl" + ) + # Parse the token once rather than once per key. + timestamp, data = Fernet._get_unverified_token_data(msg) + for f in self._fernets: + try: + return f._decrypt_data(data, timestamp, (ttl, current_time)) + except InvalidToken: + pass + raise InvalidToken + + def extract_timestamp(self, msg: bytes | str) -> int: + # Parse the token once rather than once per key. + timestamp, data = Fernet._get_unverified_token_data(msg) + for f in self._fernets: + try: + f._verify_signature(data) + return timestamp + except InvalidToken: + pass + raise InvalidToken diff --git a/py_modules/cryptography/hazmat/__init__.py b/py_modules/cryptography/hazmat/__init__.py new file mode 100644 index 0000000..b9f1187 --- /dev/null +++ b/py_modules/cryptography/hazmat/__init__.py @@ -0,0 +1,13 @@ +# This file is dual licensed under the terms of the Apache License, Version +# 2.0, and the BSD License. See the LICENSE file in the root of this repository +# for complete details. + +from __future__ import annotations + +""" +Hazardous Materials + +This is a "Hazardous Materials" module. You should ONLY use it if you're +100% absolutely sure that you know what you're doing because this module +is full of land mines, dragons, and dinosaurs with laser guns. +""" diff --git a/py_modules/cryptography/hazmat/_oid.py b/py_modules/cryptography/hazmat/_oid.py new file mode 100644 index 0000000..244a1b3 --- /dev/null +++ b/py_modules/cryptography/hazmat/_oid.py @@ -0,0 +1,373 @@ +# This file is dual licensed under the terms of the Apache License, Version +# 2.0, and the BSD License. See the LICENSE file in the root of this repository +# for complete details. + +from __future__ import annotations + +from cryptography.hazmat.bindings._rust import ( + ObjectIdentifier as ObjectIdentifier, +) +from cryptography.hazmat.primitives import hashes + + +class ExtensionOID: + SUBJECT_DIRECTORY_ATTRIBUTES = ObjectIdentifier("2.5.29.9") + SUBJECT_KEY_IDENTIFIER = ObjectIdentifier("2.5.29.14") + KEY_USAGE = ObjectIdentifier("2.5.29.15") + PRIVATE_KEY_USAGE_PERIOD = ObjectIdentifier("2.5.29.16") + SUBJECT_ALTERNATIVE_NAME = ObjectIdentifier("2.5.29.17") + ISSUER_ALTERNATIVE_NAME = ObjectIdentifier("2.5.29.18") + BASIC_CONSTRAINTS = ObjectIdentifier("2.5.29.19") + NAME_CONSTRAINTS = ObjectIdentifier("2.5.29.30") + CRL_DISTRIBUTION_POINTS = ObjectIdentifier("2.5.29.31") + CERTIFICATE_POLICIES = ObjectIdentifier("2.5.29.32") + POLICY_MAPPINGS = ObjectIdentifier("2.5.29.33") + AUTHORITY_KEY_IDENTIFIER = ObjectIdentifier("2.5.29.35") + POLICY_CONSTRAINTS = ObjectIdentifier("2.5.29.36") + EXTENDED_KEY_USAGE = ObjectIdentifier("2.5.29.37") + FRESHEST_CRL = ObjectIdentifier("2.5.29.46") + INHIBIT_ANY_POLICY = ObjectIdentifier("2.5.29.54") + ISSUING_DISTRIBUTION_POINT = ObjectIdentifier("2.5.29.28") + AUTHORITY_INFORMATION_ACCESS = ObjectIdentifier("1.3.6.1.5.5.7.1.1") + SUBJECT_INFORMATION_ACCESS = ObjectIdentifier("1.3.6.1.5.5.7.1.11") + OCSP_NO_CHECK = ObjectIdentifier("1.3.6.1.5.5.7.48.1.5") + TLS_FEATURE = ObjectIdentifier("1.3.6.1.5.5.7.1.24") + CRL_NUMBER = ObjectIdentifier("2.5.29.20") + DELTA_CRL_INDICATOR = ObjectIdentifier("2.5.29.27") + PRECERT_SIGNED_CERTIFICATE_TIMESTAMPS = ObjectIdentifier( + "1.3.6.1.4.1.11129.2.4.2" + ) + PRECERT_POISON = ObjectIdentifier("1.3.6.1.4.1.11129.2.4.3") + SIGNED_CERTIFICATE_TIMESTAMPS = ObjectIdentifier("1.3.6.1.4.1.11129.2.4.5") + MS_CERTIFICATE_TEMPLATE = ObjectIdentifier("1.3.6.1.4.1.311.21.7") + ADMISSIONS = ObjectIdentifier("1.3.36.8.3.3") + + +class OCSPExtensionOID: + NONCE = ObjectIdentifier("1.3.6.1.5.5.7.48.1.2") + ACCEPTABLE_RESPONSES = ObjectIdentifier("1.3.6.1.5.5.7.48.1.4") + + +class CRLEntryExtensionOID: + CERTIFICATE_ISSUER = ObjectIdentifier("2.5.29.29") + CRL_REASON = ObjectIdentifier("2.5.29.21") + INVALIDITY_DATE = ObjectIdentifier("2.5.29.24") + + +class NameOID: + COMMON_NAME = ObjectIdentifier("2.5.4.3") + COUNTRY_NAME = ObjectIdentifier("2.5.4.6") + LOCALITY_NAME = ObjectIdentifier("2.5.4.7") + STATE_OR_PROVINCE_NAME = ObjectIdentifier("2.5.4.8") + STREET_ADDRESS = ObjectIdentifier("2.5.4.9") + ORGANIZATION_IDENTIFIER = ObjectIdentifier("2.5.4.97") + ORGANIZATION_NAME = ObjectIdentifier("2.5.4.10") + ORGANIZATIONAL_UNIT_NAME = ObjectIdentifier("2.5.4.11") + SERIAL_NUMBER = ObjectIdentifier("2.5.4.5") + SURNAME = ObjectIdentifier("2.5.4.4") + GIVEN_NAME = ObjectIdentifier("2.5.4.42") + TITLE = ObjectIdentifier("2.5.4.12") + INITIALS = ObjectIdentifier("2.5.4.43") + GENERATION_QUALIFIER = ObjectIdentifier("2.5.4.44") + X500_UNIQUE_IDENTIFIER = ObjectIdentifier("2.5.4.45") + DN_QUALIFIER = ObjectIdentifier("2.5.4.46") + PSEUDONYM = ObjectIdentifier("2.5.4.65") + USER_ID = ObjectIdentifier("0.9.2342.19200300.100.1.1") + DOMAIN_COMPONENT = ObjectIdentifier("0.9.2342.19200300.100.1.25") + EMAIL_ADDRESS = ObjectIdentifier("1.2.840.113549.1.9.1") + JURISDICTION_COUNTRY_NAME = ObjectIdentifier("1.3.6.1.4.1.311.60.2.1.3") + JURISDICTION_LOCALITY_NAME = ObjectIdentifier("1.3.6.1.4.1.311.60.2.1.1") + JURISDICTION_STATE_OR_PROVINCE_NAME = ObjectIdentifier( + "1.3.6.1.4.1.311.60.2.1.2" + ) + BUSINESS_CATEGORY = ObjectIdentifier("2.5.4.15") + POSTAL_ADDRESS = ObjectIdentifier("2.5.4.16") + POSTAL_CODE = ObjectIdentifier("2.5.4.17") + INN = ObjectIdentifier("1.2.643.3.131.1.1") + OGRN = ObjectIdentifier("1.2.643.100.1") + SNILS = ObjectIdentifier("1.2.643.100.3") + UNSTRUCTURED_NAME = ObjectIdentifier("1.2.840.113549.1.9.2") + UNSIGNED = ObjectIdentifier("1.3.6.1.5.5.7.25.1") + + +class SignatureAlgorithmOID: + RSA_WITH_MD5 = ObjectIdentifier("1.2.840.113549.1.1.4") + RSA_WITH_SHA1 = ObjectIdentifier("1.2.840.113549.1.1.5") + # This is an alternate OID for RSA with SHA1 that is occasionally seen + _RSA_WITH_SHA1 = ObjectIdentifier("1.3.14.3.2.29") + RSA_WITH_SHA224 = ObjectIdentifier("1.2.840.113549.1.1.14") + RSA_WITH_SHA256 = ObjectIdentifier("1.2.840.113549.1.1.11") + RSA_WITH_SHA384 = ObjectIdentifier("1.2.840.113549.1.1.12") + RSA_WITH_SHA512 = ObjectIdentifier("1.2.840.113549.1.1.13") + RSA_WITH_SHA3_224 = ObjectIdentifier("2.16.840.1.101.3.4.3.13") + RSA_WITH_SHA3_256 = ObjectIdentifier("2.16.840.1.101.3.4.3.14") + RSA_WITH_SHA3_384 = ObjectIdentifier("2.16.840.1.101.3.4.3.15") + RSA_WITH_SHA3_512 = ObjectIdentifier("2.16.840.1.101.3.4.3.16") + RSASSA_PSS = ObjectIdentifier("1.2.840.113549.1.1.10") + ECDSA_WITH_SHA1 = ObjectIdentifier("1.2.840.10045.4.1") + ECDSA_WITH_SHA224 = ObjectIdentifier("1.2.840.10045.4.3.1") + ECDSA_WITH_SHA256 = ObjectIdentifier("1.2.840.10045.4.3.2") + ECDSA_WITH_SHA384 = ObjectIdentifier("1.2.840.10045.4.3.3") + ECDSA_WITH_SHA512 = ObjectIdentifier("1.2.840.10045.4.3.4") + ECDSA_WITH_SHA3_224 = ObjectIdentifier("2.16.840.1.101.3.4.3.9") + ECDSA_WITH_SHA3_256 = ObjectIdentifier("2.16.840.1.101.3.4.3.10") + ECDSA_WITH_SHA3_384 = ObjectIdentifier("2.16.840.1.101.3.4.3.11") + ECDSA_WITH_SHA3_512 = ObjectIdentifier("2.16.840.1.101.3.4.3.12") + DSA_WITH_SHA1 = ObjectIdentifier("1.2.840.10040.4.3") + DSA_WITH_SHA224 = ObjectIdentifier("2.16.840.1.101.3.4.3.1") + DSA_WITH_SHA256 = ObjectIdentifier("2.16.840.1.101.3.4.3.2") + DSA_WITH_SHA384 = ObjectIdentifier("2.16.840.1.101.3.4.3.3") + DSA_WITH_SHA512 = ObjectIdentifier("2.16.840.1.101.3.4.3.4") + ED25519 = ObjectIdentifier("1.3.101.112") + ED448 = ObjectIdentifier("1.3.101.113") + ML_DSA_44 = ObjectIdentifier("2.16.840.1.101.3.4.3.17") + ML_DSA_65 = ObjectIdentifier("2.16.840.1.101.3.4.3.18") + ML_DSA_87 = ObjectIdentifier("2.16.840.1.101.3.4.3.19") + GOSTR3411_94_WITH_3410_2001 = ObjectIdentifier("1.2.643.2.2.3") + GOSTR3410_2012_WITH_3411_2012_256 = ObjectIdentifier("1.2.643.7.1.1.3.2") + GOSTR3410_2012_WITH_3411_2012_512 = ObjectIdentifier("1.2.643.7.1.1.3.3") + UNSIGNED = ObjectIdentifier("1.3.6.1.5.5.7.6.36") + + +_SIG_OIDS_TO_HASH: dict[ObjectIdentifier, hashes.HashAlgorithm | None] = { + SignatureAlgorithmOID.RSA_WITH_MD5: hashes.MD5(), + SignatureAlgorithmOID.RSA_WITH_SHA1: hashes.SHA1(), + SignatureAlgorithmOID._RSA_WITH_SHA1: hashes.SHA1(), + SignatureAlgorithmOID.RSA_WITH_SHA224: hashes.SHA224(), + SignatureAlgorithmOID.RSA_WITH_SHA256: hashes.SHA256(), + SignatureAlgorithmOID.RSA_WITH_SHA384: hashes.SHA384(), + SignatureAlgorithmOID.RSA_WITH_SHA512: hashes.SHA512(), + SignatureAlgorithmOID.RSA_WITH_SHA3_224: hashes.SHA3_224(), + SignatureAlgorithmOID.RSA_WITH_SHA3_256: hashes.SHA3_256(), + SignatureAlgorithmOID.RSA_WITH_SHA3_384: hashes.SHA3_384(), + SignatureAlgorithmOID.RSA_WITH_SHA3_512: hashes.SHA3_512(), + SignatureAlgorithmOID.ECDSA_WITH_SHA1: hashes.SHA1(), + SignatureAlgorithmOID.ECDSA_WITH_SHA224: hashes.SHA224(), + SignatureAlgorithmOID.ECDSA_WITH_SHA256: hashes.SHA256(), + SignatureAlgorithmOID.ECDSA_WITH_SHA384: hashes.SHA384(), + SignatureAlgorithmOID.ECDSA_WITH_SHA512: hashes.SHA512(), + SignatureAlgorithmOID.ECDSA_WITH_SHA3_224: hashes.SHA3_224(), + SignatureAlgorithmOID.ECDSA_WITH_SHA3_256: hashes.SHA3_256(), + SignatureAlgorithmOID.ECDSA_WITH_SHA3_384: hashes.SHA3_384(), + SignatureAlgorithmOID.ECDSA_WITH_SHA3_512: hashes.SHA3_512(), + SignatureAlgorithmOID.DSA_WITH_SHA1: hashes.SHA1(), + SignatureAlgorithmOID.DSA_WITH_SHA224: hashes.SHA224(), + SignatureAlgorithmOID.DSA_WITH_SHA256: hashes.SHA256(), + SignatureAlgorithmOID.ED25519: None, + SignatureAlgorithmOID.ED448: None, + SignatureAlgorithmOID.ML_DSA_44: None, + SignatureAlgorithmOID.ML_DSA_65: None, + SignatureAlgorithmOID.ML_DSA_87: None, + SignatureAlgorithmOID.GOSTR3411_94_WITH_3410_2001: None, + SignatureAlgorithmOID.GOSTR3410_2012_WITH_3411_2012_256: None, + SignatureAlgorithmOID.GOSTR3410_2012_WITH_3411_2012_512: None, + SignatureAlgorithmOID.UNSIGNED: None, +} + + +class HashAlgorithmOID: + SHA1 = ObjectIdentifier("1.3.14.3.2.26") + SHA224 = ObjectIdentifier("2.16.840.1.101.3.4.2.4") + SHA256 = ObjectIdentifier("2.16.840.1.101.3.4.2.1") + SHA384 = ObjectIdentifier("2.16.840.1.101.3.4.2.2") + SHA512 = ObjectIdentifier("2.16.840.1.101.3.4.2.3") + SHA3_224 = ObjectIdentifier("1.3.6.1.4.1.37476.3.2.1.99.7.224") + SHA3_256 = ObjectIdentifier("1.3.6.1.4.1.37476.3.2.1.99.7.256") + SHA3_384 = ObjectIdentifier("1.3.6.1.4.1.37476.3.2.1.99.7.384") + SHA3_512 = ObjectIdentifier("1.3.6.1.4.1.37476.3.2.1.99.7.512") + SHA3_224_NIST = ObjectIdentifier("2.16.840.1.101.3.4.2.7") + SHA3_256_NIST = ObjectIdentifier("2.16.840.1.101.3.4.2.8") + SHA3_384_NIST = ObjectIdentifier("2.16.840.1.101.3.4.2.9") + SHA3_512_NIST = ObjectIdentifier("2.16.840.1.101.3.4.2.10") + + +class PublicKeyAlgorithmOID: + DSA = ObjectIdentifier("1.2.840.10040.4.1") + EC_PUBLIC_KEY = ObjectIdentifier("1.2.840.10045.2.1") + RSAES_PKCS1_v1_5 = ObjectIdentifier("1.2.840.113549.1.1.1") + RSASSA_PSS = ObjectIdentifier("1.2.840.113549.1.1.10") + X25519 = ObjectIdentifier("1.3.101.110") + X448 = ObjectIdentifier("1.3.101.111") + ED25519 = ObjectIdentifier("1.3.101.112") + ED448 = ObjectIdentifier("1.3.101.113") + ML_DSA_44 = ObjectIdentifier("2.16.840.1.101.3.4.3.17") + ML_DSA_65 = ObjectIdentifier("2.16.840.1.101.3.4.3.18") + ML_DSA_87 = ObjectIdentifier("2.16.840.1.101.3.4.3.19") + ML_KEM_768 = ObjectIdentifier("2.16.840.1.101.3.4.4.2") + ML_KEM_1024 = ObjectIdentifier("2.16.840.1.101.3.4.4.3") + + +class ExtendedKeyUsageOID: + SERVER_AUTH = ObjectIdentifier("1.3.6.1.5.5.7.3.1") + CLIENT_AUTH = ObjectIdentifier("1.3.6.1.5.5.7.3.2") + CODE_SIGNING = ObjectIdentifier("1.3.6.1.5.5.7.3.3") + EMAIL_PROTECTION = ObjectIdentifier("1.3.6.1.5.5.7.3.4") + TIME_STAMPING = ObjectIdentifier("1.3.6.1.5.5.7.3.8") + OCSP_SIGNING = ObjectIdentifier("1.3.6.1.5.5.7.3.9") + ANY_EXTENDED_KEY_USAGE = ObjectIdentifier("2.5.29.37.0") + SMARTCARD_LOGON = ObjectIdentifier("1.3.6.1.4.1.311.20.2.2") + KERBEROS_PKINIT_KDC = ObjectIdentifier("1.3.6.1.5.2.3.5") + IPSEC_IKE = ObjectIdentifier("1.3.6.1.5.5.7.3.17") + BUNDLE_SECURITY = ObjectIdentifier("1.3.6.1.5.5.7.3.35") + CERTIFICATE_TRANSPARENCY = ObjectIdentifier("1.3.6.1.4.1.11129.2.4.4") + + +class OtherNameFormOID: + PERMANENT_IDENTIFIER = ObjectIdentifier("1.3.6.1.5.5.7.8.3") + HW_MODULE_NAME = ObjectIdentifier("1.3.6.1.5.5.7.8.4") + DNS_SRV = ObjectIdentifier("1.3.6.1.5.5.7.8.7") + NAI_REALM = ObjectIdentifier("1.3.6.1.5.5.7.8.8") + SMTP_UTF8_MAILBOX = ObjectIdentifier("1.3.6.1.5.5.7.8.9") + ACP_NODE_NAME = ObjectIdentifier("1.3.6.1.5.5.7.8.10") + BUNDLE_EID = ObjectIdentifier("1.3.6.1.5.5.7.8.11") + + +class AuthorityInformationAccessOID: + CA_ISSUERS = ObjectIdentifier("1.3.6.1.5.5.7.48.2") + OCSP = ObjectIdentifier("1.3.6.1.5.5.7.48.1") + + +class SubjectInformationAccessOID: + CA_REPOSITORY = ObjectIdentifier("1.3.6.1.5.5.7.48.5") + + +class CertificatePoliciesOID: + CPS_QUALIFIER = ObjectIdentifier("1.3.6.1.5.5.7.2.1") + CPS_USER_NOTICE = ObjectIdentifier("1.3.6.1.5.5.7.2.2") + ANY_POLICY = ObjectIdentifier("2.5.29.32.0") + + +class AttributeOID: + CHALLENGE_PASSWORD = ObjectIdentifier("1.2.840.113549.1.9.7") + UNSTRUCTURED_NAME = ObjectIdentifier("1.2.840.113549.1.9.2") + + +_OID_NAMES = { + NameOID.COMMON_NAME: "commonName", + NameOID.COUNTRY_NAME: "countryName", + NameOID.LOCALITY_NAME: "localityName", + NameOID.STATE_OR_PROVINCE_NAME: "stateOrProvinceName", + NameOID.STREET_ADDRESS: "streetAddress", + NameOID.ORGANIZATION_NAME: "organizationName", + NameOID.ORGANIZATIONAL_UNIT_NAME: "organizationalUnitName", + NameOID.SERIAL_NUMBER: "serialNumber", + NameOID.SURNAME: "surname", + NameOID.GIVEN_NAME: "givenName", + NameOID.TITLE: "title", + NameOID.GENERATION_QUALIFIER: "generationQualifier", + NameOID.X500_UNIQUE_IDENTIFIER: "x500UniqueIdentifier", + NameOID.DN_QUALIFIER: "dnQualifier", + NameOID.PSEUDONYM: "pseudonym", + NameOID.USER_ID: "userID", + NameOID.DOMAIN_COMPONENT: "domainComponent", + NameOID.EMAIL_ADDRESS: "emailAddress", + NameOID.JURISDICTION_COUNTRY_NAME: "jurisdictionCountryName", + NameOID.JURISDICTION_LOCALITY_NAME: "jurisdictionLocalityName", + NameOID.JURISDICTION_STATE_OR_PROVINCE_NAME: ( + "jurisdictionStateOrProvinceName" + ), + NameOID.BUSINESS_CATEGORY: "businessCategory", + NameOID.POSTAL_ADDRESS: "postalAddress", + NameOID.POSTAL_CODE: "postalCode", + NameOID.INN: "INN", + NameOID.OGRN: "OGRN", + NameOID.SNILS: "SNILS", + NameOID.UNSTRUCTURED_NAME: "unstructuredName", + SignatureAlgorithmOID.RSA_WITH_MD5: "md5WithRSAEncryption", + SignatureAlgorithmOID.RSA_WITH_SHA1: "sha1WithRSAEncryption", + SignatureAlgorithmOID.RSA_WITH_SHA224: "sha224WithRSAEncryption", + SignatureAlgorithmOID.RSA_WITH_SHA256: "sha256WithRSAEncryption", + SignatureAlgorithmOID.RSA_WITH_SHA384: "sha384WithRSAEncryption", + SignatureAlgorithmOID.RSA_WITH_SHA512: "sha512WithRSAEncryption", + SignatureAlgorithmOID.RSASSA_PSS: "rsassaPss", + SignatureAlgorithmOID.ECDSA_WITH_SHA1: "ecdsa-with-SHA1", + SignatureAlgorithmOID.ECDSA_WITH_SHA224: "ecdsa-with-SHA224", + SignatureAlgorithmOID.ECDSA_WITH_SHA256: "ecdsa-with-SHA256", + SignatureAlgorithmOID.ECDSA_WITH_SHA384: "ecdsa-with-SHA384", + SignatureAlgorithmOID.ECDSA_WITH_SHA512: "ecdsa-with-SHA512", + SignatureAlgorithmOID.DSA_WITH_SHA1: "dsa-with-sha1", + SignatureAlgorithmOID.DSA_WITH_SHA224: "dsa-with-sha224", + SignatureAlgorithmOID.DSA_WITH_SHA256: "dsa-with-sha256", + SignatureAlgorithmOID.ED25519: "ed25519", + SignatureAlgorithmOID.ED448: "ed448", + SignatureAlgorithmOID.ML_DSA_44: "ML-DSA-44", + SignatureAlgorithmOID.ML_DSA_65: "ML-DSA-65", + SignatureAlgorithmOID.ML_DSA_87: "ML-DSA-87", + SignatureAlgorithmOID.GOSTR3411_94_WITH_3410_2001: ( + "GOST R 34.11-94 with GOST R 34.10-2001" + ), + SignatureAlgorithmOID.GOSTR3410_2012_WITH_3411_2012_256: ( + "GOST R 34.10-2012 with GOST R 34.11-2012 (256 bit)" + ), + SignatureAlgorithmOID.GOSTR3410_2012_WITH_3411_2012_512: ( + "GOST R 34.10-2012 with GOST R 34.11-2012 (512 bit)" + ), + HashAlgorithmOID.SHA1: "sha1", + HashAlgorithmOID.SHA224: "sha224", + HashAlgorithmOID.SHA256: "sha256", + HashAlgorithmOID.SHA384: "sha384", + HashAlgorithmOID.SHA512: "sha512", + HashAlgorithmOID.SHA3_224: "sha3_224", + HashAlgorithmOID.SHA3_256: "sha3_256", + HashAlgorithmOID.SHA3_384: "sha3_384", + HashAlgorithmOID.SHA3_512: "sha3_512", + HashAlgorithmOID.SHA3_224_NIST: "sha3_224", + HashAlgorithmOID.SHA3_256_NIST: "sha3_256", + HashAlgorithmOID.SHA3_384_NIST: "sha3_384", + HashAlgorithmOID.SHA3_512_NIST: "sha3_512", + PublicKeyAlgorithmOID.DSA: "dsaEncryption", + PublicKeyAlgorithmOID.EC_PUBLIC_KEY: "id-ecPublicKey", + PublicKeyAlgorithmOID.RSAES_PKCS1_v1_5: "rsaEncryption", + PublicKeyAlgorithmOID.X25519: "X25519", + PublicKeyAlgorithmOID.X448: "X448", + ExtendedKeyUsageOID.SERVER_AUTH: "serverAuth", + ExtendedKeyUsageOID.CLIENT_AUTH: "clientAuth", + ExtendedKeyUsageOID.CODE_SIGNING: "codeSigning", + ExtendedKeyUsageOID.EMAIL_PROTECTION: "emailProtection", + ExtendedKeyUsageOID.TIME_STAMPING: "timeStamping", + ExtendedKeyUsageOID.OCSP_SIGNING: "OCSPSigning", + ExtendedKeyUsageOID.SMARTCARD_LOGON: "msSmartcardLogin", + ExtendedKeyUsageOID.KERBEROS_PKINIT_KDC: "pkInitKDC", + ExtensionOID.SUBJECT_DIRECTORY_ATTRIBUTES: "subjectDirectoryAttributes", + ExtensionOID.SUBJECT_KEY_IDENTIFIER: "subjectKeyIdentifier", + ExtensionOID.KEY_USAGE: "keyUsage", + ExtensionOID.PRIVATE_KEY_USAGE_PERIOD: "privateKeyUsagePeriod", + ExtensionOID.SUBJECT_ALTERNATIVE_NAME: "subjectAltName", + ExtensionOID.ISSUER_ALTERNATIVE_NAME: "issuerAltName", + ExtensionOID.BASIC_CONSTRAINTS: "basicConstraints", + ExtensionOID.PRECERT_SIGNED_CERTIFICATE_TIMESTAMPS: ( + "signedCertificateTimestampList" + ), + ExtensionOID.SIGNED_CERTIFICATE_TIMESTAMPS: ( + "signedCertificateTimestampList" + ), + ExtensionOID.PRECERT_POISON: "ctPoison", + ExtensionOID.MS_CERTIFICATE_TEMPLATE: "msCertificateTemplate", + ExtensionOID.ADMISSIONS: "Admissions", + CRLEntryExtensionOID.CRL_REASON: "cRLReason", + CRLEntryExtensionOID.INVALIDITY_DATE: "invalidityDate", + CRLEntryExtensionOID.CERTIFICATE_ISSUER: "certificateIssuer", + ExtensionOID.NAME_CONSTRAINTS: "nameConstraints", + ExtensionOID.CRL_DISTRIBUTION_POINTS: "cRLDistributionPoints", + ExtensionOID.CERTIFICATE_POLICIES: "certificatePolicies", + ExtensionOID.POLICY_MAPPINGS: "policyMappings", + ExtensionOID.AUTHORITY_KEY_IDENTIFIER: "authorityKeyIdentifier", + ExtensionOID.POLICY_CONSTRAINTS: "policyConstraints", + ExtensionOID.EXTENDED_KEY_USAGE: "extendedKeyUsage", + ExtensionOID.FRESHEST_CRL: "freshestCRL", + ExtensionOID.INHIBIT_ANY_POLICY: "inhibitAnyPolicy", + ExtensionOID.ISSUING_DISTRIBUTION_POINT: "issuingDistributionPoint", + ExtensionOID.AUTHORITY_INFORMATION_ACCESS: "authorityInfoAccess", + ExtensionOID.SUBJECT_INFORMATION_ACCESS: "subjectInfoAccess", + ExtensionOID.OCSP_NO_CHECK: "OCSPNoCheck", + ExtensionOID.CRL_NUMBER: "cRLNumber", + ExtensionOID.DELTA_CRL_INDICATOR: "deltaCRLIndicator", + ExtensionOID.TLS_FEATURE: "TLSFeature", + AuthorityInformationAccessOID.OCSP: "OCSP", + AuthorityInformationAccessOID.CA_ISSUERS: "caIssuers", + SubjectInformationAccessOID.CA_REPOSITORY: "caRepository", + CertificatePoliciesOID.CPS_QUALIFIER: "id-qt-cps", + CertificatePoliciesOID.CPS_USER_NOTICE: "id-qt-unotice", + OCSPExtensionOID.NONCE: "OCSPNonce", + AttributeOID.CHALLENGE_PASSWORD: "challengePassword", +} diff --git a/py_modules/cryptography/hazmat/asn1/__init__.py b/py_modules/cryptography/hazmat/asn1/__init__.py new file mode 100644 index 0000000..d19e2a6 --- /dev/null +++ b/py_modules/cryptography/hazmat/asn1/__init__.py @@ -0,0 +1,47 @@ +# This file is dual licensed under the terms of the Apache License, Version +# 2.0, and the BSD License. See the LICENSE file in the root of this repository +# for complete details. + +from __future__ import annotations + +from cryptography.hazmat.asn1.asn1 import ( + TLV, + BitString, + Default, + Explicit, + GeneralizedTime, + IA5String, + Implicit, + Null, + PrintableString, + SetOf, + Size, + UTCTime, + Variant, + decode_der, + encode_der, + sequence, + set, + value_set, +) + +__all__ = [ + "TLV", + "BitString", + "Default", + "Explicit", + "GeneralizedTime", + "IA5String", + "Implicit", + "Null", + "PrintableString", + "SetOf", + "Size", + "UTCTime", + "Variant", + "decode_der", + "encode_der", + "sequence", + "set", + "value_set", +] diff --git a/py_modules/cryptography/hazmat/asn1/asn1.py b/py_modules/cryptography/hazmat/asn1/asn1.py new file mode 100644 index 0000000..62d3811 --- /dev/null +++ b/py_modules/cryptography/hazmat/asn1/asn1.py @@ -0,0 +1,533 @@ +# This file is dual licensed under the terms of the Apache License, Version +# 2.0, and the BSD License. See the LICENSE file in the root of this repository +# for complete details. + +from __future__ import annotations + +import builtins +import dataclasses +import enum +import sys +import types +import typing + +if sys.version_info < (3, 11): + import typing_extensions + + LiteralString = typing_extensions.LiteralString +else: + LiteralString = typing.LiteralString + +from cryptography.hazmat.bindings._rust import declarative_asn1 +from cryptography.hazmat.bindings._rust import x509 as rust_x509 + +if sys.version_info < (3, 10): + NoneType = type(None) +else: + NoneType = types.NoneType # type: ignore[nonetype-type] + +T = typing.TypeVar("T", covariant=True) +U = typing.TypeVar("U") +Tag = typing.TypeVar("Tag", bound=LiteralString) + + +@dataclasses.dataclass(frozen=True) +class Variant(typing.Generic[U, Tag]): + """ + A tagged variant for CHOICE fields with the same underlying type. + + Use this when you have multiple CHOICE alternatives with the same type + and need to distinguish between them: + + foo: ( + Annotated[Variant[int, typing.Literal["IntA"]], Implicit(0)] + | Annotated[Variant[int, typing.Literal["IntB"]], Implicit(1)] + ) + + Usage: + example = Example(foo=Variant(5, "IntA")) + decoded.foo.value # The int value + decoded.foo.tag # "IntA" or "IntB" + """ + + value: U + tag: str + + +decode_der = declarative_asn1.decode_der +encode_der = declarative_asn1.encode_der + + +_X509_TYPES = ( + rust_x509.Certificate, + rust_x509.CertificateSigningRequest, + rust_x509.CertificateRevocationList, +) + + +def _check_x509_field_annotations( + field_type: typing.Any, + annotation: declarative_asn1.Annotation, + field_name: str, +) -> None: + if field_type in _X509_TYPES and isinstance(annotation.encoding, Implicit): + raise TypeError( + f"field '{field_name}' has an IMPLICIT annotation, but " + "IMPLICIT annotations are not supported for X.509 types." + ) + + +def _is_union(field_type: type) -> bool: + # NOTE: types.UnionType for `T | U`, typing.Union for `Union[T, U]`. + # TODO: Drop the `hasattr()` once the minimum supported Python version + # is >= 3.10. + union_types = ( + (types.UnionType, typing.Union) + if hasattr(types, "UnionType") + else (typing.Union,) + ) + return typing.get_origin(field_type) in union_types + + +def _resolve_type_aliases(field_type: typing.Any) -> typing.Any: + # Recursively resolve PEP 695 (`type X = ...`) type aliases (Python + # 3.12+) to their underlying value, so that the rest of the + # normalization logic never encounters an alias. Aliases can refer + # to other aliases and can appear at any level of nesting (e.g. + # inside `Annotated[...]`, unions, or `list[...]`). + if sys.version_info < (3, 12): + return field_type + + while isinstance(field_type, typing.TypeAliasType): + field_type = field_type.__value__ + + args = typing.get_args(field_type) + resolved_args = tuple(_resolve_type_aliases(arg) for arg in args) + if resolved_args == args: + # No aliases anywhere inside: return the type unchanged. + return field_type + + if _is_union(field_type): + # `X | Y` unions can't be rebuilt through their origin like + # other generics below: `typing.get_origin` returns + # `types.UnionType` for them, which is only subscriptable on + # Python 3.14+. Rebuilding through `typing.Union` also + # flattens any nested union introduced by an alias of a union + # (e.g. `Time | int` where `type Time = UTCTime | + # GeneralizedTime`), just like `typing.Union` would have done + # if the alias had been written inline. + return typing.Union[resolved_args] + + # An alias appeared inside a generic (e.g. `Annotated[Time, ...]`, + # `list[MyInt]`, or `SetOf[MyInt]`): re-parameterize the generic + # with the resolved arguments. Subscripting with a tuple is + # equivalent to subscripting with multiple arguments. + return typing.get_origin(field_type)[resolved_args] + + +def _extract_annotation( + metadata: tuple, field_name: str +) -> declarative_asn1.Annotation: + default = None + encoding = None + size = None + for raw_annotation in metadata: + if isinstance(raw_annotation, Default): + if default is not None: + raise TypeError( + f"multiple DEFAULT annotations found in field " + f"'{field_name}'" + ) + default = raw_annotation.value + elif isinstance(raw_annotation, declarative_asn1.Encoding): + if encoding is not None: + raise TypeError( + f"multiple IMPLICIT/EXPLICIT annotations found in field " + f"'{field_name}'" + ) + encoding = raw_annotation + elif isinstance(raw_annotation, declarative_asn1.Size): + if size is not None: + raise TypeError( + f"multiple SIZE annotations found in field '{field_name}'" + ) + size = raw_annotation + else: + raise TypeError(f"unsupported annotation: {raw_annotation}") + + return declarative_asn1.Annotation( + default=default, encoding=encoding, size=size + ) + + +def _normalize_field_type( + field_type: typing.Any, field_name: str +) -> declarative_asn1.AnnotatedType: + field_type = _resolve_type_aliases(field_type) + + # Strip the `Annotated[...]` off, and populate the annotation + # from it if it exists. + if typing.get_origin(field_type) is typing.Annotated: + annotation = _extract_annotation(field_type.__metadata__, field_name) + field_type, *_ = typing.get_args(field_type) + else: + annotation = declarative_asn1.Annotation() + + if annotation.size is not None and ( + typing.get_origin(field_type) not in (builtins.list, SetOf) + and field_type + not in ( + builtins.bytes, + builtins.str, + BitString, + IA5String, + PrintableString, + ) + ): + raise TypeError( + f"field '{field_name}' has a SIZE annotation, but SIZE " + "annotations are only supported for fields of types: " + "[SEQUENCE OF, SET OF, BIT STRING, OCTET STRING, UTF8String, " + "PrintableString, IA5String]" + ) + + if field_type is TLV: + if isinstance(annotation.encoding, Implicit): + raise TypeError( + f"field '{field_name}' has an IMPLICIT annotation, but " + "IMPLICIT annotations are not supported for TLV types." + ) + elif annotation.default is not None: + raise TypeError( + f"field '{field_name}' has a DEFAULT annotation, but " + "DEFAULT annotations are not supported for TLV types." + ) + + _check_x509_field_annotations(field_type, annotation, field_name) + + if hasattr(field_type, "__asn1_root__"): + root_type = field_type.__asn1_root__ + if not isinstance( + root_type, + ( + declarative_asn1.Type.Sequence, + declarative_asn1.Type.Set, + declarative_asn1.Type.ValueSet, + ), + ): + raise TypeError(f"unsupported root type: {root_type}") + return declarative_asn1.AnnotatedType( + typing.cast(declarative_asn1.Type, root_type), annotation + ) + elif _is_union(field_type): + union_args = typing.get_args(field_type) + if len(union_args) == 2 and NoneType in union_args: + # A Union between a type and None is an OPTIONAL + optional_type = ( + union_args[0] if union_args[1] is type(None) else union_args[1] + ) + if optional_type is TLV: + raise TypeError( + "optional TLV types (`TLV | None`) are not " + "currently supported" + ) + # For optional types, the annotation is associated with the + # union, so we check it against the inner type here. + _check_x509_field_annotations( + optional_type, annotation, field_name + ) + annotated_type = _normalize_field_type(optional_type, field_name) + + if not annotated_type.annotation.is_empty(): + raise TypeError( + "optional (`X | None`) types cannot have `X` " + "annotated: annotations must apply to the union " + "(i.e: `Annotated[X | None, annotation]`)" + ) + + if annotation.default is not None: + raise TypeError( + "optional (`X | None`) types should not have a DEFAULT " + "annotation" + ) + + rust_field_type = declarative_asn1.Type.Option(annotated_type) + + else: + # Otherwise, the Union is a CHOICE + if isinstance(annotation.encoding, Implicit): + # CHOICEs cannot be IMPLICIT. See X.680 section 31.2.9. + raise TypeError( + "CHOICE (`X | Y | ...`) types should not have an IMPLICIT " + "annotation" + ) + variants = [ + _type_to_variant(arg, field_name) + for arg in union_args + if arg is not type(None) + ] + + # Union types should either be all Variants + # (`Variant[..] | Variant[..] | etc`) or all non Variants + are_union_types_tagged = variants[0].tag_name is not None + if any( + (v.tag_name is not None) != are_union_types_tagged + for v in variants + ): + raise TypeError( + "When using `asn1.Variant` in a union, all the other " + "types in the union must also be `asn1.Variant`" + ) + + if are_union_types_tagged: + tags = {v.tag_name for v in variants} + if len(variants) != len(tags): + raise TypeError( + "When using `asn1.Variant` in a union, the tags used " + "must be unique" + ) + + rust_choice_type = declarative_asn1.Type.Choice(variants) + # If None is part of the union types, this is an OPTIONAL CHOICE + rust_field_type = ( + declarative_asn1.Type.Option( + declarative_asn1.AnnotatedType( + rust_choice_type, declarative_asn1.Annotation() + ) + ) + if NoneType in union_args + else rust_choice_type + ) + + elif typing.get_origin(field_type) is builtins.list: + inner_type = _normalize_field_type( + typing.get_args(field_type)[0], field_name + ) + rust_field_type = declarative_asn1.Type.SequenceOf(inner_type) + elif typing.get_origin(field_type) is SetOf: + inner_type = _normalize_field_type( + typing.get_args(field_type)[0], field_name + ) + rust_field_type = declarative_asn1.Type.SetOf(inner_type) + else: + rust_field_type = declarative_asn1.non_root_python_to_rust(field_type) + + return declarative_asn1.AnnotatedType(rust_field_type, annotation) + + +# Convert a type to a Variant. Used with types inside Union +# annotations (T1, T2, etc in `Union[T1, T2, ...]`). +def _type_to_variant( + t: typing.Any, field_name: str +) -> declarative_asn1.Variant: + is_annotated = typing.get_origin(t) is typing.Annotated + inner_type = typing.get_args(t)[0] if is_annotated else t + + # Check if this is a Variant[T, Tag] type + if typing.get_origin(inner_type) is Variant: + value_type, tag_literal = typing.get_args(inner_type) + if typing.get_origin(tag_literal) is not typing.Literal: + raise TypeError( + "When using `asn1.Variant` in a type annotation, the second " + "type parameter must be a `typing.Literal` type. E.g: " + '`Variant[int, typing.Literal["MyInt"]]`.' + ) + tag_name = typing.get_args(tag_literal)[0] + + if hasattr(value_type, "__asn1_root__"): + rust_type = value_type.__asn1_root__ + else: + rust_type = declarative_asn1.non_root_python_to_rust(value_type) + + if is_annotated: + ann_type = declarative_asn1.AnnotatedType( + rust_type, + _extract_annotation(t.__metadata__, field_name), + ) + else: + ann_type = declarative_asn1.AnnotatedType( + rust_type, + declarative_asn1.Annotation(), + ) + + return declarative_asn1.Variant(Variant, ann_type, tag_name) + else: + # Plain type (not a tagged Variant) + return declarative_asn1.Variant( + inner_type, + _normalize_field_type(t, field_name), + None, + ) + + +def _annotate_fields( + raw_fields: dict[str, type], +) -> dict[str, declarative_asn1.AnnotatedType]: + fields = {} + for field_name, field_type in raw_fields.items(): + # Recursively normalize the field type into something that the + # Rust code can understand. + annotated_field_type = _normalize_field_type(field_type, field_name) + fields[field_name] = annotated_field_type + + return fields + + +def _register_asn1_sequence(cls: type[U]) -> None: + raw_fields = typing.get_type_hints(cls, include_extras=True) + root = declarative_asn1.Type.Sequence(cls, _annotate_fields(raw_fields)) + + setattr(cls, "__asn1_root__", root) + + +def _register_asn1_set(cls: type[U]) -> None: + raw_fields = typing.get_type_hints(cls, include_extras=True) + root = declarative_asn1.Type.Set(cls, _annotate_fields(raw_fields)) + + setattr(cls, "__asn1_root__", root) + + +# Due to https://github.com/python/mypy/issues/19731, we can't define an alias +# for `dataclass_transform` that conditionally points to `typing` or +# `typing_extensions` depending on the Python version. We work around it by +# making the whole decorated class conditional on the Python version. +if sys.version_info < (3, 11): + + @typing_extensions.dataclass_transform(kw_only_default=True) + def sequence(cls: type[U]) -> type[U]: + # We use `dataclasses.dataclass` to add an __init__ method + # to the class with keyword-only parameters. + if sys.version_info >= (3, 10): + dataclass_cls = dataclasses.dataclass( + repr=False, + eq=False, + # `match_args` was added in Python 3.10 and defaults + # to True + match_args=False, + # `kw_only` was added in Python 3.10 and defaults to + # False + kw_only=True, + )(cls) + else: + dataclass_cls = dataclasses.dataclass( + repr=False, + eq=False, + )(cls) + _register_asn1_sequence(dataclass_cls) + return dataclass_cls + + @typing_extensions.dataclass_transform(kw_only_default=True) + def set(cls: type[U]) -> type[U]: + # We use `dataclasses.dataclass` to add an __init__ method + # to the class with keyword-only parameters. + if sys.version_info >= (3, 10): + dataclass_cls = dataclasses.dataclass( + repr=False, + eq=False, + # `match_args` was added in Python 3.10 and defaults + # to True + match_args=False, + # `kw_only` was added in Python 3.10 and defaults to + # False + kw_only=True, + )(cls) + else: + dataclass_cls = dataclasses.dataclass( + repr=False, + eq=False, + )(cls) + _register_asn1_set(dataclass_cls) + return dataclass_cls + +else: + + @typing.dataclass_transform(kw_only_default=True) + def sequence(cls: type[U]) -> type[U]: + # Only add an __init__ method, with keyword-only + # parameters. + dataclass_cls = dataclasses.dataclass( + repr=False, + eq=False, + match_args=False, + kw_only=True, + )(cls) + _register_asn1_sequence(dataclass_cls) + return dataclass_cls + + @typing.dataclass_transform(kw_only_default=True) + def set(cls: type[U]) -> type[U]: + # Only add an __init__ method, with keyword-only + # parameters. + dataclass_cls = dataclasses.dataclass( + repr=False, + eq=False, + match_args=False, + kw_only=True, + )(cls) + _register_asn1_set(dataclass_cls) + return dataclass_cls + + +def value_set( + value_type: type, +) -> typing.Callable[[type[U]], type[U]]: + """ + A class decorator that registers an `enum.Enum` subclass as an + ASN.1 value set of the given underlying type. All the member + values must be instances of `value_type`. Members are encoded as + their value; decoding fails if the decoded value does not match + any member. + """ + rust_type = declarative_asn1.non_root_python_to_rust(value_type) + + def decorator(cls: type[U]) -> type[U]: + if not issubclass(cls, enum.Enum): + raise TypeError( + "value sets can only be defined from enum.Enum subclasses" + ) + members = list(cls) + if not members: + raise TypeError( + f"value set '{cls.__name__}' must have at least one member" + ) + for member in members: + if not isinstance(member.value, value_type): + raise TypeError( + f"member '{member.name}' of value set '{cls.__name__}' " + f"must have a value of type " + f"'{value_type.__name__}', got: " + f"'{type(member.value).__name__}'" + ) + inner = declarative_asn1.AnnotatedType( + rust_type, declarative_asn1.Annotation() + ) + # Map from member value to member, used for O(1) lookups when + # decoding. This requires the member values to be hashable. + value_map = {member.value: member for member in members} + root = declarative_asn1.Type.ValueSet(cls, inner, value_map) + + setattr(cls, "__asn1_root__", root) + return cls + + return decorator + + +# TODO: replace with `Default[U]` once the min Python version is >= 3.12 +@dataclasses.dataclass(frozen=True) +class Default(typing.Generic[U]): + value: U + + +SetOf = declarative_asn1.SetOf + +Explicit = declarative_asn1.Encoding.Explicit +Implicit = declarative_asn1.Encoding.Implicit +Size = declarative_asn1.Size + +PrintableString = declarative_asn1.PrintableString +IA5String = declarative_asn1.IA5String +UTCTime = declarative_asn1.UTCTime +GeneralizedTime = declarative_asn1.GeneralizedTime +BitString = declarative_asn1.BitString +TLV = declarative_asn1.Tlv +Null = declarative_asn1.Null diff --git a/py_modules/cryptography/hazmat/backends/__init__.py b/py_modules/cryptography/hazmat/backends/__init__.py new file mode 100644 index 0000000..b4400aa --- /dev/null +++ b/py_modules/cryptography/hazmat/backends/__init__.py @@ -0,0 +1,13 @@ +# This file is dual licensed under the terms of the Apache License, Version +# 2.0, and the BSD License. See the LICENSE file in the root of this repository +# for complete details. + +from __future__ import annotations + +from typing import Any + + +def default_backend() -> Any: + from cryptography.hazmat.backends.openssl.backend import backend + + return backend diff --git a/py_modules/cryptography/hazmat/backends/openssl/__init__.py b/py_modules/cryptography/hazmat/backends/openssl/__init__.py new file mode 100644 index 0000000..51b0447 --- /dev/null +++ b/py_modules/cryptography/hazmat/backends/openssl/__init__.py @@ -0,0 +1,9 @@ +# This file is dual licensed under the terms of the Apache License, Version +# 2.0, and the BSD License. See the LICENSE file in the root of this repository +# for complete details. + +from __future__ import annotations + +from cryptography.hazmat.backends.openssl.backend import backend + +__all__ = ["backend"] diff --git a/py_modules/cryptography/hazmat/backends/openssl/backend.py b/py_modules/cryptography/hazmat/backends/openssl/backend.py new file mode 100644 index 0000000..0d02c4f --- /dev/null +++ b/py_modules/cryptography/hazmat/backends/openssl/backend.py @@ -0,0 +1,309 @@ +# This file is dual licensed under the terms of the Apache License, Version +# 2.0, and the BSD License. See the LICENSE file in the root of this repository +# for complete details. + +from __future__ import annotations + +from cryptography.hazmat.bindings._rust import openssl as rust_openssl +from cryptography.hazmat.bindings.openssl import binding +from cryptography.hazmat.primitives import hashes +from cryptography.hazmat.primitives._asymmetric import AsymmetricPadding +from cryptography.hazmat.primitives.asymmetric import ec +from cryptography.hazmat.primitives.asymmetric import utils as asym_utils +from cryptography.hazmat.primitives.asymmetric.padding import ( + MGF1, + OAEP, + PSS, + PKCS1v15, +) +from cryptography.hazmat.primitives.ciphers import ( + CipherAlgorithm, +) +from cryptography.hazmat.primitives.ciphers.algorithms import ( + AES, +) +from cryptography.hazmat.primitives.ciphers.modes import ( + CBC, + Mode, +) + + +class Backend: + """ + OpenSSL API binding interfaces. + """ + + name = "openssl" + + # TripleDES encryption is disallowed/deprecated throughout 2023 in + # FIPS 140-3. To keep it simple we denylist any use of TripleDES (TDEA). + _fips_ciphers = (AES,) + # Sometimes SHA1 is still permissible. That logic is contained + # within the various *_supported methods. + _fips_hashes = ( + hashes.SHA224, + hashes.SHA256, + hashes.SHA384, + hashes.SHA512, + hashes.SHA512_224, + hashes.SHA512_256, + hashes.SHA3_224, + hashes.SHA3_256, + hashes.SHA3_384, + hashes.SHA3_512, + hashes.SHAKE128, + hashes.SHAKE256, + ) + _fips_ecdh_curves = ( + ec.SECP224R1, + ec.SECP256R1, + ec.SECP384R1, + ec.SECP521R1, + ) + _fips_rsa_min_key_size = 2048 + _fips_rsa_min_public_exponent = 65537 + _fips_dsa_min_modulus = 1 << 2048 + _fips_dh_min_key_size = 2048 + _fips_dh_min_modulus = 1 << _fips_dh_min_key_size + + def __init__(self) -> None: + self._binding = binding.Binding() + self._ffi = self._binding.ffi + self._lib = self._binding.lib + self._fips_enabled = rust_openssl.is_fips_enabled() + + def __repr__(self) -> str: + return ( + f"" + ) + + def openssl_assert(self, ok: bool) -> None: + return binding._openssl_assert(ok) + + def _enable_fips(self) -> None: + # This function enables FIPS mode for OpenSSL 3.0.0 on installs that + # have the FIPS provider installed properly. + rust_openssl.enable_fips(rust_openssl._providers) + assert rust_openssl.is_fips_enabled() + self._fips_enabled = rust_openssl.is_fips_enabled() + + def openssl_version_text(self) -> str: + """ + Friendly string name of the loaded OpenSSL library. This is not + necessarily the same version as it was compiled against. + + Example: OpenSSL 3.2.1 30 Jan 2024 + """ + return rust_openssl.openssl_version_text() + + def openssl_version_number(self) -> int: + return rust_openssl.openssl_version() + + def hash_supported(self, algorithm: hashes.HashAlgorithm) -> bool: + if self._fips_enabled and not isinstance(algorithm, self._fips_hashes): + return False + + return rust_openssl.hashes.hash_supported(algorithm) + + def signature_hash_supported( + self, algorithm: hashes.HashAlgorithm + ) -> bool: + # Dedicated check for hashing algorithm use in message digest for + # signatures, e.g. RSA PKCS#1 v1.5 SHA1 (sha1WithRSAEncryption). + if self._fips_enabled and isinstance(algorithm, hashes.SHA1): + return False + return self.hash_supported(algorithm) + + def scrypt_supported(self) -> bool: + if self._fips_enabled: + return False + else: + return hasattr(rust_openssl.kdf.Scrypt, "derive") + + def argon2_supported(self) -> bool: + if self._fips_enabled: + return False + else: + return hasattr(rust_openssl.kdf.Argon2id, "derive") + + def hmac_supported(self, algorithm: hashes.HashAlgorithm) -> bool: + # FIPS mode still allows SHA1 for HMAC + if self._fips_enabled and isinstance(algorithm, hashes.SHA1): + return True + if rust_openssl.CRYPTOGRAPHY_IS_AWSLC: + return isinstance( + algorithm, + ( + hashes.MD5, + hashes.SHA1, + hashes.SHA224, + hashes.SHA256, + hashes.SHA384, + hashes.SHA512, + hashes.SHA512_224, + hashes.SHA512_256, + hashes.SHA3_224, + hashes.SHA3_256, + hashes.SHA3_384, + hashes.SHA3_512, + ), + ) + return self.hash_supported(algorithm) + + def cipher_supported(self, cipher: CipherAlgorithm, mode: Mode) -> bool: + if self._fips_enabled: + # FIPS mode requires AES. TripleDES is disallowed/deprecated in + # FIPS 140-3. + if not isinstance(cipher, self._fips_ciphers): + return False + + return rust_openssl.ciphers.cipher_supported(cipher, mode) + + def _consume_errors(self) -> list[rust_openssl.OpenSSLError]: + return rust_openssl.capture_error_stack() + + def _oaep_hash_supported(self, algorithm: hashes.HashAlgorithm) -> bool: + if self._fips_enabled and isinstance(algorithm, hashes.SHA1): + return False + + return isinstance( + algorithm, + ( + hashes.SHA1, + hashes.SHA224, + hashes.SHA256, + hashes.SHA384, + hashes.SHA512, + ), + ) + + def rsa_padding_supported(self, padding: AsymmetricPadding) -> bool: + if isinstance(padding, PKCS1v15): + return True + elif isinstance(padding, PSS) and isinstance(padding._mgf, MGF1): + # FIPS 186-4 only allows salt length == digest length for PSS + # It is technically acceptable to set an explicit salt length + # equal to the digest length and this will incorrectly fail, but + # since we don't do that in the tests and this method is + # private, we'll ignore that until we need to do otherwise. + if ( + self._fips_enabled + and padding._salt_length != PSS.DIGEST_LENGTH + ): + return False + return self.hash_supported(padding._mgf._algorithm) + elif isinstance(padding, OAEP) and isinstance(padding._mgf, MGF1): + return self._oaep_hash_supported( + padding._mgf._algorithm + ) and self._oaep_hash_supported(padding._algorithm) + else: + return False + + def rsa_encryption_supported(self, padding: AsymmetricPadding) -> bool: + if self._fips_enabled and isinstance(padding, PKCS1v15): + return False + else: + return self.rsa_padding_supported(padding) + + def dsa_supported(self) -> bool: + return ( + not rust_openssl.CRYPTOGRAPHY_IS_BORINGSSL + and not self._fips_enabled + ) + + def dsa_hash_supported(self, algorithm: hashes.HashAlgorithm) -> bool: + if not self.dsa_supported(): + return False + return self.signature_hash_supported(algorithm) + + def cmac_algorithm_supported(self, algorithm) -> bool: + return self.cipher_supported( + algorithm, CBC(b"\x00" * algorithm.block_size) + ) + + def elliptic_curve_supported(self, curve: ec.EllipticCurve) -> bool: + if self._fips_enabled and not isinstance( + curve, self._fips_ecdh_curves + ): + return False + + return rust_openssl.ec.curve_supported(curve) + + def elliptic_curve_signature_algorithm_supported( + self, + signature_algorithm: ec.EllipticCurveSignatureAlgorithm, + curve: ec.EllipticCurve, + ) -> bool: + # We only support ECDSA right now. + if not isinstance(signature_algorithm, ec.ECDSA): + return False + + return self.elliptic_curve_supported(curve) and ( + isinstance(signature_algorithm.algorithm, asym_utils.Prehashed) + or self.hash_supported(signature_algorithm.algorithm) + ) + + def elliptic_curve_exchange_algorithm_supported( + self, algorithm: ec.ECDH, curve: ec.EllipticCurve + ) -> bool: + return self.elliptic_curve_supported(curve) and isinstance( + algorithm, ec.ECDH + ) + + def dh_supported(self) -> bool: + return not rust_openssl.CRYPTOGRAPHY_IS_BORINGSSL + + def dh_x942_serialization_supported(self) -> bool: + return self._lib.Cryptography_HAS_EVP_PKEY_DHX == 1 + + def x25519_supported(self) -> bool: + return not self._fips_enabled + + def x448_supported(self) -> bool: + if self._fips_enabled: + return False + return ( + not rust_openssl.CRYPTOGRAPHY_IS_LIBRESSL + and not rust_openssl.CRYPTOGRAPHY_IS_BORINGSSL + and not rust_openssl.CRYPTOGRAPHY_IS_AWSLC + ) + + def mlkem_supported(self) -> bool: + return ( + rust_openssl.CRYPTOGRAPHY_IS_AWSLC + or rust_openssl.CRYPTOGRAPHY_IS_BORINGSSL + or rust_openssl.CRYPTOGRAPHY_OPENSSL_350_OR_GREATER + ) + + def mldsa_supported(self) -> bool: + return ( + rust_openssl.CRYPTOGRAPHY_IS_AWSLC + or rust_openssl.CRYPTOGRAPHY_IS_BORINGSSL + or rust_openssl.CRYPTOGRAPHY_OPENSSL_350_OR_GREATER + ) + + def ed25519_supported(self) -> bool: + return True + + def ed448_supported(self) -> bool: + return ( + not rust_openssl.CRYPTOGRAPHY_IS_LIBRESSL + and not rust_openssl.CRYPTOGRAPHY_IS_BORINGSSL + and not rust_openssl.CRYPTOGRAPHY_IS_AWSLC + ) + + def ecdsa_deterministic_supported(self) -> bool: + return ( + rust_openssl.CRYPTOGRAPHY_OPENSSL_320_OR_GREATER + and not self._fips_enabled + ) + + def poly1305_supported(self) -> bool: + if rust_openssl.CRYPTOGRAPHY_IS_AWSLC: + return True + return not self._fips_enabled + + +backend = Backend() diff --git a/py_modules/cryptography/hazmat/bindings/__init__.py b/py_modules/cryptography/hazmat/bindings/__init__.py new file mode 100644 index 0000000..41d7318 --- /dev/null +++ b/py_modules/cryptography/hazmat/bindings/__init__.py @@ -0,0 +1,5 @@ +# This file is dual licensed under the terms of the Apache License, Version +# 2.0, and the BSD License. See the LICENSE file in the root of this repository +# for complete details. + +from __future__ import annotations diff --git a/py_modules/cryptography/hazmat/bindings/_rust.abi3.so b/py_modules/cryptography/hazmat/bindings/_rust.abi3.so new file mode 100755 index 0000000..de6c78e Binary files /dev/null and b/py_modules/cryptography/hazmat/bindings/_rust.abi3.so differ diff --git a/py_modules/cryptography/hazmat/bindings/_rust/__init__.pyi b/py_modules/cryptography/hazmat/bindings/_rust/__init__.pyi new file mode 100644 index 0000000..c3148f1 --- /dev/null +++ b/py_modules/cryptography/hazmat/bindings/_rust/__init__.pyi @@ -0,0 +1,67 @@ +# This file is dual licensed under the terms of the Apache License, Version +# 2.0, and the BSD License. See the LICENSE file in the root of this repository +# for complete details. + +import typing + +from cryptography.hazmat.primitives import padding +from cryptography.hazmat.primitives._serialization import ( + KeySerializationEncryptionBuilder, +) +from cryptography.utils import Buffer + +class PKCS7PaddingContext(padding.PaddingContext): + def __init__(self, block_size: int) -> None: ... + def update(self, data: Buffer) -> bytes: ... + def finalize(self) -> bytes: ... + +class ANSIX923PaddingContext(padding.PaddingContext): + def __init__(self, block_size: int) -> None: ... + def update(self, data: Buffer) -> bytes: ... + def finalize(self) -> bytes: ... + +class PKCS7UnpaddingContext(padding.PaddingContext): + def __init__(self, block_size: int) -> None: ... + def update(self, data: Buffer) -> bytes: ... + def finalize(self) -> bytes: ... + +class ANSIX923UnpaddingContext(padding.PaddingContext): + def __init__(self, block_size: int) -> None: ... + def update(self, data: Buffer) -> bytes: ... + def finalize(self) -> bytes: ... + +class Encoding: + PEM: typing.ClassVar[Encoding] + DER: typing.ClassVar[Encoding] + OpenSSH: typing.ClassVar[Encoding] + Raw: typing.ClassVar[Encoding] + X962: typing.ClassVar[Encoding] + SMIME: typing.ClassVar[Encoding] + +class PrivateFormat: + PKCS8: typing.ClassVar[PrivateFormat] + TraditionalOpenSSL: typing.ClassVar[PrivateFormat] + Raw: typing.ClassVar[PrivateFormat] + OpenSSH: typing.ClassVar[PrivateFormat] + PKCS12: typing.ClassVar[PrivateFormat] + def encryption_builder(self) -> KeySerializationEncryptionBuilder: ... + +class PublicFormat: + SubjectPublicKeyInfo: typing.ClassVar[PublicFormat] + PKCS1: typing.ClassVar[PublicFormat] + OpenSSH: typing.ClassVar[PublicFormat] + Raw: typing.ClassVar[PublicFormat] + CompressedPoint: typing.ClassVar[PublicFormat] + UncompressedPoint: typing.ClassVar[PublicFormat] + +class ParameterFormat: + PKCS3: typing.ClassVar[ParameterFormat] + +class ObjectIdentifier: + def __init__(self, value: str) -> None: ... + @property + def dotted_string(self) -> str: ... + @property + def _name(self) -> str: ... + +T = typing.TypeVar("T") diff --git a/py_modules/cryptography/hazmat/bindings/_rust/_openssl.pyi b/py_modules/cryptography/hazmat/bindings/_rust/_openssl.pyi new file mode 100644 index 0000000..3d4ea4e --- /dev/null +++ b/py_modules/cryptography/hazmat/bindings/_rust/_openssl.pyi @@ -0,0 +1,8 @@ +# This file is dual licensed under the terms of the Apache License, Version +# 2.0, and the BSD License. See the LICENSE file in the root of this repository +# for complete details. + +import typing + +lib: typing.Any +ffi: typing.Any diff --git a/py_modules/cryptography/hazmat/bindings/_rust/asn1.pyi b/py_modules/cryptography/hazmat/bindings/_rust/asn1.pyi new file mode 100644 index 0000000..3b5f208 --- /dev/null +++ b/py_modules/cryptography/hazmat/bindings/_rust/asn1.pyi @@ -0,0 +1,7 @@ +# This file is dual licensed under the terms of the Apache License, Version +# 2.0, and the BSD License. See the LICENSE file in the root of this repository +# for complete details. + +def decode_dss_signature(signature: bytes) -> tuple[int, int]: ... +def encode_dss_signature(r: int, s: int) -> bytes: ... +def parse_spki_for_data(data: bytes) -> bytes: ... diff --git a/py_modules/cryptography/hazmat/bindings/_rust/cobblestone.pyi b/py_modules/cryptography/hazmat/bindings/_rust/cobblestone.pyi new file mode 100644 index 0000000..95e9583 --- /dev/null +++ b/py_modules/cryptography/hazmat/bindings/_rust/cobblestone.pyi @@ -0,0 +1,33 @@ +# This file is dual licensed under the terms of the Apache License, Version +# 2.0, and the BSD License. See the LICENSE file in the root of this repository +# for complete details. + +from cryptography.utils import Buffer + +class Cobblestone128Encryptor: + def __init__(self, key: Buffer, context: Buffer) -> None: ... + @staticmethod + def generate_key() -> bytes: ... + def update(self, data: Buffer) -> bytes: ... + def update_into(self, data: Buffer, buf: Buffer) -> int: ... + def finalize(self) -> bytes: ... + +class Cobblestone128Decryptor: + def __init__(self, key: Buffer, context: Buffer) -> None: ... + def update(self, data: Buffer) -> bytes: ... + def update_into(self, data: Buffer, buf: Buffer) -> int: ... + def finalize(self) -> bytes: ... + +class Cobblestone256Encryptor: + def __init__(self, key: Buffer, context: Buffer) -> None: ... + @staticmethod + def generate_key() -> bytes: ... + def update(self, data: Buffer) -> bytes: ... + def update_into(self, data: Buffer, buf: Buffer) -> int: ... + def finalize(self) -> bytes: ... + +class Cobblestone256Decryptor: + def __init__(self, key: Buffer, context: Buffer) -> None: ... + def update(self, data: Buffer) -> bytes: ... + def update_into(self, data: Buffer, buf: Buffer) -> int: ... + def finalize(self) -> bytes: ... diff --git a/py_modules/cryptography/hazmat/bindings/_rust/declarative_asn1.pyi b/py_modules/cryptography/hazmat/bindings/_rust/declarative_asn1.pyi new file mode 100644 index 0000000..66fc065 --- /dev/null +++ b/py_modules/cryptography/hazmat/bindings/_rust/declarative_asn1.pyi @@ -0,0 +1,133 @@ +# This file is dual licensed under the terms of the Apache License, Version +# 2.0, and the BSD License. See the LICENSE file in the root of this repository +# for complete details. + +import datetime +import typing + +T = typing.TypeVar("T") + +def decode_der(cls: type[T], value: bytes) -> T: ... +def encode_der(value: typing.Any) -> bytes: ... +def non_root_python_to_rust(cls: type) -> Type: ... + +# Type is a Rust enum with tuple variants. For now, we express the type +# annotations like this: +class Type: + Sequence: typing.ClassVar[type] + SequenceOf: typing.ClassVar[type] + Set: typing.ClassVar[type] + SetOf: typing.ClassVar[type] + Option: typing.ClassVar[type] + Choice: typing.ClassVar[type] + ValueSet: typing.ClassVar[type] + PyBool: typing.ClassVar[type] + PyInt: typing.ClassVar[type] + PyBytes: typing.ClassVar[type] + PyStr: typing.ClassVar[type] + +class Annotation: + default: typing.Any | None + encoding: Encoding | None + size: Size | None + def __new__( + cls, + default: typing.Any | None = None, + encoding: Encoding | None = None, + size: Size | None = None, + ) -> Annotation: ... + def is_empty(self) -> bool: ... + +# Encoding is a Rust enum with tuple variants. For now, we express the type +# annotations like this: +class Encoding: + Implicit: typing.ClassVar[type] + Explicit: typing.ClassVar[type] + +class Size: + min: int + max: int | None + + def __new__(cls, min: int, max: int | None) -> Size: ... + @staticmethod + def exact(n: int) -> Size: ... + +class AnnotatedType: + inner: Type + annotation: Annotation + + def __new__(cls, inner: Type, annotation: Annotation) -> AnnotatedType: ... + +class AnnotatedTypeObject: + annotated_type: AnnotatedType + value: typing.Any + + def __new__( + cls, annotated_type: AnnotatedType, value: typing.Any + ) -> AnnotatedTypeObject: ... + +class Variant: + python_class: type + ann_type: AnnotatedType + tag_name: str | None + + def __new__( + cls, + python_class: type, + ann_type: AnnotatedType, + tag_name: str | None, + ) -> Variant: ... + +class PrintableString: + def __new__(cls, inner: str) -> PrintableString: ... + def __repr__(self) -> str: ... + def __eq__(self, other: object) -> bool: ... + def __hash__(self) -> int: ... + def as_str(self) -> str: ... + +class IA5String: + def __new__(cls, inner: str) -> IA5String: ... + def __repr__(self) -> str: ... + def __eq__(self, other: object) -> bool: ... + def __hash__(self) -> int: ... + def as_str(self) -> str: ... + +class UTCTime: + def __new__(cls, inner: datetime.datetime) -> UTCTime: ... + def __repr__(self) -> str: ... + def __eq__(self, other: object) -> bool: ... + def __hash__(self) -> int: ... + def as_datetime(self) -> datetime.datetime: ... + +class GeneralizedTime: + def __new__(cls, inner: datetime.datetime) -> GeneralizedTime: ... + def __repr__(self) -> str: ... + def __eq__(self, other: object) -> bool: ... + def __hash__(self) -> int: ... + def as_datetime(self) -> datetime.datetime: ... + +class BitString: + def __new__(cls, data: bytes, padding_bits: int) -> BitString: ... + def __repr__(self) -> str: ... + def __eq__(self, other: object) -> bool: ... + def __hash__(self) -> int: ... + def as_bytes(self) -> bytes: ... + def padding_bits(self) -> int: ... + +class Tlv: + @property + def tag_bytes(self) -> bytes: ... + @property + def data(self) -> memoryview: ... + def parse(self, cls: type): ... + +class SetOf(typing.Generic[T]): + def __new__(cls, inner: list[T]) -> SetOf[T]: ... + def as_list(self) -> list[T]: ... + def __eq__(self, other: object) -> bool: ... + def __repr__(self) -> str: ... + +class Null: + def __new__(cls) -> Null: ... + def __repr__(self) -> str: ... + def __eq__(self, other: object) -> bool: ... diff --git a/py_modules/cryptography/hazmat/bindings/_rust/exceptions.pyi b/py_modules/cryptography/hazmat/bindings/_rust/exceptions.pyi new file mode 100644 index 0000000..09f46b1 --- /dev/null +++ b/py_modules/cryptography/hazmat/bindings/_rust/exceptions.pyi @@ -0,0 +1,17 @@ +# This file is dual licensed under the terms of the Apache License, Version +# 2.0, and the BSD License. See the LICENSE file in the root of this repository +# for complete details. + +class _Reasons: + BACKEND_MISSING_INTERFACE: _Reasons + UNSUPPORTED_HASH: _Reasons + UNSUPPORTED_CIPHER: _Reasons + UNSUPPORTED_PADDING: _Reasons + UNSUPPORTED_MGF: _Reasons + UNSUPPORTED_PUBLIC_KEY_ALGORITHM: _Reasons + UNSUPPORTED_ELLIPTIC_CURVE: _Reasons + UNSUPPORTED_SERIALIZATION: _Reasons + UNSUPPORTED_X509: _Reasons + UNSUPPORTED_EXCHANGE_ALGORITHM: _Reasons + UNSUPPORTED_DIFFIE_HELLMAN: _Reasons + UNSUPPORTED_MAC: _Reasons diff --git a/py_modules/cryptography/hazmat/bindings/_rust/ocsp.pyi b/py_modules/cryptography/hazmat/bindings/_rust/ocsp.pyi new file mode 100644 index 0000000..103e96c --- /dev/null +++ b/py_modules/cryptography/hazmat/bindings/_rust/ocsp.pyi @@ -0,0 +1,117 @@ +# This file is dual licensed under the terms of the Apache License, Version +# 2.0, and the BSD License. See the LICENSE file in the root of this repository +# for complete details. + +import datetime +from collections.abc import Iterator + +from cryptography import x509 +from cryptography.hazmat.primitives import hashes, serialization +from cryptography.hazmat.primitives.asymmetric.types import PrivateKeyTypes +from cryptography.x509 import ocsp + +class OCSPRequest: + @property + def issuer_key_hash(self) -> bytes: ... + @property + def issuer_name_hash(self) -> bytes: ... + @property + def hash_algorithm(self) -> hashes.HashAlgorithm: ... + @property + def serial_number(self) -> int: ... + def public_bytes(self, encoding: serialization.Encoding) -> bytes: ... + @property + def extensions(self) -> x509.Extensions: ... + +class OCSPResponse: + @property + def responses(self) -> Iterator[OCSPSingleResponse]: ... + @property + def response_status(self) -> ocsp.OCSPResponseStatus: ... + @property + def signature_algorithm_oid(self) -> x509.ObjectIdentifier: ... + @property + def signature_hash_algorithm( + self, + ) -> hashes.HashAlgorithm | None: ... + @property + def signature(self) -> bytes: ... + @property + def tbs_response_bytes(self) -> bytes: ... + @property + def certificates(self) -> list[x509.Certificate]: ... + @property + def responder_key_hash(self) -> bytes | None: ... + @property + def responder_name(self) -> x509.Name | None: ... + @property + def produced_at(self) -> datetime.datetime: ... + @property + def produced_at_utc(self) -> datetime.datetime: ... + @property + def certificate_status(self) -> ocsp.OCSPCertStatus: ... + @property + def revocation_time(self) -> datetime.datetime | None: ... + @property + def revocation_time_utc(self) -> datetime.datetime | None: ... + @property + def revocation_reason(self) -> x509.ReasonFlags | None: ... + @property + def this_update(self) -> datetime.datetime: ... + @property + def this_update_utc(self) -> datetime.datetime: ... + @property + def next_update(self) -> datetime.datetime | None: ... + @property + def next_update_utc(self) -> datetime.datetime | None: ... + @property + def issuer_key_hash(self) -> bytes: ... + @property + def issuer_name_hash(self) -> bytes: ... + @property + def hash_algorithm(self) -> hashes.HashAlgorithm: ... + @property + def serial_number(self) -> int: ... + @property + def extensions(self) -> x509.Extensions: ... + @property + def single_extensions(self) -> x509.Extensions: ... + def public_bytes(self, encoding: serialization.Encoding) -> bytes: ... + +class OCSPSingleResponse: + @property + def certificate_status(self) -> ocsp.OCSPCertStatus: ... + @property + def revocation_time(self) -> datetime.datetime | None: ... + @property + def revocation_time_utc(self) -> datetime.datetime | None: ... + @property + def revocation_reason(self) -> x509.ReasonFlags | None: ... + @property + def this_update(self) -> datetime.datetime: ... + @property + def this_update_utc(self) -> datetime.datetime: ... + @property + def next_update(self) -> datetime.datetime | None: ... + @property + def next_update_utc(self) -> datetime.datetime | None: ... + @property + def issuer_key_hash(self) -> bytes: ... + @property + def issuer_name_hash(self) -> bytes: ... + @property + def hash_algorithm(self) -> hashes.HashAlgorithm: ... + @property + def serial_number(self) -> int: ... + +def load_der_ocsp_request(data: bytes) -> ocsp.OCSPRequest: ... +def load_der_ocsp_response(data: bytes) -> ocsp.OCSPResponse: ... +def create_ocsp_request( + builder: ocsp.OCSPRequestBuilder, +) -> ocsp.OCSPRequest: ... +def create_ocsp_response( + status: ocsp.OCSPResponseStatus, + builder: ocsp.OCSPResponseBuilder | None, + private_key: PrivateKeyTypes | None, + hash_algorithm: hashes.HashAlgorithm | None, +) -> ocsp.OCSPResponse: ... diff --git a/py_modules/cryptography/hazmat/bindings/_rust/openssl/__init__.pyi b/py_modules/cryptography/hazmat/bindings/_rust/openssl/__init__.pyi new file mode 100644 index 0000000..e41ba78 --- /dev/null +++ b/py_modules/cryptography/hazmat/bindings/_rust/openssl/__init__.pyi @@ -0,0 +1,82 @@ +# This file is dual licensed under the terms of the Apache License, Version +# 2.0, and the BSD License. See the LICENSE file in the root of this repository +# for complete details. + +import typing + +from cryptography.hazmat.bindings._rust.openssl import ( + aead, + ciphers, + cmac, + dh, + dsa, + ec, + ed448, + ed25519, + hashes, + hmac, + hpke, + kdf, + keys, + keywrap, + mldsa, + mlkem, + poly1305, + rsa, + x448, + x25519, +) + +__all__ = [ + "aead", + "ciphers", + "cmac", + "dh", + "dsa", + "ec", + "ed448", + "ed25519", + "hashes", + "hmac", + "hpke", + "kdf", + "keys", + "keywrap", + "mldsa", + "mlkem", + "openssl_version", + "openssl_version_text", + "poly1305", + "raise_openssl_error", + "rsa", + "x448", + "x25519", +] + +CRYPTOGRAPHY_IS_LIBRESSL: bool +CRYPTOGRAPHY_IS_BORINGSSL: bool +CRYPTOGRAPHY_IS_AWSLC: bool +CRYPTOGRAPHY_OPENSSL_309_OR_GREATER: bool +CRYPTOGRAPHY_OPENSSL_320_OR_GREATER: bool +CRYPTOGRAPHY_OPENSSL_330_OR_GREATER: bool +CRYPTOGRAPHY_OPENSSL_350_OR_GREATER: bool + +class Providers: ... + +_legacy_provider_loaded: bool +_providers: Providers + +def openssl_version() -> int: ... +def openssl_version_text() -> str: ... +def raise_openssl_error() -> typing.NoReturn: ... +def capture_error_stack() -> list[OpenSSLError]: ... +def is_fips_enabled() -> bool: ... +def enable_fips(providers: Providers) -> None: ... + +class OpenSSLError: + @property + def lib(self) -> int: ... + @property + def reason(self) -> int: ... + @property + def reason_text(self) -> bytes: ... diff --git a/py_modules/cryptography/hazmat/bindings/_rust/openssl/aead.pyi b/py_modules/cryptography/hazmat/bindings/_rust/openssl/aead.pyi new file mode 100644 index 0000000..eb44608 --- /dev/null +++ b/py_modules/cryptography/hazmat/bindings/_rust/openssl/aead.pyi @@ -0,0 +1,189 @@ +# This file is dual licensed under the terms of the Apache License, Version +# 2.0, and the BSD License. See the LICENSE file in the root of this repository +# for complete details. + +from collections.abc import Sequence + +from cryptography.utils import Buffer + +class AESGCM: + def __init__(self, key: Buffer) -> None: ... + @staticmethod + def generate_key(bit_length: int) -> bytes: ... + def encrypt( + self, + nonce: Buffer, + data: Buffer, + associated_data: Buffer | None, + ) -> bytes: ... + def decrypt( + self, + nonce: Buffer, + data: Buffer, + associated_data: Buffer | None, + ) -> bytes: ... + def encrypt_into( + self, + nonce: Buffer, + data: Buffer, + associated_data: Buffer | None, + buf: Buffer, + ) -> int: ... + def decrypt_into( + self, + nonce: Buffer, + data: Buffer, + associated_data: Buffer | None, + buf: Buffer, + ) -> int: ... + +class ChaCha20Poly1305: + def __init__(self, key: Buffer) -> None: ... + @staticmethod + def generate_key() -> bytes: ... + def encrypt( + self, + nonce: Buffer, + data: Buffer, + associated_data: Buffer | None, + ) -> bytes: ... + def encrypt_into( + self, + nonce: Buffer, + data: Buffer, + associated_data: Buffer | None, + buf: Buffer, + ) -> int: ... + def decrypt( + self, + nonce: Buffer, + data: Buffer, + associated_data: Buffer | None, + ) -> bytes: ... + def decrypt_into( + self, + nonce: Buffer, + data: Buffer, + associated_data: Buffer | None, + buf: Buffer, + ) -> int: ... + +class AESCCM: + def __init__(self, key: Buffer, tag_length: int = 16) -> None: ... + @staticmethod + def generate_key(bit_length: int) -> bytes: ... + def encrypt( + self, + nonce: Buffer, + data: Buffer, + associated_data: Buffer | None, + ) -> bytes: ... + def encrypt_into( + self, + nonce: Buffer, + data: Buffer, + associated_data: Buffer | None, + buf: Buffer, + ) -> int: ... + def decrypt( + self, + nonce: Buffer, + data: Buffer, + associated_data: Buffer | None, + ) -> bytes: ... + def decrypt_into( + self, + nonce: Buffer, + data: Buffer, + associated_data: Buffer | None, + buf: Buffer, + ) -> int: ... + +class AESSIV: + def __init__(self, key: Buffer) -> None: ... + @staticmethod + def generate_key(bit_length: int) -> bytes: ... + def encrypt( + self, + data: Buffer, + associated_data: Sequence[Buffer] | None, + ) -> bytes: ... + def encrypt_into( + self, + data: Buffer, + associated_data: Sequence[Buffer] | None, + buf: Buffer, + ) -> int: ... + def decrypt( + self, + data: Buffer, + associated_data: Sequence[Buffer] | None, + ) -> bytes: ... + def decrypt_into( + self, + data: Buffer, + associated_data: Sequence[Buffer] | None, + buf: Buffer, + ) -> int: ... + +class AESOCB3: + def __init__(self, key: Buffer) -> None: ... + @staticmethod + def generate_key(bit_length: int) -> bytes: ... + def encrypt( + self, + nonce: Buffer, + data: Buffer, + associated_data: Buffer | None, + ) -> bytes: ... + def encrypt_into( + self, + nonce: Buffer, + data: Buffer, + associated_data: Buffer | None, + buf: Buffer, + ) -> int: ... + def decrypt( + self, + nonce: Buffer, + data: Buffer, + associated_data: Buffer | None, + ) -> bytes: ... + def decrypt_into( + self, + nonce: Buffer, + data: Buffer, + associated_data: Buffer | None, + buf: Buffer, + ) -> int: ... + +class AESGCMSIV: + def __init__(self, key: Buffer) -> None: ... + @staticmethod + def generate_key(bit_length: int) -> bytes: ... + def encrypt( + self, + nonce: Buffer, + data: Buffer, + associated_data: Buffer | None, + ) -> bytes: ... + def encrypt_into( + self, + nonce: Buffer, + data: Buffer, + associated_data: Buffer | None, + buf: Buffer, + ) -> int: ... + def decrypt( + self, + nonce: Buffer, + data: Buffer, + associated_data: Buffer | None, + ) -> bytes: ... + def decrypt_into( + self, + nonce: Buffer, + data: Buffer, + associated_data: Buffer | None, + buf: Buffer, + ) -> int: ... diff --git a/py_modules/cryptography/hazmat/bindings/_rust/openssl/ciphers.pyi b/py_modules/cryptography/hazmat/bindings/_rust/openssl/ciphers.pyi new file mode 100644 index 0000000..a48fb01 --- /dev/null +++ b/py_modules/cryptography/hazmat/bindings/_rust/openssl/ciphers.pyi @@ -0,0 +1,38 @@ +# This file is dual licensed under the terms of the Apache License, Version +# 2.0, and the BSD License. See the LICENSE file in the root of this repository +# for complete details. + +import typing + +from cryptography.hazmat.primitives import ciphers +from cryptography.hazmat.primitives.ciphers import modes + +@typing.overload +def create_encryption_ctx( + algorithm: ciphers.CipherAlgorithm, mode: modes.ModeWithAuthenticationTag +) -> ciphers.AEADEncryptionContext: ... +@typing.overload +def create_encryption_ctx( + algorithm: ciphers.CipherAlgorithm, mode: modes.Mode | None +) -> ciphers.CipherContext: ... +@typing.overload +def create_decryption_ctx( + algorithm: ciphers.CipherAlgorithm, mode: modes.ModeWithAuthenticationTag +) -> ciphers.AEADDecryptionContext: ... +@typing.overload +def create_decryption_ctx( + algorithm: ciphers.CipherAlgorithm, mode: modes.Mode | None +) -> ciphers.CipherContext: ... +def cipher_supported( + algorithm: ciphers.CipherAlgorithm, mode: modes.Mode +) -> bool: ... +def _advance( + ctx: ciphers.AEADEncryptionContext | ciphers.AEADDecryptionContext, n: int +) -> None: ... +def _advance_aad( + ctx: ciphers.AEADEncryptionContext | ciphers.AEADDecryptionContext, n: int +) -> None: ... + +class CipherContext: ... +class AEADEncryptionContext: ... +class AEADDecryptionContext: ... diff --git a/py_modules/cryptography/hazmat/bindings/_rust/openssl/cmac.pyi b/py_modules/cryptography/hazmat/bindings/_rust/openssl/cmac.pyi new file mode 100644 index 0000000..9c03508 --- /dev/null +++ b/py_modules/cryptography/hazmat/bindings/_rust/openssl/cmac.pyi @@ -0,0 +1,18 @@ +# This file is dual licensed under the terms of the Apache License, Version +# 2.0, and the BSD License. See the LICENSE file in the root of this repository +# for complete details. + +import typing + +from cryptography.hazmat.primitives import ciphers + +class CMAC: + def __init__( + self, + algorithm: ciphers.BlockCipherAlgorithm, + backend: typing.Any = None, + ) -> None: ... + def update(self, data: bytes) -> None: ... + def finalize(self) -> bytes: ... + def verify(self, signature: bytes) -> None: ... + def copy(self) -> CMAC: ... diff --git a/py_modules/cryptography/hazmat/bindings/_rust/openssl/dh.pyi b/py_modules/cryptography/hazmat/bindings/_rust/openssl/dh.pyi new file mode 100644 index 0000000..08733d7 --- /dev/null +++ b/py_modules/cryptography/hazmat/bindings/_rust/openssl/dh.pyi @@ -0,0 +1,51 @@ +# This file is dual licensed under the terms of the Apache License, Version +# 2.0, and the BSD License. See the LICENSE file in the root of this repository +# for complete details. + +import typing + +from cryptography.hazmat.primitives.asymmetric import dh + +MIN_MODULUS_SIZE: int + +class DHPrivateKey: ... +class DHPublicKey: ... +class DHParameters: ... + +class DHPrivateNumbers: + def __init__(self, x: int, public_numbers: DHPublicNumbers) -> None: ... + def private_key(self, backend: typing.Any = None) -> dh.DHPrivateKey: ... + @property + def x(self) -> int: ... + @property + def public_numbers(self) -> DHPublicNumbers: ... + +class DHPublicNumbers: + def __init__( + self, y: int, parameter_numbers: DHParameterNumbers + ) -> None: ... + def public_key(self, backend: typing.Any = None) -> dh.DHPublicKey: ... + @property + def y(self) -> int: ... + @property + def parameter_numbers(self) -> DHParameterNumbers: ... + +class DHParameterNumbers: + def __init__(self, p: int, g: int, q: int | None = None) -> None: ... + def parameters(self, backend: typing.Any = None) -> dh.DHParameters: ... + @property + def p(self) -> int: ... + @property + def g(self) -> int: ... + @property + def q(self) -> int | None: ... + +def generate_parameters( + generator: int, key_size: int, backend: typing.Any = None +) -> dh.DHParameters: ... +def from_pem_parameters( + data: bytes, backend: typing.Any = None +) -> dh.DHParameters: ... +def from_der_parameters( + data: bytes, backend: typing.Any = None +) -> dh.DHParameters: ... diff --git a/py_modules/cryptography/hazmat/bindings/_rust/openssl/dsa.pyi b/py_modules/cryptography/hazmat/bindings/_rust/openssl/dsa.pyi new file mode 100644 index 0000000..0922a4c --- /dev/null +++ b/py_modules/cryptography/hazmat/bindings/_rust/openssl/dsa.pyi @@ -0,0 +1,41 @@ +# This file is dual licensed under the terms of the Apache License, Version +# 2.0, and the BSD License. See the LICENSE file in the root of this repository +# for complete details. + +import typing + +from cryptography.hazmat.primitives.asymmetric import dsa + +class DSAPrivateKey: ... +class DSAPublicKey: ... +class DSAParameters: ... + +class DSAPrivateNumbers: + def __init__(self, x: int, public_numbers: DSAPublicNumbers) -> None: ... + @property + def x(self) -> int: ... + @property + def public_numbers(self) -> DSAPublicNumbers: ... + def private_key(self, backend: typing.Any = None) -> dsa.DSAPrivateKey: ... + +class DSAPublicNumbers: + def __init__( + self, y: int, parameter_numbers: DSAParameterNumbers + ) -> None: ... + @property + def y(self) -> int: ... + @property + def parameter_numbers(self) -> DSAParameterNumbers: ... + def public_key(self, backend: typing.Any = None) -> dsa.DSAPublicKey: ... + +class DSAParameterNumbers: + def __init__(self, p: int, q: int, g: int) -> None: ... + @property + def p(self) -> int: ... + @property + def q(self) -> int: ... + @property + def g(self) -> int: ... + def parameters(self, backend: typing.Any = None) -> dsa.DSAParameters: ... + +def generate_parameters(key_size: int) -> dsa.DSAParameters: ... diff --git a/py_modules/cryptography/hazmat/bindings/_rust/openssl/ec.pyi b/py_modules/cryptography/hazmat/bindings/_rust/openssl/ec.pyi new file mode 100644 index 0000000..5c3b7bf --- /dev/null +++ b/py_modules/cryptography/hazmat/bindings/_rust/openssl/ec.pyi @@ -0,0 +1,52 @@ +# This file is dual licensed under the terms of the Apache License, Version +# 2.0, and the BSD License. See the LICENSE file in the root of this repository +# for complete details. + +import typing + +from cryptography.hazmat.primitives.asymmetric import ec + +class ECPrivateKey: ... +class ECPublicKey: ... + +class EllipticCurvePrivateNumbers: + def __init__( + self, private_value: int, public_numbers: EllipticCurvePublicNumbers + ) -> None: ... + def private_key( + self, backend: typing.Any = None + ) -> ec.EllipticCurvePrivateKey: ... + @property + def private_value(self) -> int: ... + @property + def public_numbers(self) -> EllipticCurvePublicNumbers: ... + +class EllipticCurvePublicNumbers: + def __init__(self, x: int, y: int, curve: ec.EllipticCurve) -> None: ... + def public_key( + self, backend: typing.Any = None + ) -> ec.EllipticCurvePublicKey: ... + @property + def x(self) -> int: ... + @property + def y(self) -> int: ... + @property + def curve(self) -> ec.EllipticCurve: ... + def __eq__(self, other: object) -> bool: ... + +def curve_supported(curve: ec.EllipticCurve) -> bool: ... +def generate_private_key( + curve: ec.EllipticCurve, backend: typing.Any = None +) -> ec.EllipticCurvePrivateKey: ... +def from_private_numbers( + numbers: ec.EllipticCurvePrivateNumbers, +) -> ec.EllipticCurvePrivateKey: ... +def from_public_numbers( + numbers: ec.EllipticCurvePublicNumbers, +) -> ec.EllipticCurvePublicKey: ... +def from_public_bytes( + curve: ec.EllipticCurve, data: bytes +) -> ec.EllipticCurvePublicKey: ... +def derive_private_key( + private_value: int, curve: ec.EllipticCurve +) -> ec.EllipticCurvePrivateKey: ... diff --git a/py_modules/cryptography/hazmat/bindings/_rust/openssl/ed25519.pyi b/py_modules/cryptography/hazmat/bindings/_rust/openssl/ed25519.pyi new file mode 100644 index 0000000..f85b3d1 --- /dev/null +++ b/py_modules/cryptography/hazmat/bindings/_rust/openssl/ed25519.pyi @@ -0,0 +1,13 @@ +# This file is dual licensed under the terms of the Apache License, Version +# 2.0, and the BSD License. See the LICENSE file in the root of this repository +# for complete details. + +from cryptography.hazmat.primitives.asymmetric import ed25519 +from cryptography.utils import Buffer + +class Ed25519PrivateKey: ... +class Ed25519PublicKey: ... + +def generate_key() -> ed25519.Ed25519PrivateKey: ... +def from_private_bytes(data: Buffer) -> ed25519.Ed25519PrivateKey: ... +def from_public_bytes(data: bytes) -> ed25519.Ed25519PublicKey: ... diff --git a/py_modules/cryptography/hazmat/bindings/_rust/openssl/ed448.pyi b/py_modules/cryptography/hazmat/bindings/_rust/openssl/ed448.pyi new file mode 100644 index 0000000..c8ca0ec --- /dev/null +++ b/py_modules/cryptography/hazmat/bindings/_rust/openssl/ed448.pyi @@ -0,0 +1,13 @@ +# This file is dual licensed under the terms of the Apache License, Version +# 2.0, and the BSD License. See the LICENSE file in the root of this repository +# for complete details. + +from cryptography.hazmat.primitives.asymmetric import ed448 +from cryptography.utils import Buffer + +class Ed448PrivateKey: ... +class Ed448PublicKey: ... + +def generate_key() -> ed448.Ed448PrivateKey: ... +def from_private_bytes(data: Buffer) -> ed448.Ed448PrivateKey: ... +def from_public_bytes(data: bytes) -> ed448.Ed448PublicKey: ... diff --git a/py_modules/cryptography/hazmat/bindings/_rust/openssl/hashes.pyi b/py_modules/cryptography/hazmat/bindings/_rust/openssl/hashes.pyi new file mode 100644 index 0000000..106b531 --- /dev/null +++ b/py_modules/cryptography/hazmat/bindings/_rust/openssl/hashes.pyi @@ -0,0 +1,30 @@ +# This file is dual licensed under the terms of the Apache License, Version +# 2.0, and the BSD License. See the LICENSE file in the root of this repository +# for complete details. + +import typing + +from cryptography.hazmat.primitives import hashes +from cryptography.utils import Buffer + +class Hash(hashes.HashContext): + def __init__( + self, algorithm: hashes.HashAlgorithm, backend: typing.Any = None + ) -> None: ... + @property + def algorithm(self) -> hashes.HashAlgorithm: ... + def update(self, data: Buffer) -> None: ... + def finalize(self) -> bytes: ... + def copy(self) -> Hash: ... + @staticmethod + def hash(algorithm: hashes.HashAlgorithm, data: Buffer) -> bytes: ... + +def hash_supported(algorithm: hashes.HashAlgorithm) -> bool: ... + +class XOFHash: + def __init__(self, algorithm: hashes.ExtendableOutputFunction) -> None: ... + @property + def algorithm(self) -> hashes.ExtendableOutputFunction: ... + def update(self, data: Buffer) -> None: ... + def squeeze(self, length: int) -> bytes: ... + def copy(self) -> XOFHash: ... diff --git a/py_modules/cryptography/hazmat/bindings/_rust/openssl/hmac.pyi b/py_modules/cryptography/hazmat/bindings/_rust/openssl/hmac.pyi new file mode 100644 index 0000000..3883d1b --- /dev/null +++ b/py_modules/cryptography/hazmat/bindings/_rust/openssl/hmac.pyi @@ -0,0 +1,22 @@ +# This file is dual licensed under the terms of the Apache License, Version +# 2.0, and the BSD License. See the LICENSE file in the root of this repository +# for complete details. + +import typing + +from cryptography.hazmat.primitives import hashes +from cryptography.utils import Buffer + +class HMAC(hashes.HashContext): + def __init__( + self, + key: Buffer, + algorithm: hashes.HashAlgorithm, + backend: typing.Any = None, + ) -> None: ... + @property + def algorithm(self) -> hashes.HashAlgorithm: ... + def update(self, data: Buffer) -> None: ... + def finalize(self) -> bytes: ... + def verify(self, signature: bytes) -> None: ... + def copy(self) -> HMAC: ... diff --git a/py_modules/cryptography/hazmat/bindings/_rust/openssl/hpke.pyi b/py_modules/cryptography/hazmat/bindings/_rust/openssl/hpke.pyi new file mode 100644 index 0000000..7ea1711 --- /dev/null +++ b/py_modules/cryptography/hazmat/bindings/_rust/openssl/hpke.pyi @@ -0,0 +1,109 @@ +# This file is dual licensed under the terms of the Apache License, Version +# 2.0, and the BSD License. See the LICENSE file in the root of this repository +# for complete details. + +from cryptography.hazmat.primitives.asymmetric import ec, mlkem, x25519 +from cryptography.utils import Buffer + +class KEM: + X25519: KEM + P256: KEM + P384: KEM + P521: KEM + MLKEM768: KEM + MLKEM1024: KEM + MLKEM768_X25519: KEM + MLKEM1024_P384: KEM + def enc_length(self) -> int: ... + +class KDF: + HKDF_SHA256: KDF + HKDF_SHA384: KDF + HKDF_SHA512: KDF + SHAKE128: KDF + SHAKE256: KDF + +class AEAD: + AES_128_GCM: AEAD + AES_256_GCM: AEAD + CHACHA20_POLY1305: AEAD + +class MLKEM768X25519PrivateKey: + def __init__( + self, + mlkem_key: mlkem.MLKEM768PrivateKey, + x25519_key: x25519.X25519PrivateKey, + ) -> None: ... + def public_key(self) -> MLKEM768X25519PublicKey: ... + +class MLKEM768X25519PublicKey: + def __init__( + self, + mlkem_key: mlkem.MLKEM768PublicKey, + x25519_key: x25519.X25519PublicKey, + ) -> None: ... + +class MLKEM1024P384PrivateKey: + def __init__( + self, + mlkem_key: mlkem.MLKEM1024PrivateKey, + p384_key: ec.EllipticCurvePrivateKey, + ) -> None: ... + def public_key(self) -> MLKEM1024P384PublicKey: ... + +class MLKEM1024P384PublicKey: + def __init__( + self, + mlkem_key: mlkem.MLKEM1024PublicKey, + p384_key: ec.EllipticCurvePublicKey, + ) -> None: ... + +class Suite: + def __init__(self, kem: KEM, kdf: KDF, aead: AEAD) -> None: ... + def encrypt( + self, + plaintext: Buffer, + public_key: x25519.X25519PublicKey + | ec.EllipticCurvePublicKey + | mlkem.MLKEM768PublicKey + | mlkem.MLKEM1024PublicKey + | MLKEM768X25519PublicKey + | MLKEM1024P384PublicKey, + info: Buffer | None = None, + ) -> bytes: ... + def decrypt( + self, + ciphertext: Buffer, + private_key: x25519.X25519PrivateKey + | ec.EllipticCurvePrivateKey + | mlkem.MLKEM768PrivateKey + | mlkem.MLKEM1024PrivateKey + | MLKEM768X25519PrivateKey + | MLKEM1024P384PrivateKey, + info: Buffer | None = None, + ) -> bytes: ... + +def _encrypt_with_aad( + suite: Suite, + plaintext: Buffer, + public_key: x25519.X25519PublicKey + | ec.EllipticCurvePublicKey + | mlkem.MLKEM768PublicKey + | mlkem.MLKEM1024PublicKey + | MLKEM768X25519PublicKey + | MLKEM1024P384PublicKey, + info: Buffer | None = None, + aad: Buffer | None = None, +) -> bytes: ... +def _decrypt_with_aad( + suite: Suite, + ciphertext: Buffer, + private_key: x25519.X25519PrivateKey + | ec.EllipticCurvePrivateKey + | mlkem.MLKEM768PrivateKey + | mlkem.MLKEM1024PrivateKey + | MLKEM768X25519PrivateKey + | MLKEM1024P384PrivateKey, + info: Buffer | None = None, + aad: Buffer | None = None, +) -> bytes: ... diff --git a/py_modules/cryptography/hazmat/bindings/_rust/openssl/kdf.pyi b/py_modules/cryptography/hazmat/bindings/_rust/openssl/kdf.pyi new file mode 100644 index 0000000..bb37b96 --- /dev/null +++ b/py_modules/cryptography/hazmat/bindings/_rust/openssl/kdf.pyi @@ -0,0 +1,205 @@ +# This file is dual licensed under the terms of the Apache License, Version +# 2.0, and the BSD License. See the LICENSE file in the root of this repository +# for complete details. + +import typing + +from cryptography.hazmat.primitives.hashes import HashAlgorithm +from cryptography.hazmat.primitives.kdf.kbkdf import CounterLocation, Mode +from cryptography.utils import Buffer + +class PBKDF2HMAC: + def __init__( + self, + algorithm: HashAlgorithm, + length: int, + salt: bytes, + iterations: int, + backend: typing.Any = None, + ) -> None: ... + def derive(self, key_material: Buffer) -> bytes: ... + def derive_into(self, key_material: Buffer, buffer: Buffer) -> int: ... + def verify(self, key_material: bytes, expected_key: bytes) -> None: ... + +class Scrypt: + def __init__( + self, + salt: bytes, + length: int, + n: int, + r: int, + p: int, + backend: typing.Any = None, + ) -> None: ... + def derive(self, key_material: Buffer) -> bytes: ... + def derive_into(self, key_material: Buffer, buffer: Buffer) -> int: ... + def verify(self, key_material: bytes, expected_key: bytes) -> None: ... + +class Argon2d: + def __init__( + self, + *, + salt: bytes, + length: int, + iterations: int, + lanes: int, + memory_cost: int, + ad: bytes | None = None, + secret: bytes | None = None, + ) -> None: ... + def derive(self, key_material: bytes) -> bytes: ... + def derive_into(self, key_material: bytes, buffer: Buffer) -> int: ... + def verify(self, key_material: bytes, expected_key: bytes) -> None: ... + def derive_phc_encoded(self, key_material: bytes) -> str: ... + @classmethod + def verify_phc_encoded( + cls, key_material: bytes, phc_encoded: str, secret: bytes | None = None + ) -> None: ... + +class Argon2i: + def __init__( + self, + *, + salt: bytes, + length: int, + iterations: int, + lanes: int, + memory_cost: int, + ad: bytes | None = None, + secret: bytes | None = None, + ) -> None: ... + def derive(self, key_material: bytes) -> bytes: ... + def derive_into(self, key_material: bytes, buffer: Buffer) -> int: ... + def verify(self, key_material: bytes, expected_key: bytes) -> None: ... + def derive_phc_encoded(self, key_material: bytes) -> str: ... + @classmethod + def verify_phc_encoded( + cls, key_material: bytes, phc_encoded: str, secret: bytes | None = None + ) -> None: ... + +class Argon2id: + def __init__( + self, + *, + salt: bytes, + length: int, + iterations: int, + lanes: int, + memory_cost: int, + ad: bytes | None = None, + secret: bytes | None = None, + ) -> None: ... + def derive(self, key_material: bytes) -> bytes: ... + def derive_into(self, key_material: bytes, buffer: Buffer) -> int: ... + def verify(self, key_material: bytes, expected_key: bytes) -> None: ... + def derive_phc_encoded(self, key_material: bytes) -> str: ... + @classmethod + def verify_phc_encoded( + cls, key_material: bytes, phc_encoded: str, secret: bytes | None = None + ) -> None: ... + +class HKDF: + def __init__( + self, + algorithm: HashAlgorithm, + length: int, + salt: bytes | None, + info: bytes | None, + backend: typing.Any = None, + ): ... + @staticmethod + def extract( + algorithm: HashAlgorithm, salt: bytes | None, key_material: Buffer + ) -> bytes: ... + def derive(self, key_material: Buffer) -> bytes: ... + def derive_into(self, key_material: Buffer, buffer: Buffer) -> int: ... + def verify(self, key_material: bytes, expected_key: bytes) -> None: ... + +class HKDFExpand: + def __init__( + self, + algorithm: HashAlgorithm, + length: int, + info: bytes | None, + backend: typing.Any = None, + ): ... + def derive(self, key_material: Buffer) -> bytes: ... + def derive_into(self, key_material: Buffer, buffer: Buffer) -> int: ... + def verify(self, key_material: bytes, expected_key: bytes) -> None: ... + +class X963KDF: + def __init__( + self, + algorithm: HashAlgorithm, + length: int, + sharedinfo: bytes | None, + backend: typing.Any = None, + ) -> None: ... + def derive(self, key_material: Buffer) -> bytes: ... + def derive_into(self, key_material: Buffer, buffer: Buffer) -> int: ... + def verify(self, key_material: bytes, expected_key: bytes) -> None: ... + +class ConcatKDFHash: + def __init__( + self, + algorithm: HashAlgorithm, + length: int, + otherinfo: bytes | None, + backend: typing.Any = None, + ) -> None: ... + def derive(self, key_material: Buffer) -> bytes: ... + def derive_into(self, key_material: Buffer, buffer: Buffer) -> int: ... + def verify(self, key_material: bytes, expected_key: bytes) -> None: ... + +class ConcatKDFHMAC: + def __init__( + self, + algorithm: HashAlgorithm, + length: int, + salt: bytes | None, + otherinfo: bytes | None, + backend: typing.Any = None, + ) -> None: ... + def derive(self, key_material: Buffer) -> bytes: ... + def derive_into(self, key_material: Buffer, buffer: Buffer) -> int: ... + def verify(self, key_material: bytes, expected_key: bytes) -> None: ... + +class KBKDFHMAC: + def __init__( + self, + algorithm: HashAlgorithm, + mode: Mode, + length: int, + rlen: int, + llen: int | None, + location: CounterLocation, + label: bytes | None, + context: bytes | None, + fixed: bytes | None, + backend: typing.Any = None, + *, + break_location: int | None = None, + ) -> None: ... + def derive(self, key_material: Buffer) -> bytes: ... + def derive_into(self, key_material: Buffer, buffer: Buffer) -> int: ... + def verify(self, key_material: bytes, expected_key: bytes) -> None: ... + +class KBKDFCMAC: + def __init__( + self, + algorithm: typing.Any, + mode: Mode, + length: int, + rlen: int, + llen: int | None, + location: CounterLocation, + label: bytes | None, + context: bytes | None, + fixed: bytes | None, + backend: typing.Any = None, + *, + break_location: int | None = None, + ) -> None: ... + def derive(self, key_material: Buffer) -> bytes: ... + def derive_into(self, key_material: Buffer, buffer: Buffer) -> int: ... + def verify(self, key_material: bytes, expected_key: bytes) -> None: ... diff --git a/py_modules/cryptography/hazmat/bindings/_rust/openssl/keys.pyi b/py_modules/cryptography/hazmat/bindings/_rust/openssl/keys.pyi new file mode 100644 index 0000000..404057e --- /dev/null +++ b/py_modules/cryptography/hazmat/bindings/_rust/openssl/keys.pyi @@ -0,0 +1,34 @@ +# This file is dual licensed under the terms of the Apache License, Version +# 2.0, and the BSD License. See the LICENSE file in the root of this repository +# for complete details. + +import typing + +from cryptography.hazmat.primitives.asymmetric.types import ( + PrivateKeyTypes, + PublicKeyTypes, +) +from cryptography.utils import Buffer + +def load_der_private_key( + data: Buffer, + password: bytes | None, + backend: typing.Any = None, + *, + unsafe_skip_rsa_key_validation: bool = False, +) -> PrivateKeyTypes: ... +def load_pem_private_key( + data: Buffer, + password: bytes | None, + backend: typing.Any = None, + *, + unsafe_skip_rsa_key_validation: bool = False, +) -> PrivateKeyTypes: ... +def load_der_public_key( + data: bytes, + backend: typing.Any = None, +) -> PublicKeyTypes: ... +def load_pem_public_key( + data: bytes, + backend: typing.Any = None, +) -> PublicKeyTypes: ... diff --git a/py_modules/cryptography/hazmat/bindings/_rust/openssl/keywrap.pyi b/py_modules/cryptography/hazmat/bindings/_rust/openssl/keywrap.pyi new file mode 100644 index 0000000..dd445ea --- /dev/null +++ b/py_modules/cryptography/hazmat/bindings/_rust/openssl/keywrap.pyi @@ -0,0 +1,28 @@ +# This file is dual licensed under the terms of the Apache License, Version +# 2.0, and the BSD License. See the LICENSE file in the root of this repository +# for complete details. + +import typing + +from cryptography.utils import Buffer + +def aes_key_wrap( + wrapping_key: Buffer, + key_to_wrap: Buffer, + backend: typing.Any = None, +) -> bytes: ... +def aes_key_unwrap( + wrapping_key: Buffer, + wrapped_key: Buffer, + backend: typing.Any = None, +) -> bytes: ... +def aes_key_wrap_with_padding( + wrapping_key: Buffer, + key_to_wrap: Buffer, + backend: typing.Any = None, +) -> bytes: ... +def aes_key_unwrap_with_padding( + wrapping_key: Buffer, + wrapped_key: Buffer, + backend: typing.Any = None, +) -> bytes: ... diff --git a/py_modules/cryptography/hazmat/bindings/_rust/openssl/mldsa.pyi b/py_modules/cryptography/hazmat/bindings/_rust/openssl/mldsa.pyi new file mode 100644 index 0000000..cea9cb1 --- /dev/null +++ b/py_modules/cryptography/hazmat/bindings/_rust/openssl/mldsa.pyi @@ -0,0 +1,35 @@ +# This file is dual licensed under the terms of the Apache License, Version +# 2.0, and the BSD License. See the LICENSE file in the root of this repository +# for complete details. + +from cryptography.hazmat.primitives.asymmetric import mldsa +from cryptography.utils import Buffer + +class MLDSA44PrivateKey: ... +class MLDSA44PublicKey: ... +class MLDSA65PrivateKey: ... +class MLDSA65PublicKey: ... +class MLDSA87PrivateKey: ... +class MLDSA87PublicKey: ... + +class MLDSAMuHasher: + def __init__( + self, + public_key: mldsa.MLDSA44PublicKey + | mldsa.MLDSA65PublicKey + | mldsa.MLDSA87PublicKey, + context: Buffer | None = None, + ) -> None: ... + def update(self, data: Buffer) -> None: ... + def copy(self) -> MLDSAMuHasher: ... + def finalize(self) -> bytes: ... + +def generate_mldsa44_key() -> mldsa.MLDSA44PrivateKey: ... +def from_mldsa44_public_bytes(data: bytes) -> mldsa.MLDSA44PublicKey: ... +def from_mldsa44_seed_bytes(data: Buffer) -> mldsa.MLDSA44PrivateKey: ... +def generate_mldsa65_key() -> mldsa.MLDSA65PrivateKey: ... +def from_mldsa65_public_bytes(data: bytes) -> mldsa.MLDSA65PublicKey: ... +def from_mldsa65_seed_bytes(data: Buffer) -> mldsa.MLDSA65PrivateKey: ... +def generate_mldsa87_key() -> mldsa.MLDSA87PrivateKey: ... +def from_mldsa87_public_bytes(data: bytes) -> mldsa.MLDSA87PublicKey: ... +def from_mldsa87_seed_bytes(data: Buffer) -> mldsa.MLDSA87PrivateKey: ... diff --git a/py_modules/cryptography/hazmat/bindings/_rust/openssl/mlkem.pyi b/py_modules/cryptography/hazmat/bindings/_rust/openssl/mlkem.pyi new file mode 100644 index 0000000..768a340 --- /dev/null +++ b/py_modules/cryptography/hazmat/bindings/_rust/openssl/mlkem.pyi @@ -0,0 +1,18 @@ +# This file is dual licensed under the terms of the Apache License, Version +# 2.0, and the BSD License. See the LICENSE file in the root of this repository +# for complete details. + +from cryptography.hazmat.primitives.asymmetric import mlkem +from cryptography.utils import Buffer + +class MLKEM768PrivateKey: ... +class MLKEM768PublicKey: ... +class MLKEM1024PrivateKey: ... +class MLKEM1024PublicKey: ... + +def generate_mlkem768_key() -> mlkem.MLKEM768PrivateKey: ... +def from_mlkem768_seed_bytes(data: Buffer) -> mlkem.MLKEM768PrivateKey: ... +def from_mlkem768_public_bytes(data: Buffer) -> mlkem.MLKEM768PublicKey: ... +def generate_mlkem1024_key() -> mlkem.MLKEM1024PrivateKey: ... +def from_mlkem1024_seed_bytes(data: Buffer) -> mlkem.MLKEM1024PrivateKey: ... +def from_mlkem1024_public_bytes(data: Buffer) -> mlkem.MLKEM1024PublicKey: ... diff --git a/py_modules/cryptography/hazmat/bindings/_rust/openssl/poly1305.pyi b/py_modules/cryptography/hazmat/bindings/_rust/openssl/poly1305.pyi new file mode 100644 index 0000000..45a2a39 --- /dev/null +++ b/py_modules/cryptography/hazmat/bindings/_rust/openssl/poly1305.pyi @@ -0,0 +1,15 @@ +# This file is dual licensed under the terms of the Apache License, Version +# 2.0, and the BSD License. See the LICENSE file in the root of this repository +# for complete details. + +from cryptography.utils import Buffer + +class Poly1305: + def __init__(self, key: Buffer) -> None: ... + @staticmethod + def generate_tag(key: Buffer, data: Buffer) -> bytes: ... + @staticmethod + def verify_tag(key: Buffer, data: Buffer, tag: bytes) -> None: ... + def update(self, data: Buffer) -> None: ... + def finalize(self) -> bytes: ... + def verify(self, tag: bytes) -> None: ... diff --git a/py_modules/cryptography/hazmat/bindings/_rust/openssl/rsa.pyi b/py_modules/cryptography/hazmat/bindings/_rust/openssl/rsa.pyi new file mode 100644 index 0000000..ef7752d --- /dev/null +++ b/py_modules/cryptography/hazmat/bindings/_rust/openssl/rsa.pyi @@ -0,0 +1,55 @@ +# This file is dual licensed under the terms of the Apache License, Version +# 2.0, and the BSD License. See the LICENSE file in the root of this repository +# for complete details. + +import typing + +from cryptography.hazmat.primitives.asymmetric import rsa + +class RSAPrivateKey: ... +class RSAPublicKey: ... + +class RSAPrivateNumbers: + def __init__( + self, + p: int, + q: int, + d: int, + dmp1: int, + dmq1: int, + iqmp: int, + public_numbers: RSAPublicNumbers, + ) -> None: ... + @property + def p(self) -> int: ... + @property + def q(self) -> int: ... + @property + def d(self) -> int: ... + @property + def dmp1(self) -> int: ... + @property + def dmq1(self) -> int: ... + @property + def iqmp(self) -> int: ... + @property + def public_numbers(self) -> RSAPublicNumbers: ... + def private_key( + self, + backend: typing.Any = None, + *, + unsafe_skip_rsa_key_validation: bool = False, + ) -> rsa.RSAPrivateKey: ... + +class RSAPublicNumbers: + def __init__(self, e: int, n: int) -> None: ... + @property + def n(self) -> int: ... + @property + def e(self) -> int: ... + def public_key(self, backend: typing.Any = None) -> rsa.RSAPublicKey: ... + +def generate_private_key( + public_exponent: int, + key_size: int, +) -> rsa.RSAPrivateKey: ... diff --git a/py_modules/cryptography/hazmat/bindings/_rust/openssl/x25519.pyi b/py_modules/cryptography/hazmat/bindings/_rust/openssl/x25519.pyi new file mode 100644 index 0000000..38d2add --- /dev/null +++ b/py_modules/cryptography/hazmat/bindings/_rust/openssl/x25519.pyi @@ -0,0 +1,13 @@ +# This file is dual licensed under the terms of the Apache License, Version +# 2.0, and the BSD License. See the LICENSE file in the root of this repository +# for complete details. + +from cryptography.hazmat.primitives.asymmetric import x25519 +from cryptography.utils import Buffer + +class X25519PrivateKey: ... +class X25519PublicKey: ... + +def generate_key() -> x25519.X25519PrivateKey: ... +def from_private_bytes(data: Buffer) -> x25519.X25519PrivateKey: ... +def from_public_bytes(data: bytes) -> x25519.X25519PublicKey: ... diff --git a/py_modules/cryptography/hazmat/bindings/_rust/openssl/x448.pyi b/py_modules/cryptography/hazmat/bindings/_rust/openssl/x448.pyi new file mode 100644 index 0000000..3ac0980 --- /dev/null +++ b/py_modules/cryptography/hazmat/bindings/_rust/openssl/x448.pyi @@ -0,0 +1,13 @@ +# This file is dual licensed under the terms of the Apache License, Version +# 2.0, and the BSD License. See the LICENSE file in the root of this repository +# for complete details. + +from cryptography.hazmat.primitives.asymmetric import x448 +from cryptography.utils import Buffer + +class X448PrivateKey: ... +class X448PublicKey: ... + +def generate_key() -> x448.X448PrivateKey: ... +def from_private_bytes(data: Buffer) -> x448.X448PrivateKey: ... +def from_public_bytes(data: bytes) -> x448.X448PublicKey: ... diff --git a/py_modules/cryptography/hazmat/bindings/_rust/pkcs12.pyi b/py_modules/cryptography/hazmat/bindings/_rust/pkcs12.pyi new file mode 100644 index 0000000..b25becb --- /dev/null +++ b/py_modules/cryptography/hazmat/bindings/_rust/pkcs12.pyi @@ -0,0 +1,52 @@ +# This file is dual licensed under the terms of the Apache License, Version +# 2.0, and the BSD License. See the LICENSE file in the root of this repository +# for complete details. + +import typing +from collections.abc import Iterable + +from cryptography import x509 +from cryptography.hazmat.primitives.asymmetric.types import PrivateKeyTypes +from cryptography.hazmat.primitives.serialization import ( + KeySerializationEncryption, +) +from cryptography.hazmat.primitives.serialization.pkcs12 import ( + PKCS12KeyAndCertificates, + PKCS12PrivateKeyTypes, +) +from cryptography.utils import Buffer + +class PKCS12Certificate: + def __init__( + self, cert: x509.Certificate, friendly_name: bytes | None + ) -> None: ... + @property + def friendly_name(self) -> bytes | None: ... + @property + def certificate(self) -> x509.Certificate: ... + +def load_key_and_certificates( + data: Buffer, + password: Buffer | None, + backend: typing.Any = None, +) -> tuple[ + PrivateKeyTypes | None, + x509.Certificate | None, + list[x509.Certificate], +]: ... +def load_pkcs12( + data: bytes, + password: bytes | None, + backend: typing.Any = None, +) -> PKCS12KeyAndCertificates: ... +def serialize_java_truststore( + certs: Iterable[PKCS12Certificate], + encryption_algorithm: KeySerializationEncryption, +) -> bytes: ... +def serialize_key_and_certificates( + name: bytes | None, + key: PKCS12PrivateKeyTypes | None, + cert: x509.Certificate | None, + cas: Iterable[x509.Certificate | PKCS12Certificate] | None, + encryption_algorithm: KeySerializationEncryption, +) -> bytes: ... diff --git a/py_modules/cryptography/hazmat/bindings/_rust/pkcs7.pyi b/py_modules/cryptography/hazmat/bindings/_rust/pkcs7.pyi new file mode 100644 index 0000000..358b135 --- /dev/null +++ b/py_modules/cryptography/hazmat/bindings/_rust/pkcs7.pyi @@ -0,0 +1,50 @@ +# This file is dual licensed under the terms of the Apache License, Version +# 2.0, and the BSD License. See the LICENSE file in the root of this repository +# for complete details. + +from collections.abc import Iterable + +from cryptography import x509 +from cryptography.hazmat.primitives import serialization +from cryptography.hazmat.primitives.asymmetric import rsa +from cryptography.hazmat.primitives.serialization import pkcs7 + +def serialize_certificates( + certs: list[x509.Certificate], + encoding: serialization.Encoding, +) -> bytes: ... +def encrypt_and_serialize( + builder: pkcs7.PKCS7EnvelopeBuilder, + content_encryption_algorithm: pkcs7.ContentEncryptionAlgorithm, + encoding: serialization.Encoding, + options: Iterable[pkcs7.PKCS7Options], +) -> bytes: ... +def sign_and_serialize( + builder: pkcs7.PKCS7SignatureBuilder, + encoding: serialization.Encoding, + options: Iterable[pkcs7.PKCS7Options], +) -> bytes: ... +def decrypt_der( + data: bytes, + certificate: x509.Certificate, + private_key: rsa.RSAPrivateKey, + options: Iterable[pkcs7.PKCS7Options], +) -> bytes: ... +def decrypt_pem( + data: bytes, + certificate: x509.Certificate, + private_key: rsa.RSAPrivateKey, + options: Iterable[pkcs7.PKCS7Options], +) -> bytes: ... +def decrypt_smime( + data: bytes, + certificate: x509.Certificate, + private_key: rsa.RSAPrivateKey, + options: Iterable[pkcs7.PKCS7Options], +) -> bytes: ... +def load_pem_pkcs7_certificates( + data: bytes, +) -> list[x509.Certificate]: ... +def load_der_pkcs7_certificates( + data: bytes, +) -> list[x509.Certificate]: ... diff --git a/py_modules/cryptography/hazmat/bindings/_rust/test_support.pyi b/py_modules/cryptography/hazmat/bindings/_rust/test_support.pyi new file mode 100644 index 0000000..c6c6d0b --- /dev/null +++ b/py_modules/cryptography/hazmat/bindings/_rust/test_support.pyi @@ -0,0 +1,23 @@ +# This file is dual licensed under the terms of the Apache License, Version +# 2.0, and the BSD License. See the LICENSE file in the root of this repository +# for complete details. + +from cryptography import x509 +from cryptography.hazmat.primitives import serialization +from cryptography.hazmat.primitives.serialization import pkcs7 +from cryptography.utils import Buffer + +class TestCertificate: + not_after_tag: int + not_before_tag: int + issuer_value_tags: list[int] + subject_value_tags: list[int] + +def test_parse_certificate(data: bytes) -> TestCertificate: ... +def pkcs7_verify( + encoding: serialization.Encoding, + sig: bytes, + msg: Buffer | None, + certs: list[x509.Certificate], + options: list[pkcs7.PKCS7Options], +) -> None: ... diff --git a/py_modules/cryptography/hazmat/bindings/_rust/x509.pyi b/py_modules/cryptography/hazmat/bindings/_rust/x509.pyi new file mode 100644 index 0000000..4438cc7 --- /dev/null +++ b/py_modules/cryptography/hazmat/bindings/_rust/x509.pyi @@ -0,0 +1,313 @@ +# This file is dual licensed under the terms of the Apache License, Version +# 2.0, and the BSD License. See the LICENSE file in the root of this repository +# for complete details. + +import datetime +import typing +from collections.abc import Iterator + +from cryptography import x509 +from cryptography.hazmat.primitives import hashes, serialization +from cryptography.hazmat.primitives.asymmetric.ec import ECDSA +from cryptography.hazmat.primitives.asymmetric.padding import PSS, PKCS1v15 +from cryptography.hazmat.primitives.asymmetric.types import ( + CertificateIssuerPublicKeyTypes, + CertificatePublicKeyTypes, + PrivateKeyTypes, +) +from cryptography.x509 import certificate_transparency + +def load_pem_x509_certificate( + data: bytes, backend: typing.Any = None +) -> x509.Certificate: ... +def load_der_x509_certificate( + data: bytes, backend: typing.Any = None +) -> x509.Certificate: ... +def load_pem_x509_certificates( + data: bytes, +) -> list[x509.Certificate]: ... +def load_pem_x509_crl( + data: bytes, backend: typing.Any = None +) -> x509.CertificateRevocationList: ... +def load_der_x509_crl( + data: bytes, backend: typing.Any = None +) -> x509.CertificateRevocationList: ... +def load_pem_x509_csr( + data: bytes, backend: typing.Any = None +) -> x509.CertificateSigningRequest: ... +def load_der_x509_csr( + data: bytes, backend: typing.Any = None +) -> x509.CertificateSigningRequest: ... +def encode_name_bytes(name: x509.Name) -> bytes: ... +def parse_name_bytes(data: bytes) -> x509.Name: ... +def encode_extension_value(extension: x509.ExtensionType) -> bytes: ... +def create_x509_certificate( + builder: x509.CertificateBuilder, + private_key: PrivateKeyTypes | None, + hash_algorithm: hashes.HashAlgorithm | None, + rsa_padding: PKCS1v15 | PSS | None, + ecdsa_deterministic: bool | None, +) -> x509.Certificate: ... +def create_x509_csr( + builder: x509.CertificateSigningRequestBuilder, + private_key: PrivateKeyTypes, + hash_algorithm: hashes.HashAlgorithm | None, + rsa_padding: PKCS1v15 | PSS | None, + ecdsa_deterministic: bool | None, +) -> x509.CertificateSigningRequest: ... +def create_revoked_certificate( + builder: x509.RevokedCertificateBuilder, +) -> x509.RevokedCertificate: ... +def create_x509_crl( + builder: x509.CertificateRevocationListBuilder, + private_key: PrivateKeyTypes, + hash_algorithm: hashes.HashAlgorithm | None, + rsa_padding: PKCS1v15 | PSS | None, + ecdsa_deterministic: bool | None, +) -> x509.CertificateRevocationList: ... + +class Sct: + @property + def version(self) -> certificate_transparency.Version: ... + @property + def log_id(self) -> bytes: ... + @property + def timestamp(self) -> datetime.datetime: ... + @property + def entry_type(self) -> certificate_transparency.LogEntryType: ... + @property + def signature_hash_algorithm(self) -> hashes.HashAlgorithm: ... + @property + def signature_algorithm( + self, + ) -> certificate_transparency.SignatureAlgorithm: ... + @property + def signature(self) -> bytes: ... + @property + def extension_bytes(self) -> bytes: ... + +class Certificate: + def fingerprint(self, algorithm: hashes.HashAlgorithm) -> bytes: ... + @property + def serial_number(self) -> int: ... + @property + def version(self) -> x509.Version: ... + def public_key(self) -> CertificatePublicKeyTypes: ... + @property + def public_key_algorithm_oid(self) -> x509.ObjectIdentifier: ... + @property + def not_valid_before(self) -> datetime.datetime: ... + @property + def not_valid_before_utc(self) -> datetime.datetime: ... + @property + def not_valid_after(self) -> datetime.datetime: ... + @property + def not_valid_after_utc(self) -> datetime.datetime: ... + @property + def issuer(self) -> x509.Name: ... + @property + def subject(self) -> x509.Name: ... + @property + def signature_hash_algorithm( + self, + ) -> hashes.HashAlgorithm | None: ... + @property + def signature_algorithm_oid(self) -> x509.ObjectIdentifier: ... + @property + def signature_algorithm_parameters( + self, + ) -> PSS | PKCS1v15 | ECDSA | None: ... + @property + def extensions(self) -> x509.Extensions: ... + @property + def signature(self) -> bytes: ... + @property + def tbs_certificate_bytes(self) -> bytes: ... + @property + def tbs_precertificate_bytes(self) -> bytes: ... + def __eq__(self, other: object) -> bool: ... + def __hash__(self) -> int: ... + def public_bytes(self, encoding: serialization.Encoding) -> bytes: ... + def verify_directly_issued_by(self, issuer: Certificate) -> None: ... + +class RevokedCertificate: + @property + def serial_number(self) -> int: ... + @property + def revocation_date(self) -> datetime.datetime: ... + @property + def revocation_date_utc(self) -> datetime.datetime: ... + @property + def extensions(self) -> x509.Extensions: ... + +class CertificateRevocationList: + def public_bytes(self, encoding: serialization.Encoding) -> bytes: ... + def fingerprint(self, algorithm: hashes.HashAlgorithm) -> bytes: ... + def get_revoked_certificate_by_serial_number( + self, serial_number: int + ) -> x509.RevokedCertificate | None: ... + @property + def signature_hash_algorithm( + self, + ) -> hashes.HashAlgorithm | None: ... + @property + def signature_algorithm_oid(self) -> x509.ObjectIdentifier: ... + @property + def signature_algorithm_parameters( + self, + ) -> PSS | PKCS1v15 | ECDSA | None: ... + @property + def issuer(self) -> x509.Name: ... + @property + def next_update(self) -> datetime.datetime | None: ... + @property + def next_update_utc(self) -> datetime.datetime | None: ... + @property + def last_update(self) -> datetime.datetime: ... + @property + def last_update_utc(self) -> datetime.datetime: ... + @property + def extensions(self) -> x509.Extensions: ... + @property + def signature(self) -> bytes: ... + @property + def tbs_certlist_bytes(self) -> bytes: ... + def __eq__(self, other: object) -> bool: ... + def __len__(self) -> int: ... + @typing.overload + def __getitem__(self, idx: int) -> x509.RevokedCertificate: ... + @typing.overload + def __getitem__(self, idx: slice) -> list[x509.RevokedCertificate]: ... + def __iter__(self) -> Iterator[x509.RevokedCertificate]: ... + def is_signature_valid( + self, public_key: CertificateIssuerPublicKeyTypes + ) -> bool: ... + +class CertificateSigningRequest: + def __eq__(self, other: object) -> bool: ... + def __hash__(self) -> int: ... + def public_key(self) -> CertificatePublicKeyTypes: ... + @property + def subject(self) -> x509.Name: ... + @property + def signature_hash_algorithm( + self, + ) -> hashes.HashAlgorithm | None: ... + @property + def signature_algorithm_oid(self) -> x509.ObjectIdentifier: ... + @property + def signature_algorithm_parameters( + self, + ) -> PSS | PKCS1v15 | ECDSA | None: ... + @property + def extensions(self) -> x509.Extensions: ... + @property + def attributes(self) -> x509.Attributes: ... + def public_bytes(self, encoding: serialization.Encoding) -> bytes: ... + @property + def signature(self) -> bytes: ... + @property + def tbs_certrequest_bytes(self) -> bytes: ... + @property + def is_signature_valid(self) -> bool: ... + +class PolicyBuilder: + def time(self, time: datetime.datetime) -> PolicyBuilder: ... + def store(self, store: Store) -> PolicyBuilder: ... + def max_chain_depth(self, max_chain_depth: int) -> PolicyBuilder: ... + def extension_policies( + self, *, ca_policy: ExtensionPolicy, ee_policy: ExtensionPolicy + ) -> PolicyBuilder: ... + def build_client_verifier(self) -> ClientVerifier: ... + def build_server_verifier( + self, subject: x509.verification.Subject + ) -> ServerVerifier: ... + +class Policy: + @property + def max_chain_depth(self) -> int: ... + @property + def subject(self) -> x509.verification.Subject | None: ... + @property + def validation_time(self) -> datetime.datetime: ... + @property + def extended_key_usage(self) -> x509.ObjectIdentifier: ... + @property + def minimum_rsa_modulus(self) -> int: ... + +class Criticality: + CRITICAL: Criticality + AGNOSTIC: Criticality + NON_CRITICAL: Criticality + +T = typing.TypeVar("T", contravariant=True, bound=x509.ExtensionType) + +MaybeExtensionValidatorCallback = typing.Callable[ + [ + Policy, + x509.Certificate, + T | None, + ], + None, +] + +PresentExtensionValidatorCallback = typing.Callable[ + [Policy, x509.Certificate, T], + None, +] + +class ExtensionPolicy: + @staticmethod + def permit_all() -> ExtensionPolicy: ... + @staticmethod + def webpki_defaults_ca() -> ExtensionPolicy: ... + @staticmethod + def webpki_defaults_ee() -> ExtensionPolicy: ... + def require_not_present( + self, extension_type: type[x509.ExtensionType] + ) -> ExtensionPolicy: ... + def may_be_present( + self, + extension_type: type[T], + criticality: Criticality, + validator: MaybeExtensionValidatorCallback[T] | None, + ) -> ExtensionPolicy: ... + def require_present( + self, + extension_type: type[T], + criticality: Criticality, + validator: PresentExtensionValidatorCallback[T] | None, + ) -> ExtensionPolicy: ... + +class VerifiedClient: + @property + def subjects(self) -> list[x509.GeneralName] | None: ... + @property + def chain(self) -> list[x509.Certificate]: ... + +class ClientVerifier: + @property + def policy(self) -> Policy: ... + @property + def store(self) -> Store: ... + def verify( + self, + leaf: x509.Certificate, + intermediates: list[x509.Certificate], + ) -> VerifiedClient: ... + +class ServerVerifier: + @property + def policy(self) -> Policy: ... + @property + def store(self) -> Store: ... + def verify( + self, + leaf: x509.Certificate, + intermediates: list[x509.Certificate], + ) -> list[x509.Certificate]: ... + +class Store: + def __init__(self, certs: list[x509.Certificate]) -> None: ... + +class VerificationError(Exception): ... diff --git a/py_modules/cryptography/hazmat/bindings/openssl/__init__.py b/py_modules/cryptography/hazmat/bindings/openssl/__init__.py new file mode 100644 index 0000000..41d7318 --- /dev/null +++ b/py_modules/cryptography/hazmat/bindings/openssl/__init__.py @@ -0,0 +1,5 @@ +# This file is dual licensed under the terms of the Apache License, Version +# 2.0, and the BSD License. See the LICENSE file in the root of this repository +# for complete details. + +from __future__ import annotations diff --git a/py_modules/cryptography/hazmat/bindings/openssl/_conditional.py b/py_modules/cryptography/hazmat/bindings/openssl/_conditional.py new file mode 100644 index 0000000..1e447a5 --- /dev/null +++ b/py_modules/cryptography/hazmat/bindings/openssl/_conditional.py @@ -0,0 +1,199 @@ +# This file is dual licensed under the terms of the Apache License, Version +# 2.0, and the BSD License. See the LICENSE file in the root of this repository +# for complete details. + +from __future__ import annotations + + +def cryptography_has_set_cert_cb() -> list[str]: + return [ + "SSL_CTX_set_cert_cb", + "SSL_set_cert_cb", + ] + + +def cryptography_has_ssl_st() -> list[str]: + return [ + "SSL_ST_BEFORE", + "SSL_ST_OK", + "SSL_ST_INIT", + "SSL_ST_RENEGOTIATE", + ] + + +def cryptography_has_tls_st() -> list[str]: + return [ + "TLS_ST_BEFORE", + "TLS_ST_OK", + ] + + +def cryptography_has_ssl_sigalgs() -> list[str]: + return [ + "SSL_CTX_set1_sigalgs_list", + ] + + +def cryptography_has_psk() -> list[str]: + return [ + "SSL_CTX_use_psk_identity_hint", + "SSL_CTX_set_psk_server_callback", + "SSL_CTX_set_psk_client_callback", + ] + + +def cryptography_has_psk_tlsv13() -> list[str]: + return [ + "SSL_CTX_set_psk_find_session_callback", + "SSL_CTX_set_psk_use_session_callback", + "Cryptography_SSL_SESSION_new", + "SSL_CIPHER_find", + "SSL_SESSION_set1_master_key", + "SSL_SESSION_set_cipher", + "SSL_SESSION_set_protocol_version", + ] + + +def cryptography_has_custom_ext() -> list[str]: + return [ + "SSL_CTX_add_client_custom_ext", + "SSL_CTX_add_server_custom_ext", + "SSL_extension_supported", + ] + + +def cryptography_has_tlsv13_functions() -> list[str]: + return [ + "SSL_CTX_set_ciphersuites", + ] + + +def cryptography_has_tlsv13_hs_functions() -> list[str]: + return [ + "SSL_VERIFY_POST_HANDSHAKE", + "SSL_verify_client_post_handshake", + "SSL_CTX_set_post_handshake_auth", + "SSL_set_post_handshake_auth", + "SSL_SESSION_get_max_early_data", + "SSL_write_early_data", + "SSL_read_early_data", + "SSL_CTX_set_max_early_data", + ] + + +def cryptography_has_ssl_verify_client_post_handshake() -> list[str]: + return [ + "SSL_verify_client_post_handshake", + ] + + +def cryptography_has_engine() -> list[str]: + return [ + "ENGINE_by_id", + "ENGINE_init", + "ENGINE_finish", + "ENGINE_get_default_RAND", + "ENGINE_set_default_RAND", + "ENGINE_unregister_RAND", + "ENGINE_ctrl_cmd", + "ENGINE_free", + "ENGINE_get_name", + "ENGINE_ctrl_cmd_string", + "ENGINE_load_builtin_engines", + "ENGINE_load_private_key", + "ENGINE_load_public_key", + "SSL_CTX_set_client_cert_engine", + ] + + +def cryptography_has_verified_chain() -> list[str]: + return [ + "SSL_get0_verified_chain", + ] + + +def cryptography_has_srtp() -> list[str]: + return [ + "SSL_CTX_set_tlsext_use_srtp", + "SSL_set_tlsext_use_srtp", + "SSL_get_selected_srtp_profile", + ] + + +def cryptography_has_dtls_get_data_mtu() -> list[str]: + return [ + "DTLS_get_data_mtu", + ] + + +def cryptography_has_ssl_cookie() -> list[str]: + return [ + "SSL_OP_COOKIE_EXCHANGE", + "DTLS1_COOKIE_LENGTH", + "DTLSv1_listen", + "SSL_CTX_set_cookie_generate_cb", + "SSL_CTX_set_cookie_verify_cb", + ] + + +def cryptography_has_prime_checks() -> list[str]: + return [ + "BN_prime_checks_for_size", + ] + + +def cryptography_has_unexpected_eof_while_reading() -> list[str]: + return ["SSL_R_UNEXPECTED_EOF_WHILE_READING"] + + +def cryptography_has_ssl_op_ignore_unexpected_eof() -> list[str]: + return [ + "SSL_OP_IGNORE_UNEXPECTED_EOF", + ] + + +def cryptography_has_get_extms_support() -> list[str]: + return ["SSL_get_extms_support"] + + +def cryptography_has_ssl_get0_group_name() -> list[str]: + return ["SSL_get0_group_name"] + + +# This is a mapping of +# {condition: function-returning-names-dependent-on-that-condition} so we can +# loop over them and delete unsupported names at runtime. It will be removed +# when cffi supports #if in cdef. We use functions instead of just a dict of +# lists so we can use coverage to measure which are used. +CONDITIONAL_NAMES = { + "Cryptography_HAS_SET_CERT_CB": cryptography_has_set_cert_cb, + "Cryptography_HAS_SSL_ST": cryptography_has_ssl_st, + "Cryptography_HAS_TLS_ST": cryptography_has_tls_st, + "Cryptography_HAS_SIGALGS": cryptography_has_ssl_sigalgs, + "Cryptography_HAS_PSK": cryptography_has_psk, + "Cryptography_HAS_PSK_TLSv1_3": cryptography_has_psk_tlsv13, + "Cryptography_HAS_CUSTOM_EXT": cryptography_has_custom_ext, + "Cryptography_HAS_TLSv1_3_FUNCTIONS": cryptography_has_tlsv13_functions, + "Cryptography_HAS_TLSv1_3_HS_FUNCTIONS": ( + cryptography_has_tlsv13_hs_functions + ), + "Cryptography_HAS_SSL_VERIFY_CLIENT_POST_HANDSHAKE": ( + cryptography_has_ssl_verify_client_post_handshake + ), + "Cryptography_HAS_ENGINE": cryptography_has_engine, + "Cryptography_HAS_VERIFIED_CHAIN": cryptography_has_verified_chain, + "Cryptography_HAS_SRTP": cryptography_has_srtp, + "Cryptography_HAS_DTLS_GET_DATA_MTU": cryptography_has_dtls_get_data_mtu, + "Cryptography_HAS_SSL_COOKIE": cryptography_has_ssl_cookie, + "Cryptography_HAS_PRIME_CHECKS": cryptography_has_prime_checks, + "Cryptography_HAS_UNEXPECTED_EOF_WHILE_READING": ( + cryptography_has_unexpected_eof_while_reading + ), + "Cryptography_HAS_SSL_OP_IGNORE_UNEXPECTED_EOF": ( + cryptography_has_ssl_op_ignore_unexpected_eof + ), + "Cryptography_HAS_GET_EXTMS_SUPPORT": cryptography_has_get_extms_support, + "Cryptography_HAS_SSL_GET0_GROUP_NAME": ( + cryptography_has_ssl_get0_group_name + ), +} diff --git a/py_modules/cryptography/hazmat/bindings/openssl/binding.py b/py_modules/cryptography/hazmat/bindings/openssl/binding.py new file mode 100644 index 0000000..6e13df1 --- /dev/null +++ b/py_modules/cryptography/hazmat/bindings/openssl/binding.py @@ -0,0 +1,107 @@ +# This file is dual licensed under the terms of the Apache License, Version +# 2.0, and the BSD License. See the LICENSE file in the root of this repository +# for complete details. + +from __future__ import annotations + +import threading +import types +import typing +from collections.abc import Callable, Mapping + +import cryptography +from cryptography.exceptions import InternalError +from cryptography.hazmat.bindings._rust import _openssl, openssl +from cryptography.hazmat.bindings.openssl._conditional import CONDITIONAL_NAMES + + +def _openssl_assert(ok: bool) -> None: + if not ok: + errors = openssl.capture_error_stack() + + raise InternalError( + "Unknown OpenSSL error. This error is commonly encountered when " + "another library is not cleaning up the OpenSSL error stack. If " + "you are using cryptography with another library that uses " + "OpenSSL try disabling it before reporting a bug. Otherwise " + "please file an issue at https://github.com/pyca/cryptography/" + "issues with information on how to reproduce " + f"this. ({errors!r})", + errors, + ) + + +def build_conditional_library( + lib: typing.Any, + conditional_names: Mapping[str, Callable[[], list[str]]], +) -> typing.Any: + conditional_lib = types.ModuleType("lib") + conditional_lib._original_lib = lib # type: ignore[attr-defined] + excluded_names = set() + for condition, names_cb in conditional_names.items(): + if not getattr(lib, condition): + excluded_names.update(names_cb()) + + for attr in dir(lib): + if attr not in excluded_names: + setattr(conditional_lib, attr, getattr(lib, attr)) + + return conditional_lib + + +class Binding: + """ + OpenSSL API wrapper. + """ + + lib: typing.ClassVar[typing.Any] = None + ffi: typing.Any = _openssl.ffi + _lib_loaded = False + _init_lock = threading.Lock() + + def __init__(self) -> None: + self._ensure_ffi_initialized() + + @classmethod + def _ensure_ffi_initialized(cls) -> None: + with cls._init_lock: + if not cls._lib_loaded: + cls.lib = build_conditional_library( + _openssl.lib, CONDITIONAL_NAMES + ) + cls._lib_loaded = True + + @classmethod + def init_static_locks(cls) -> None: + cls._ensure_ffi_initialized() + + +def _verify_package_version(version: str) -> None: + # Occasionally we run into situations where the version of the Python + # package does not match the version of the shared object that is loaded. + # This may occur in environments where multiple versions of cryptography + # are installed and available in the python path. To avoid errors cropping + # up later this code checks that the currently imported package and the + # shared object that were loaded have the same version and raise an + # ImportError if they do not + so_package_version = _openssl.ffi.string( + _openssl.lib.CRYPTOGRAPHY_PACKAGE_VERSION + ) + if version.encode("ascii") != so_package_version: + raise ImportError( + "The version of cryptography does not match the loaded " + "shared object. This can happen if you have multiple copies of " + "cryptography installed in your Python path. Please try creating " + "a new virtual environment to resolve this issue. " + f"Loaded python version: {version}, " + f"shared object version: {so_package_version}" + ) + + _openssl_assert( + _openssl.lib.OpenSSL_version_num() == openssl.openssl_version(), + ) + + +_verify_package_version(cryptography.__version__) + +Binding.init_static_locks() diff --git a/py_modules/cryptography/hazmat/decrepit/__init__.py b/py_modules/cryptography/hazmat/decrepit/__init__.py new file mode 100644 index 0000000..41d7318 --- /dev/null +++ b/py_modules/cryptography/hazmat/decrepit/__init__.py @@ -0,0 +1,5 @@ +# This file is dual licensed under the terms of the Apache License, Version +# 2.0, and the BSD License. See the LICENSE file in the root of this repository +# for complete details. + +from __future__ import annotations diff --git a/py_modules/cryptography/hazmat/decrepit/ciphers/__init__.py b/py_modules/cryptography/hazmat/decrepit/ciphers/__init__.py new file mode 100644 index 0000000..41d7318 --- /dev/null +++ b/py_modules/cryptography/hazmat/decrepit/ciphers/__init__.py @@ -0,0 +1,5 @@ +# This file is dual licensed under the terms of the Apache License, Version +# 2.0, and the BSD License. See the LICENSE file in the root of this repository +# for complete details. + +from __future__ import annotations diff --git a/py_modules/cryptography/hazmat/decrepit/ciphers/algorithms.py b/py_modules/cryptography/hazmat/decrepit/ciphers/algorithms.py new file mode 100644 index 0000000..72dad04 --- /dev/null +++ b/py_modules/cryptography/hazmat/decrepit/ciphers/algorithms.py @@ -0,0 +1,145 @@ +# This file is dual licensed under the terms of the Apache License, Version +# 2.0, and the BSD License. See the LICENSE file in the root of this repository +# for complete details. + +from __future__ import annotations + +import warnings + +from cryptography import utils +from cryptography.hazmat.primitives._cipheralgorithm import ( + BlockCipherAlgorithm, + CipherAlgorithm, + _verify_key_size, +) + + +class ARC4(CipherAlgorithm): + name = "RC4" + key_sizes = frozenset([40, 56, 64, 80, 128, 160, 192, 256]) + + def __init__(self, key: bytes): + self.key = _verify_key_size(self, key) + + @property + def key_size(self) -> int: + return len(self.key) * 8 + + +class TripleDES(BlockCipherAlgorithm): + name = "3DES" + block_size = 64 + key_sizes = frozenset([64, 128, 192]) + + def __init__(self, key: bytes): + # Check the key type before the length-based deprecation warnings so + # an invalid key type doesn't trigger a spurious warning. + utils._check_byteslike("key", key) + if len(key) == 8: + warnings.warn( + "Single-key TripleDES (8-byte keys) is deprecated and " + "support will be removed in a future release. Use 24-byte " + "keys instead (e.g., key + key + key).", + utils.DeprecatedIn47, + stacklevel=2, + ) + key = key + key + key + elif len(key) == 16: + warnings.warn( + "Two-key TripleDES (16-byte keys) is deprecated and " + "support will be removed in a future release. Use 24-byte " + "keys instead (e.g., key + key[:8]).", + utils.DeprecatedIn47, + stacklevel=2, + ) + key = key + key[:8] + self.key = _verify_key_size(self, key) + + @property + def key_size(self) -> int: + return len(self.key) * 8 + + +# Not actually supported, marker for tests +class _DES: + key_size = 64 + + +class Blowfish(BlockCipherAlgorithm): + name = "Blowfish" + block_size = 64 + key_sizes = frozenset(range(32, 449, 8)) + + def __init__(self, key: bytes): + self.key = _verify_key_size(self, key) + + @property + def key_size(self) -> int: + return len(self.key) * 8 + + +class CAST5(BlockCipherAlgorithm): + name = "CAST5" + block_size = 64 + key_sizes = frozenset(range(40, 129, 8)) + + def __init__(self, key: bytes): + self.key = _verify_key_size(self, key) + + @property + def key_size(self) -> int: + return len(self.key) * 8 + + +class SEED(BlockCipherAlgorithm): + name = "SEED" + block_size = 128 + key_sizes = frozenset([128]) + + def __init__(self, key: bytes): + self.key = _verify_key_size(self, key) + + @property + def key_size(self) -> int: + return len(self.key) * 8 + + +class IDEA(BlockCipherAlgorithm): + name = "IDEA" + block_size = 64 + key_sizes = frozenset([128]) + + def __init__(self, key: bytes): + self.key = _verify_key_size(self, key) + + @property + def key_size(self) -> int: + return len(self.key) * 8 + + +class Camellia(BlockCipherAlgorithm): + name = "camellia" + block_size = 128 + key_sizes = frozenset([128, 192, 256]) + + def __init__(self, key: bytes): + self.key = _verify_key_size(self, key) + + @property + def key_size(self) -> int: + return len(self.key) * 8 + + +# This class only allows RC2 with a 128-bit key. No support for +# effective key bits or other key sizes is provided. +class RC2(BlockCipherAlgorithm): + name = "RC2" + block_size = 64 + key_sizes = frozenset([128]) + + def __init__(self, key: bytes): + self.key = _verify_key_size(self, key) + + @property + def key_size(self) -> int: + return len(self.key) * 8 diff --git a/py_modules/cryptography/hazmat/decrepit/ciphers/modes.py b/py_modules/cryptography/hazmat/decrepit/ciphers/modes.py new file mode 100644 index 0000000..1786bb0 --- /dev/null +++ b/py_modules/cryptography/hazmat/decrepit/ciphers/modes.py @@ -0,0 +1,53 @@ +# This file is dual licensed under the terms of the Apache License, Version +# 2.0, and the BSD License. See the LICENSE file in the root of this repository +# for complete details. + +from __future__ import annotations + +from cryptography import utils +from cryptography.hazmat.primitives._modes import ( + ModeWithInitializationVector, + _check_iv_and_key_length, +) + + +class OFB(ModeWithInitializationVector): + name = "OFB" + + def __init__(self, initialization_vector: utils.Buffer): + utils._check_byteslike("initialization_vector", initialization_vector) + self._initialization_vector = initialization_vector + + @property + def initialization_vector(self) -> utils.Buffer: + return self._initialization_vector + + validate_for_algorithm = _check_iv_and_key_length + + +class CFB(ModeWithInitializationVector): + name = "CFB" + + def __init__(self, initialization_vector: utils.Buffer): + utils._check_byteslike("initialization_vector", initialization_vector) + self._initialization_vector = initialization_vector + + @property + def initialization_vector(self) -> utils.Buffer: + return self._initialization_vector + + validate_for_algorithm = _check_iv_and_key_length + + +class CFB8(ModeWithInitializationVector): + name = "CFB8" + + def __init__(self, initialization_vector: utils.Buffer): + utils._check_byteslike("initialization_vector", initialization_vector) + self._initialization_vector = initialization_vector + + @property + def initialization_vector(self) -> utils.Buffer: + return self._initialization_vector + + validate_for_algorithm = _check_iv_and_key_length diff --git a/py_modules/cryptography/hazmat/primitives/__init__.py b/py_modules/cryptography/hazmat/primitives/__init__.py new file mode 100644 index 0000000..41d7318 --- /dev/null +++ b/py_modules/cryptography/hazmat/primitives/__init__.py @@ -0,0 +1,5 @@ +# This file is dual licensed under the terms of the Apache License, Version +# 2.0, and the BSD License. See the LICENSE file in the root of this repository +# for complete details. + +from __future__ import annotations diff --git a/py_modules/cryptography/hazmat/primitives/_asymmetric.py b/py_modules/cryptography/hazmat/primitives/_asymmetric.py new file mode 100644 index 0000000..ea55ffd --- /dev/null +++ b/py_modules/cryptography/hazmat/primitives/_asymmetric.py @@ -0,0 +1,19 @@ +# This file is dual licensed under the terms of the Apache License, Version +# 2.0, and the BSD License. See the LICENSE file in the root of this repository +# for complete details. + +from __future__ import annotations + +import abc + +# This exists to break an import cycle. It is normally accessible from the +# asymmetric padding module. + + +class AsymmetricPadding(metaclass=abc.ABCMeta): + @property + @abc.abstractmethod + def name(self) -> str: + """ + A string naming this padding (e.g. "PSS", "PKCS1"). + """ diff --git a/py_modules/cryptography/hazmat/primitives/_cipheralgorithm.py b/py_modules/cryptography/hazmat/primitives/_cipheralgorithm.py new file mode 100644 index 0000000..305a9fd --- /dev/null +++ b/py_modules/cryptography/hazmat/primitives/_cipheralgorithm.py @@ -0,0 +1,60 @@ +# This file is dual licensed under the terms of the Apache License, Version +# 2.0, and the BSD License. See the LICENSE file in the root of this repository +# for complete details. + +from __future__ import annotations + +import abc + +from cryptography import utils + +# This exists to break an import cycle. It is normally accessible from the +# ciphers module. + + +class CipherAlgorithm(metaclass=abc.ABCMeta): + @property + @abc.abstractmethod + def name(self) -> str: + """ + A string naming this mode (e.g. "AES", "Camellia"). + """ + + @property + @abc.abstractmethod + def key_sizes(self) -> frozenset[int]: + """ + Valid key sizes for this algorithm in bits + """ + + @property + @abc.abstractmethod + def key_size(self) -> int: + """ + The size of the key being used as an integer in bits (e.g. 128, 256). + """ + + +class BlockCipherAlgorithm(CipherAlgorithm): + key: utils.Buffer + + @property + @abc.abstractmethod + def block_size(self) -> int: + """ + The size of a block as an integer in bits (e.g. 64, 128). + """ + + +def _verify_key_size( + algorithm: CipherAlgorithm, key: utils.Buffer +) -> utils.Buffer: + # Verify that the key is instance of bytes + utils._check_byteslike("key", key) + + # Verify that the key size matches the expected key size + if len(key) * 8 not in algorithm.key_sizes: + raise ValueError( + f"Invalid key size ({len(key) * 8}) for {algorithm.name}." + ) + return key diff --git a/py_modules/cryptography/hazmat/primitives/_modes.py b/py_modules/cryptography/hazmat/primitives/_modes.py new file mode 100644 index 0000000..deae8bc --- /dev/null +++ b/py_modules/cryptography/hazmat/primitives/_modes.py @@ -0,0 +1,105 @@ +# This file is dual licensed under the terms of the Apache License, Version +# 2.0, and the BSD License. See the LICENSE file in the root of this repository +# for complete details. + +from __future__ import annotations + +import abc + +from cryptography import utils +from cryptography.exceptions import UnsupportedAlgorithm, _Reasons +from cryptography.hazmat.primitives._cipheralgorithm import ( + BlockCipherAlgorithm, + CipherAlgorithm, +) + + +class Mode(metaclass=abc.ABCMeta): + @property + @abc.abstractmethod + def name(self) -> str: + """ + A string naming this mode (e.g. "ECB", "CBC"). + """ + + @abc.abstractmethod + def validate_for_algorithm(self, algorithm: CipherAlgorithm) -> None: + """ + Checks that all the necessary invariants of this (mode, algorithm) + combination are met. + """ + + +class ModeWithInitializationVector(Mode, metaclass=abc.ABCMeta): + @property + @abc.abstractmethod + def initialization_vector(self) -> utils.Buffer: + """ + The value of the initialization vector for this mode as bytes. + """ + + +class ModeWithTweak(Mode, metaclass=abc.ABCMeta): + @property + @abc.abstractmethod + def tweak(self) -> utils.Buffer: + """ + The value of the tweak for this mode as bytes. + """ + + +class ModeWithNonce(Mode, metaclass=abc.ABCMeta): + @property + @abc.abstractmethod + def nonce(self) -> utils.Buffer: + """ + The value of the nonce for this mode as bytes. + """ + + +class ModeWithAuthenticationTag(Mode, metaclass=abc.ABCMeta): + @property + @abc.abstractmethod + def tag(self) -> bytes | None: + """ + The value of the tag supplied to the constructor of this mode. + """ + + +def _check_aes_key_length(self: Mode, algorithm: CipherAlgorithm) -> None: + if algorithm.key_size > 256 and algorithm.name == "AES": + raise ValueError( + "Only 128, 192, and 256 bit keys are allowed for this AES mode" + ) + + +def _check_iv_length( + self: ModeWithInitializationVector, algorithm: BlockCipherAlgorithm +) -> None: + iv_len = len(self.initialization_vector) + if iv_len * 8 != algorithm.block_size: + raise ValueError(f"Invalid IV size ({iv_len}) for {self.name}.") + + +def _check_nonce_length( + nonce: utils.Buffer, name: str, algorithm: CipherAlgorithm +) -> None: + if not isinstance(algorithm, BlockCipherAlgorithm): + raise UnsupportedAlgorithm( + f"{name} requires a block cipher algorithm", + _Reasons.UNSUPPORTED_CIPHER, + ) + if len(nonce) * 8 != algorithm.block_size: + raise ValueError(f"Invalid nonce size ({len(nonce)}) for {name}.") + + +def _check_iv_and_key_length( + self: ModeWithInitializationVector, algorithm: CipherAlgorithm +) -> None: + if not isinstance(algorithm, BlockCipherAlgorithm): + raise UnsupportedAlgorithm( + f"{self} requires a block cipher algorithm", + _Reasons.UNSUPPORTED_CIPHER, + ) + _check_aes_key_length(self, algorithm) + _check_iv_length(self, algorithm) diff --git a/py_modules/cryptography/hazmat/primitives/_serialization.py b/py_modules/cryptography/hazmat/primitives/_serialization.py new file mode 100644 index 0000000..9c3c474 --- /dev/null +++ b/py_modules/cryptography/hazmat/primitives/_serialization.py @@ -0,0 +1,136 @@ +# This file is dual licensed under the terms of the Apache License, Version +# 2.0, and the BSD License. See the LICENSE file in the root of this repository +# for complete details. + +from __future__ import annotations + +import abc + +from cryptography import utils +from cryptography.hazmat.bindings._rust import Encoding as Encoding +from cryptography.hazmat.bindings._rust import ( + ParameterFormat as ParameterFormat, +) +from cryptography.hazmat.bindings._rust import PrivateFormat as PrivateFormat +from cryptography.hazmat.bindings._rust import PublicFormat as PublicFormat +from cryptography.hazmat.primitives.hashes import HashAlgorithm + +# This exists to break an import cycle. These classes are normally accessible +# from the serialization module. + + +class PBES(utils.Enum): + PBESv1SHA1And3KeyTripleDESCBC = "PBESv1 using SHA1 and 3-Key TripleDES" + PBESv2SHA256AndAES256CBC = "PBESv2 using SHA256 PBKDF2 and AES256 CBC" + + +class KeySerializationEncryption(metaclass=abc.ABCMeta): + pass + + +class BestAvailableEncryption(KeySerializationEncryption): + def __init__(self, password: bytes): + if not isinstance(password, bytes) or len(password) == 0: + raise ValueError("Password must be 1 or more bytes.") + + self.password = password + + +class NoEncryption(KeySerializationEncryption): + pass + + +class KeySerializationEncryptionBuilder: + def __init__( + self, + format: PrivateFormat, + *, + _kdf_rounds: int | None = None, + _hmac_hash: HashAlgorithm | None = None, + _key_cert_algorithm: PBES | None = None, + ) -> None: + self._format = format + + self._kdf_rounds = _kdf_rounds + self._hmac_hash = _hmac_hash + self._key_cert_algorithm = _key_cert_algorithm + + def kdf_rounds(self, rounds: int) -> KeySerializationEncryptionBuilder: + if self._kdf_rounds is not None: + raise ValueError("kdf_rounds already set") + + if not isinstance(rounds, int): + raise TypeError("kdf_rounds must be an integer") + + if rounds < 1: + raise ValueError("kdf_rounds must be a positive integer") + + return KeySerializationEncryptionBuilder( + self._format, + _kdf_rounds=rounds, + _hmac_hash=self._hmac_hash, + _key_cert_algorithm=self._key_cert_algorithm, + ) + + def hmac_hash( + self, algorithm: HashAlgorithm + ) -> KeySerializationEncryptionBuilder: + if self._format is not PrivateFormat.PKCS12: + raise TypeError( + "hmac_hash only supported with PrivateFormat.PKCS12" + ) + + if self._hmac_hash is not None: + raise ValueError("hmac_hash already set") + return KeySerializationEncryptionBuilder( + self._format, + _kdf_rounds=self._kdf_rounds, + _hmac_hash=algorithm, + _key_cert_algorithm=self._key_cert_algorithm, + ) + + def key_cert_algorithm( + self, algorithm: PBES + ) -> KeySerializationEncryptionBuilder: + if self._format is not PrivateFormat.PKCS12: + raise TypeError( + "key_cert_algorithm only supported with PrivateFormat.PKCS12" + ) + if self._key_cert_algorithm is not None: + raise ValueError("key_cert_algorithm already set") + return KeySerializationEncryptionBuilder( + self._format, + _kdf_rounds=self._kdf_rounds, + _hmac_hash=self._hmac_hash, + _key_cert_algorithm=algorithm, + ) + + def build(self, password: bytes) -> KeySerializationEncryption: + if not isinstance(password, bytes) or len(password) == 0: + raise ValueError("Password must be 1 or more bytes.") + + return _KeySerializationEncryption( + self._format, + password, + kdf_rounds=self._kdf_rounds, + hmac_hash=self._hmac_hash, + key_cert_algorithm=self._key_cert_algorithm, + ) + + +class _KeySerializationEncryption(KeySerializationEncryption): + def __init__( + self, + format: PrivateFormat, + password: bytes, + *, + kdf_rounds: int | None, + hmac_hash: HashAlgorithm | None, + key_cert_algorithm: PBES | None, + ): + self._format = format + self.password = password + + self._kdf_rounds = kdf_rounds + self._hmac_hash = hmac_hash + self._key_cert_algorithm = key_cert_algorithm diff --git a/py_modules/cryptography/hazmat/primitives/asymmetric/__init__.py b/py_modules/cryptography/hazmat/primitives/asymmetric/__init__.py new file mode 100644 index 0000000..41d7318 --- /dev/null +++ b/py_modules/cryptography/hazmat/primitives/asymmetric/__init__.py @@ -0,0 +1,5 @@ +# This file is dual licensed under the terms of the Apache License, Version +# 2.0, and the BSD License. See the LICENSE file in the root of this repository +# for complete details. + +from __future__ import annotations diff --git a/py_modules/cryptography/hazmat/primitives/asymmetric/dh.py b/py_modules/cryptography/hazmat/primitives/asymmetric/dh.py new file mode 100644 index 0000000..814fdd3 --- /dev/null +++ b/py_modules/cryptography/hazmat/primitives/asymmetric/dh.py @@ -0,0 +1,253 @@ +# This file is dual licensed under the terms of the Apache License, Version +# 2.0, and the BSD License. See the LICENSE file in the root of this repository +# for complete details. + +from __future__ import annotations + +import abc + +from cryptography import utils +from cryptography.hazmat.bindings._rust import openssl as rust_openssl +from cryptography.hazmat.primitives import _serialization + +_FFDH_DEPRECATION_MSG = ( + "Diffie-Hellman over finite fields (FFDH) is deprecated and support " + "will be removed in a future release. Use a more modern key exchange " + "algorithm." +) + +generate_parameters = rust_openssl.dh.generate_parameters + + +DHPrivateNumbers = rust_openssl.dh.DHPrivateNumbers +DHPublicNumbers = rust_openssl.dh.DHPublicNumbers +DHParameterNumbers = rust_openssl.dh.DHParameterNumbers + + +class DHParameters(metaclass=abc.ABCMeta): + @abc.abstractmethod + def generate_private_key(self) -> DHPrivateKey: + """ + Generates and returns a DHPrivateKey. + """ + + @abc.abstractmethod + def parameter_bytes( + self, + encoding: _serialization.Encoding, + format: _serialization.ParameterFormat, + ) -> bytes: + """ + Returns the parameters serialized as bytes. + """ + + @abc.abstractmethod + def parameter_numbers(self) -> DHParameterNumbers: + """ + Returns a DHParameterNumbers. + """ + + +DHParametersWithSerialization = DHParameters +DHParameters.register(rust_openssl.dh.DHParameters) + + +class DHPublicKey(metaclass=abc.ABCMeta): + @property + @abc.abstractmethod + def key_size(self) -> int: + """ + The bit length of the prime modulus. + """ + + @abc.abstractmethod + def parameters(self) -> DHParameters: + """ + The DHParameters object associated with this public key. + """ + + @abc.abstractmethod + def public_numbers(self) -> DHPublicNumbers: + """ + Returns a DHPublicNumbers. + """ + + @abc.abstractmethod + def public_bytes( + self, + encoding: _serialization.Encoding, + format: _serialization.PublicFormat, + ) -> bytes: + """ + Returns the key serialized as bytes. + """ + + @abc.abstractmethod + def __eq__(self, other: object) -> bool: + """ + Checks equality. + """ + + @abc.abstractmethod + def __copy__(self) -> DHPublicKey: + """ + Returns a copy. + """ + + @abc.abstractmethod + def __deepcopy__(self, memo: dict) -> DHPublicKey: + """ + Returns a deep copy. + """ + + +DHPublicKeyWithSerialization = DHPublicKey +DHPublicKey.register(rust_openssl.dh.DHPublicKey) + + +class DHPrivateKey(metaclass=abc.ABCMeta): + @property + @abc.abstractmethod + def key_size(self) -> int: + """ + The bit length of the prime modulus. + """ + + @abc.abstractmethod + def public_key(self) -> DHPublicKey: + """ + The DHPublicKey associated with this private key. + """ + + @abc.abstractmethod + def parameters(self) -> DHParameters: + """ + The DHParameters object associated with this private key. + """ + + @abc.abstractmethod + def exchange(self, peer_public_key: DHPublicKey) -> bytes: + """ + Given peer's DHPublicKey, carry out the key exchange and + return shared key as bytes. + """ + + @abc.abstractmethod + def private_numbers(self) -> DHPrivateNumbers: + """ + Returns a DHPrivateNumbers. + """ + + @abc.abstractmethod + def private_bytes( + self, + encoding: _serialization.Encoding, + format: _serialization.PrivateFormat, + encryption_algorithm: _serialization.KeySerializationEncryption, + ) -> bytes: + """ + Returns the key serialized as bytes. + """ + + @abc.abstractmethod + def __copy__(self) -> DHPrivateKey: + """ + Returns a copy. + """ + + @abc.abstractmethod + def __deepcopy__(self, memo: dict) -> DHPrivateKey: + """ + Returns a deep copy. + """ + + +DHPrivateKeyWithSerialization = DHPrivateKey +DHPrivateKey.register(rust_openssl.dh.DHPrivateKey) + +# Aliases that do not emit the deprecation warning on attribute access, for +# internal use (e.g. the unions in +# cryptography.hazmat.primitives.asymmetric.types, which are evaluated at +# import time). +_DHPublicKey = DHPublicKey +_DHPrivateKey = DHPrivateKey + +utils.deprecated( + generate_parameters, + __name__, + _FFDH_DEPRECATION_MSG, + utils.DeprecatedIn50, + name="generate_parameters", +) + +utils.deprecated( + DHPrivateNumbers, + __name__, + _FFDH_DEPRECATION_MSG, + utils.DeprecatedIn50, + name="DHPrivateNumbers", +) + +utils.deprecated( + DHPublicNumbers, + __name__, + _FFDH_DEPRECATION_MSG, + utils.DeprecatedIn50, + name="DHPublicNumbers", +) + +utils.deprecated( + DHParameterNumbers, + __name__, + _FFDH_DEPRECATION_MSG, + utils.DeprecatedIn50, + name="DHParameterNumbers", +) + +utils.deprecated( + DHParameters, + __name__, + _FFDH_DEPRECATION_MSG, + utils.DeprecatedIn50, + name="DHParameters", +) + +utils.deprecated( + DHParameters, + __name__, + _FFDH_DEPRECATION_MSG, + utils.DeprecatedIn50, + name="DHParametersWithSerialization", +) + +utils.deprecated( + DHPublicKey, + __name__, + _FFDH_DEPRECATION_MSG, + utils.DeprecatedIn50, + name="DHPublicKey", +) + +utils.deprecated( + DHPublicKey, + __name__, + _FFDH_DEPRECATION_MSG, + utils.DeprecatedIn50, + name="DHPublicKeyWithSerialization", +) + +utils.deprecated( + DHPrivateKey, + __name__, + _FFDH_DEPRECATION_MSG, + utils.DeprecatedIn50, + name="DHPrivateKey", +) + +utils.deprecated( + DHPrivateKey, + __name__, + _FFDH_DEPRECATION_MSG, + utils.DeprecatedIn50, + name="DHPrivateKeyWithSerialization", +) diff --git a/py_modules/cryptography/hazmat/primitives/asymmetric/dsa.py b/py_modules/cryptography/hazmat/primitives/asymmetric/dsa.py new file mode 100644 index 0000000..f245557 --- /dev/null +++ b/py_modules/cryptography/hazmat/primitives/asymmetric/dsa.py @@ -0,0 +1,179 @@ +# This file is dual licensed under the terms of the Apache License, Version +# 2.0, and the BSD License. See the LICENSE file in the root of this repository +# for complete details. + +from __future__ import annotations + +import abc +import typing + +from cryptography.hazmat.bindings._rust import openssl as rust_openssl +from cryptography.hazmat.primitives import _serialization, hashes +from cryptography.hazmat.primitives.asymmetric import utils as asym_utils +from cryptography.utils import Buffer + + +class DSAParameters(metaclass=abc.ABCMeta): + @abc.abstractmethod + def generate_private_key(self) -> DSAPrivateKey: + """ + Generates and returns a DSAPrivateKey. + """ + + @abc.abstractmethod + def parameter_numbers(self) -> DSAParameterNumbers: + """ + Returns a DSAParameterNumbers. + """ + + +DSAParametersWithNumbers = DSAParameters +DSAParameters.register(rust_openssl.dsa.DSAParameters) + + +class DSAPrivateKey(metaclass=abc.ABCMeta): + @property + @abc.abstractmethod + def key_size(self) -> int: + """ + The bit length of the prime modulus. + """ + + @abc.abstractmethod + def public_key(self) -> DSAPublicKey: + """ + The DSAPublicKey associated with this private key. + """ + + @abc.abstractmethod + def parameters(self) -> DSAParameters: + """ + The DSAParameters object associated with this private key. + """ + + @abc.abstractmethod + def sign( + self, + data: Buffer, + algorithm: asym_utils.Prehashed | hashes.HashAlgorithm, + ) -> bytes: + """ + Signs the data + """ + + @abc.abstractmethod + def private_numbers(self) -> DSAPrivateNumbers: + """ + Returns a DSAPrivateNumbers. + """ + + @abc.abstractmethod + def private_bytes( + self, + encoding: _serialization.Encoding, + format: _serialization.PrivateFormat, + encryption_algorithm: _serialization.KeySerializationEncryption, + ) -> bytes: + """ + Returns the key serialized as bytes. + """ + + @abc.abstractmethod + def __copy__(self) -> DSAPrivateKey: + """ + Returns a copy. + """ + + @abc.abstractmethod + def __deepcopy__(self, memo: dict) -> DSAPrivateKey: + """ + Returns a deep copy. + """ + + +DSAPrivateKeyWithSerialization = DSAPrivateKey +DSAPrivateKey.register(rust_openssl.dsa.DSAPrivateKey) + + +class DSAPublicKey(metaclass=abc.ABCMeta): + @property + @abc.abstractmethod + def key_size(self) -> int: + """ + The bit length of the prime modulus. + """ + + @abc.abstractmethod + def parameters(self) -> DSAParameters: + """ + The DSAParameters object associated with this public key. + """ + + @abc.abstractmethod + def public_numbers(self) -> DSAPublicNumbers: + """ + Returns a DSAPublicNumbers. + """ + + @abc.abstractmethod + def public_bytes( + self, + encoding: _serialization.Encoding, + format: _serialization.PublicFormat, + ) -> bytes: + """ + Returns the key serialized as bytes. + """ + + @abc.abstractmethod + def verify( + self, + signature: Buffer, + data: Buffer, + algorithm: asym_utils.Prehashed | hashes.HashAlgorithm, + ) -> None: + """ + Verifies the signature of the data. + """ + + @abc.abstractmethod + def __eq__(self, other: object) -> bool: + """ + Checks equality. + """ + + @abc.abstractmethod + def __copy__(self) -> DSAPublicKey: + """ + Returns a copy. + """ + + @abc.abstractmethod + def __deepcopy__(self, memo: dict) -> DSAPublicKey: + """ + Returns a deep copy. + """ + + +DSAPublicKeyWithSerialization = DSAPublicKey +DSAPublicKey.register(rust_openssl.dsa.DSAPublicKey) + +DSAPrivateNumbers = rust_openssl.dsa.DSAPrivateNumbers +DSAPublicNumbers = rust_openssl.dsa.DSAPublicNumbers +DSAParameterNumbers = rust_openssl.dsa.DSAParameterNumbers + + +def generate_parameters( + key_size: int, backend: typing.Any = None +) -> DSAParameters: + if key_size not in (1024, 2048, 3072, 4096): + raise ValueError("Key size must be 1024, 2048, 3072, or 4096 bits.") + + return rust_openssl.dsa.generate_parameters(key_size) + + +def generate_private_key( + key_size: int, backend: typing.Any = None +) -> DSAPrivateKey: + parameters = generate_parameters(key_size) + return parameters.generate_private_key() diff --git a/py_modules/cryptography/hazmat/primitives/asymmetric/ec.py b/py_modules/cryptography/hazmat/primitives/asymmetric/ec.py new file mode 100644 index 0000000..39e6751 --- /dev/null +++ b/py_modules/cryptography/hazmat/primitives/asymmetric/ec.py @@ -0,0 +1,369 @@ +# This file is dual licensed under the terms of the Apache License, Version +# 2.0, and the BSD License. See the LICENSE file in the root of this repository +# for complete details. + +from __future__ import annotations + +import abc +import typing + +from cryptography import utils +from cryptography.exceptions import UnsupportedAlgorithm, _Reasons +from cryptography.hazmat._oid import ObjectIdentifier +from cryptography.hazmat.bindings._rust import openssl as rust_openssl +from cryptography.hazmat.primitives import _serialization, hashes +from cryptography.hazmat.primitives.asymmetric import utils as asym_utils + + +class EllipticCurveOID: + SECP192R1 = ObjectIdentifier("1.2.840.10045.3.1.1") + SECP224R1 = ObjectIdentifier("1.3.132.0.33") + SECP256K1 = ObjectIdentifier("1.3.132.0.10") + SECP256R1 = ObjectIdentifier("1.2.840.10045.3.1.7") + SECP384R1 = ObjectIdentifier("1.3.132.0.34") + SECP521R1 = ObjectIdentifier("1.3.132.0.35") + BRAINPOOLP256R1 = ObjectIdentifier("1.3.36.3.3.2.8.1.1.7") + BRAINPOOLP384R1 = ObjectIdentifier("1.3.36.3.3.2.8.1.1.11") + BRAINPOOLP512R1 = ObjectIdentifier("1.3.36.3.3.2.8.1.1.13") + + +class EllipticCurve(metaclass=abc.ABCMeta): + @property + @abc.abstractmethod + def name(self) -> str: + """ + The name of the curve. e.g. secp256r1. + """ + + @property + @abc.abstractmethod + def key_size(self) -> int: + """ + Bit size of a secret scalar for the curve. + """ + + @property + @abc.abstractmethod + def group_order(self) -> int: + """ + The order of the curve's group. + """ + + +class EllipticCurveSignatureAlgorithm(metaclass=abc.ABCMeta): + @property + @abc.abstractmethod + def algorithm( + self, + ) -> asym_utils.Prehashed | hashes.HashAlgorithm: + """ + The digest algorithm used with this signature. + """ + + +class EllipticCurvePrivateKey(metaclass=abc.ABCMeta): + @abc.abstractmethod + def exchange( + self, algorithm: ECDH, peer_public_key: EllipticCurvePublicKey + ) -> bytes: + """ + Performs a key exchange operation using the provided algorithm with the + provided peer's public key. + """ + + @abc.abstractmethod + def public_key(self) -> EllipticCurvePublicKey: + """ + The EllipticCurvePublicKey for this private key. + """ + + @property + @abc.abstractmethod + def curve(self) -> EllipticCurve: + """ + The EllipticCurve that this key is on. + """ + + @property + @abc.abstractmethod + def key_size(self) -> int: + """ + Bit size of a secret scalar for the curve. + """ + + @abc.abstractmethod + def sign( + self, + data: utils.Buffer, + signature_algorithm: EllipticCurveSignatureAlgorithm, + ) -> bytes: + """ + Signs the data + """ + + @abc.abstractmethod + def private_numbers(self) -> EllipticCurvePrivateNumbers: + """ + Returns an EllipticCurvePrivateNumbers. + """ + + @abc.abstractmethod + def private_bytes( + self, + encoding: _serialization.Encoding, + format: _serialization.PrivateFormat, + encryption_algorithm: _serialization.KeySerializationEncryption, + ) -> bytes: + """ + Returns the key serialized as bytes. + """ + + @abc.abstractmethod + def __copy__(self) -> EllipticCurvePrivateKey: + """ + Returns a copy. + """ + + @abc.abstractmethod + def __deepcopy__(self, memo: dict) -> EllipticCurvePrivateKey: + """ + Returns a deep copy. + """ + + +EllipticCurvePrivateKeyWithSerialization = EllipticCurvePrivateKey +EllipticCurvePrivateKey.register(rust_openssl.ec.ECPrivateKey) + + +class EllipticCurvePublicKey(metaclass=abc.ABCMeta): + @property + @abc.abstractmethod + def curve(self) -> EllipticCurve: + """ + The EllipticCurve that this key is on. + """ + + @property + @abc.abstractmethod + def key_size(self) -> int: + """ + Bit size of a secret scalar for the curve. + """ + + @abc.abstractmethod + def public_numbers(self) -> EllipticCurvePublicNumbers: + """ + Returns an EllipticCurvePublicNumbers. + """ + + @abc.abstractmethod + def public_bytes( + self, + encoding: _serialization.Encoding, + format: _serialization.PublicFormat, + ) -> bytes: + """ + Returns the key serialized as bytes. + """ + + @abc.abstractmethod + def verify( + self, + signature: utils.Buffer, + data: utils.Buffer, + signature_algorithm: EllipticCurveSignatureAlgorithm, + ) -> None: + """ + Verifies the signature of the data. + """ + + @classmethod + def from_encoded_point( + cls, curve: EllipticCurve, data: bytes + ) -> EllipticCurvePublicKey: + utils._check_bytes("data", data) + + if len(data) == 0: + raise ValueError("data must not be an empty byte string") + + if data[0] not in [0x02, 0x03, 0x04]: + raise ValueError("Unsupported elliptic curve point type") + + return rust_openssl.ec.from_public_bytes(curve, data) + + @abc.abstractmethod + def __eq__(self, other: object) -> bool: + """ + Checks equality. + """ + + @abc.abstractmethod + def __copy__(self) -> EllipticCurvePublicKey: + """ + Returns a copy. + """ + + @abc.abstractmethod + def __deepcopy__(self, memo: dict) -> EllipticCurvePublicKey: + """ + Returns a deep copy. + """ + + +EllipticCurvePublicKeyWithSerialization = EllipticCurvePublicKey +EllipticCurvePublicKey.register(rust_openssl.ec.ECPublicKey) + +EllipticCurvePrivateNumbers = rust_openssl.ec.EllipticCurvePrivateNumbers +EllipticCurvePublicNumbers = rust_openssl.ec.EllipticCurvePublicNumbers + + +class SECP521R1(EllipticCurve): + name = "secp521r1" + key_size = 521 + group_order = 0x1FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFA51868783BF2F966B7FCC0148F709A5D03BB5C9B8899C47AEBB6FB71E91386409 # noqa: E501 + + +class SECP384R1(EllipticCurve): + name = "secp384r1" + key_size = 384 + group_order = 0xFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFC7634D81F4372DDF581A0DB248B0A77AECEC196ACCC52973 # noqa: E501 + + +class SECP256R1(EllipticCurve): + name = "secp256r1" + key_size = 256 + group_order = ( + 0xFFFFFFFF00000000FFFFFFFFFFFFFFFFBCE6FAADA7179E84F3B9CAC2FC632551 + ) + + +class SECP256K1(EllipticCurve): + name = "secp256k1" + key_size = 256 + group_order = ( + 0xFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFEBAAEDCE6AF48A03BBFD25E8CD0364141 + ) + + +class SECP224R1(EllipticCurve): + name = "secp224r1" + key_size = 224 + group_order = 0xFFFFFFFFFFFFFFFFFFFFFFFFFFFF16A2E0B8F03E13DD29455C5C2A3D + + +class SECP192R1(EllipticCurve): + name = "secp192r1" + key_size = 192 + group_order = 0xFFFFFFFFFFFFFFFFFFFFFFFF99DEF836146BC9B1B4D22831 + + +class BrainpoolP256R1(EllipticCurve): + name = "brainpoolP256r1" + key_size = 256 + group_order = ( + 0xA9FB57DBA1EEA9BC3E660A909D838D718C397AA3B561A6F7901E0E82974856A7 + ) + + +class BrainpoolP384R1(EllipticCurve): + name = "brainpoolP384r1" + key_size = 384 + group_order = 0x8CB91E82A3386D280F5D6F7E50E641DF152F7109ED5456B31F166E6CAC0425A7CF3AB6AF6B7FC3103B883202E9046565 # noqa: E501 + + +class BrainpoolP512R1(EllipticCurve): + name = "brainpoolP512r1" + key_size = 512 + group_order = 0xAADD9DB8DBE9C48B3FD4E6AE33C9FC07CB308DB3B3C9D20ED6639CCA70330870553E5C414CA92619418661197FAC10471DB1D381085DDADDB58796829CA90069 # noqa: E501 + + +_CURVE_TYPES: dict[str, EllipticCurve] = { + "prime192v1": SECP192R1(), + "prime256v1": SECP256R1(), + "secp192r1": SECP192R1(), + "secp224r1": SECP224R1(), + "secp256r1": SECP256R1(), + "secp384r1": SECP384R1(), + "secp521r1": SECP521R1(), + "secp256k1": SECP256K1(), + "brainpoolP256r1": BrainpoolP256R1(), + "brainpoolP384r1": BrainpoolP384R1(), + "brainpoolP512r1": BrainpoolP512R1(), +} + + +class ECDSA(EllipticCurveSignatureAlgorithm): + def __init__( + self, + algorithm: asym_utils.Prehashed | hashes.HashAlgorithm, + deterministic_signing: bool = False, + ): + from cryptography.hazmat.backends.openssl.backend import backend + + if ( + deterministic_signing + and not backend.ecdsa_deterministic_supported() + ): + raise UnsupportedAlgorithm( + "ECDSA with deterministic signature (RFC 6979) is not " + "supported by this version of OpenSSL.", + _Reasons.UNSUPPORTED_PUBLIC_KEY_ALGORITHM, + ) + self._algorithm = algorithm + self._deterministic_signing = deterministic_signing + + @property + def algorithm( + self, + ) -> asym_utils.Prehashed | hashes.HashAlgorithm: + return self._algorithm + + @property + def deterministic_signing( + self, + ) -> bool: + return self._deterministic_signing + + +generate_private_key = rust_openssl.ec.generate_private_key + + +def derive_private_key( + private_value: int, + curve: EllipticCurve, + backend: typing.Any = None, +) -> EllipticCurvePrivateKey: + if not isinstance(private_value, int): + raise TypeError("private_value must be an integer type.") + + if private_value <= 0: + raise ValueError("private_value must be a positive integer.") + + return rust_openssl.ec.derive_private_key(private_value, curve) + + +class ECDH: + pass + + +_OID_TO_CURVE = { + EllipticCurveOID.SECP192R1: SECP192R1, + EllipticCurveOID.SECP224R1: SECP224R1, + EllipticCurveOID.SECP256K1: SECP256K1, + EllipticCurveOID.SECP256R1: SECP256R1, + EllipticCurveOID.SECP384R1: SECP384R1, + EllipticCurveOID.SECP521R1: SECP521R1, + EllipticCurveOID.BRAINPOOLP256R1: BrainpoolP256R1, + EllipticCurveOID.BRAINPOOLP384R1: BrainpoolP384R1, + EllipticCurveOID.BRAINPOOLP512R1: BrainpoolP512R1, +} + + +def get_curve_for_oid(oid: ObjectIdentifier) -> type[EllipticCurve]: + try: + return _OID_TO_CURVE[oid] + except KeyError: + raise LookupError( + "The provided object identifier has no matching elliptic " + "curve class" + ) diff --git a/py_modules/cryptography/hazmat/primitives/asymmetric/ed25519.py b/py_modules/cryptography/hazmat/primitives/asymmetric/ed25519.py new file mode 100644 index 0000000..70aec5b --- /dev/null +++ b/py_modules/cryptography/hazmat/primitives/asymmetric/ed25519.py @@ -0,0 +1,116 @@ +# This file is dual licensed under the terms of the Apache License, Version +# 2.0, and the BSD License. See the LICENSE file in the root of this repository +# for complete details. + +from __future__ import annotations + +import abc + +from cryptography.hazmat.bindings._rust import openssl as rust_openssl +from cryptography.hazmat.primitives import _serialization +from cryptography.utils import Buffer + + +class Ed25519PublicKey(metaclass=abc.ABCMeta): + @classmethod + def from_public_bytes(cls, data: bytes) -> Ed25519PublicKey: + return rust_openssl.ed25519.from_public_bytes(data) + + @abc.abstractmethod + def public_bytes( + self, + encoding: _serialization.Encoding, + format: _serialization.PublicFormat, + ) -> bytes: + """ + The serialized bytes of the public key. + """ + + @abc.abstractmethod + def public_bytes_raw(self) -> bytes: + """ + The raw bytes of the public key. + Equivalent to public_bytes(Raw, Raw). + """ + + @abc.abstractmethod + def verify(self, signature: Buffer, data: Buffer) -> None: + """ + Verify the signature. + """ + + @abc.abstractmethod + def __eq__(self, other: object) -> bool: + """ + Checks equality. + """ + + @abc.abstractmethod + def __copy__(self) -> Ed25519PublicKey: + """ + Returns a copy. + """ + + @abc.abstractmethod + def __deepcopy__(self, memo: dict) -> Ed25519PublicKey: + """ + Returns a deep copy. + """ + + +Ed25519PublicKey.register(rust_openssl.ed25519.Ed25519PublicKey) + + +class Ed25519PrivateKey(metaclass=abc.ABCMeta): + @classmethod + def generate(cls) -> Ed25519PrivateKey: + return rust_openssl.ed25519.generate_key() + + @classmethod + def from_private_bytes(cls, data: Buffer) -> Ed25519PrivateKey: + return rust_openssl.ed25519.from_private_bytes(data) + + @abc.abstractmethod + def public_key(self) -> Ed25519PublicKey: + """ + The Ed25519PublicKey derived from the private key. + """ + + @abc.abstractmethod + def private_bytes( + self, + encoding: _serialization.Encoding, + format: _serialization.PrivateFormat, + encryption_algorithm: _serialization.KeySerializationEncryption, + ) -> bytes: + """ + The serialized bytes of the private key. + """ + + @abc.abstractmethod + def private_bytes_raw(self) -> bytes: + """ + The raw bytes of the private key. + Equivalent to private_bytes(Raw, Raw, NoEncryption()). + """ + + @abc.abstractmethod + def sign(self, data: Buffer) -> bytes: + """ + Signs the data. + """ + + @abc.abstractmethod + def __copy__(self) -> Ed25519PrivateKey: + """ + Returns a copy. + """ + + @abc.abstractmethod + def __deepcopy__(self, memo: dict) -> Ed25519PrivateKey: + """ + Returns a deep copy. + """ + + +Ed25519PrivateKey.register(rust_openssl.ed25519.Ed25519PrivateKey) diff --git a/py_modules/cryptography/hazmat/primitives/asymmetric/ed448.py b/py_modules/cryptography/hazmat/primitives/asymmetric/ed448.py new file mode 100644 index 0000000..9ecb478 --- /dev/null +++ b/py_modules/cryptography/hazmat/primitives/asymmetric/ed448.py @@ -0,0 +1,143 @@ +# This file is dual licensed under the terms of the Apache License, Version +# 2.0, and the BSD License. See the LICENSE file in the root of this repository +# for complete details. + +from __future__ import annotations + +import abc + +from cryptography.exceptions import UnsupportedAlgorithm, _Reasons +from cryptography.hazmat.bindings._rust import openssl as rust_openssl +from cryptography.hazmat.primitives import _serialization +from cryptography.utils import Buffer + + +class Ed448PublicKey(metaclass=abc.ABCMeta): + @classmethod + def from_public_bytes(cls, data: bytes) -> Ed448PublicKey: + from cryptography.hazmat.backends.openssl.backend import backend + + if not backend.ed448_supported(): + raise UnsupportedAlgorithm( + "ed448 is not supported by this version of OpenSSL.", + _Reasons.UNSUPPORTED_PUBLIC_KEY_ALGORITHM, + ) + + return rust_openssl.ed448.from_public_bytes(data) + + @abc.abstractmethod + def public_bytes( + self, + encoding: _serialization.Encoding, + format: _serialization.PublicFormat, + ) -> bytes: + """ + The serialized bytes of the public key. + """ + + @abc.abstractmethod + def public_bytes_raw(self) -> bytes: + """ + The raw bytes of the public key. + Equivalent to public_bytes(Raw, Raw). + """ + + @abc.abstractmethod + def verify(self, signature: Buffer, data: Buffer) -> None: + """ + Verify the signature. + """ + + @abc.abstractmethod + def __eq__(self, other: object) -> bool: + """ + Checks equality. + """ + + @abc.abstractmethod + def __copy__(self) -> Ed448PublicKey: + """ + Returns a copy. + """ + + @abc.abstractmethod + def __deepcopy__(self, memo: dict) -> Ed448PublicKey: + """ + Returns a deep copy. + """ + + +if hasattr(rust_openssl, "ed448"): + Ed448PublicKey.register(rust_openssl.ed448.Ed448PublicKey) + + +class Ed448PrivateKey(metaclass=abc.ABCMeta): + @classmethod + def generate(cls) -> Ed448PrivateKey: + from cryptography.hazmat.backends.openssl.backend import backend + + if not backend.ed448_supported(): + raise UnsupportedAlgorithm( + "ed448 is not supported by this version of OpenSSL.", + _Reasons.UNSUPPORTED_PUBLIC_KEY_ALGORITHM, + ) + + return rust_openssl.ed448.generate_key() + + @classmethod + def from_private_bytes(cls, data: Buffer) -> Ed448PrivateKey: + from cryptography.hazmat.backends.openssl.backend import backend + + if not backend.ed448_supported(): + raise UnsupportedAlgorithm( + "ed448 is not supported by this version of OpenSSL.", + _Reasons.UNSUPPORTED_PUBLIC_KEY_ALGORITHM, + ) + + return rust_openssl.ed448.from_private_bytes(data) + + @abc.abstractmethod + def public_key(self) -> Ed448PublicKey: + """ + The Ed448PublicKey derived from the private key. + """ + + @abc.abstractmethod + def sign(self, data: Buffer) -> bytes: + """ + Signs the data. + """ + + @abc.abstractmethod + def private_bytes( + self, + encoding: _serialization.Encoding, + format: _serialization.PrivateFormat, + encryption_algorithm: _serialization.KeySerializationEncryption, + ) -> bytes: + """ + The serialized bytes of the private key. + """ + + @abc.abstractmethod + def private_bytes_raw(self) -> bytes: + """ + The raw bytes of the private key. + Equivalent to private_bytes(Raw, Raw, NoEncryption()). + """ + + @abc.abstractmethod + def __copy__(self) -> Ed448PrivateKey: + """ + Returns a copy. + """ + + @abc.abstractmethod + def __deepcopy__(self, memo: dict) -> Ed448PrivateKey: + """ + Returns a deep copy. + """ + + +if hasattr(rust_openssl, "x448"): + Ed448PrivateKey.register(rust_openssl.ed448.Ed448PrivateKey) diff --git a/py_modules/cryptography/hazmat/primitives/asymmetric/mldsa.py b/py_modules/cryptography/hazmat/primitives/asymmetric/mldsa.py new file mode 100644 index 0000000..beba90f --- /dev/null +++ b/py_modules/cryptography/hazmat/primitives/asymmetric/mldsa.py @@ -0,0 +1,505 @@ +# This file is dual licensed under the terms of the Apache License, Version +# 2.0, and the BSD License. See the LICENSE file in the root of this repository +# for complete details. + +from __future__ import annotations + +import abc + +from cryptography.exceptions import UnsupportedAlgorithm, _Reasons +from cryptography.hazmat.bindings._rust import openssl as rust_openssl +from cryptography.hazmat.primitives import _serialization +from cryptography.utils import Buffer + + +class MLDSA44PublicKey(metaclass=abc.ABCMeta): + @classmethod + def from_public_bytes(cls, data: bytes) -> MLDSA44PublicKey: + from cryptography.hazmat.backends.openssl.backend import backend + + if not backend.mldsa_supported(): + raise UnsupportedAlgorithm( + "ML-DSA-44 is not supported by this backend.", + _Reasons.UNSUPPORTED_PUBLIC_KEY_ALGORITHM, + ) + + return rust_openssl.mldsa.from_mldsa44_public_bytes(data) + + @abc.abstractmethod + def public_bytes( + self, + encoding: _serialization.Encoding, + format: _serialization.PublicFormat, + ) -> bytes: + """ + The serialized bytes of the public key. + """ + + @abc.abstractmethod + def public_bytes_raw(self) -> bytes: + """ + The raw bytes of the public key. + Equivalent to public_bytes(Raw, Raw). + + The public key is 1,312 bytes for MLDSA-44. + """ + + @abc.abstractmethod + def verify( + self, + signature: Buffer, + data: Buffer, + context: Buffer | None = None, + ) -> None: + """ + Verify the signature. + """ + + @abc.abstractmethod + def verify_mu( + self, + signature: Buffer, + mu: Buffer, + ) -> None: + """ + Verify the signature over a precomputed mu (message representative). + + mu must be 64 bytes. + """ + + @abc.abstractmethod + def __eq__(self, other: object) -> bool: + """ + Checks equality. + """ + + @abc.abstractmethod + def __copy__(self) -> MLDSA44PublicKey: + """ + Returns a copy. + """ + + @abc.abstractmethod + def __deepcopy__(self, memo: dict) -> MLDSA44PublicKey: + """ + Returns a deep copy. + """ + + +if hasattr(rust_openssl, "mldsa"): + MLDSA44PublicKey.register(rust_openssl.mldsa.MLDSA44PublicKey) + + +class MLDSA44PrivateKey(metaclass=abc.ABCMeta): + @classmethod + def generate(cls) -> MLDSA44PrivateKey: + from cryptography.hazmat.backends.openssl.backend import backend + + if not backend.mldsa_supported(): + raise UnsupportedAlgorithm( + "ML-DSA-44 is not supported by this backend.", + _Reasons.UNSUPPORTED_PUBLIC_KEY_ALGORITHM, + ) + + return rust_openssl.mldsa.generate_mldsa44_key() + + @classmethod + def from_seed_bytes(cls, data: Buffer) -> MLDSA44PrivateKey: + from cryptography.hazmat.backends.openssl.backend import backend + + if not backend.mldsa_supported(): + raise UnsupportedAlgorithm( + "ML-DSA-44 is not supported by this backend.", + _Reasons.UNSUPPORTED_PUBLIC_KEY_ALGORITHM, + ) + + return rust_openssl.mldsa.from_mldsa44_seed_bytes(data) + + @abc.abstractmethod + def public_key(self) -> MLDSA44PublicKey: + """ + The MLDSA44PublicKey derived from the private key. + """ + + @abc.abstractmethod + def private_bytes( + self, + encoding: _serialization.Encoding, + format: _serialization.PrivateFormat, + encryption_algorithm: _serialization.KeySerializationEncryption, + ) -> bytes: + """ + The serialized bytes of the private key. + + This method only returns the serialization of the seed form of the + private key, never the expanded one. + """ + + @abc.abstractmethod + def private_bytes_raw(self) -> bytes: + """ + The raw bytes of the private key. + Equivalent to private_bytes(Raw, Raw, NoEncryption()). + + This method only returns the seed form of the private key (32 bytes). + """ + + @abc.abstractmethod + def sign(self, data: Buffer, context: Buffer | None = None) -> bytes: + """ + Signs the data. + """ + + @abc.abstractmethod + def sign_mu(self, mu: Buffer) -> bytes: + """ + Signs a precomputed mu (message representative). + + mu must be 64 bytes and already incorporates the context, so no + context is accepted here. + """ + + @abc.abstractmethod + def __copy__(self) -> MLDSA44PrivateKey: + """ + Returns a copy. + """ + + @abc.abstractmethod + def __deepcopy__(self, memo: dict) -> MLDSA44PrivateKey: + """ + Returns a deep copy. + """ + + +if hasattr(rust_openssl, "mldsa"): + MLDSA44PrivateKey.register(rust_openssl.mldsa.MLDSA44PrivateKey) + + +class MLDSA65PublicKey(metaclass=abc.ABCMeta): + @classmethod + def from_public_bytes(cls, data: bytes) -> MLDSA65PublicKey: + from cryptography.hazmat.backends.openssl.backend import backend + + if not backend.mldsa_supported(): + raise UnsupportedAlgorithm( + "ML-DSA-65 is not supported by this backend.", + _Reasons.UNSUPPORTED_PUBLIC_KEY_ALGORITHM, + ) + + return rust_openssl.mldsa.from_mldsa65_public_bytes(data) + + @abc.abstractmethod + def public_bytes( + self, + encoding: _serialization.Encoding, + format: _serialization.PublicFormat, + ) -> bytes: + """ + The serialized bytes of the public key. + """ + + @abc.abstractmethod + def public_bytes_raw(self) -> bytes: + """ + The raw bytes of the public key. + Equivalent to public_bytes(Raw, Raw). + + The public key is 1,952 bytes for MLDSA-65. + """ + + @abc.abstractmethod + def verify( + self, + signature: Buffer, + data: Buffer, + context: Buffer | None = None, + ) -> None: + """ + Verify the signature. + """ + + @abc.abstractmethod + def verify_mu( + self, + signature: Buffer, + mu: Buffer, + ) -> None: + """ + Verify the signature over a precomputed mu (message representative). + + mu must be 64 bytes. + """ + + @abc.abstractmethod + def __eq__(self, other: object) -> bool: + """ + Checks equality. + """ + + @abc.abstractmethod + def __copy__(self) -> MLDSA65PublicKey: + """ + Returns a copy. + """ + + @abc.abstractmethod + def __deepcopy__(self, memo: dict) -> MLDSA65PublicKey: + """ + Returns a deep copy. + """ + + +if hasattr(rust_openssl, "mldsa"): + MLDSA65PublicKey.register(rust_openssl.mldsa.MLDSA65PublicKey) + + +class MLDSA65PrivateKey(metaclass=abc.ABCMeta): + @classmethod + def generate(cls) -> MLDSA65PrivateKey: + from cryptography.hazmat.backends.openssl.backend import backend + + if not backend.mldsa_supported(): + raise UnsupportedAlgorithm( + "ML-DSA-65 is not supported by this backend.", + _Reasons.UNSUPPORTED_PUBLIC_KEY_ALGORITHM, + ) + + return rust_openssl.mldsa.generate_mldsa65_key() + + @classmethod + def from_seed_bytes(cls, data: Buffer) -> MLDSA65PrivateKey: + from cryptography.hazmat.backends.openssl.backend import backend + + if not backend.mldsa_supported(): + raise UnsupportedAlgorithm( + "ML-DSA-65 is not supported by this backend.", + _Reasons.UNSUPPORTED_PUBLIC_KEY_ALGORITHM, + ) + + return rust_openssl.mldsa.from_mldsa65_seed_bytes(data) + + @abc.abstractmethod + def public_key(self) -> MLDSA65PublicKey: + """ + The MLDSA65PublicKey derived from the private key. + """ + + @abc.abstractmethod + def private_bytes( + self, + encoding: _serialization.Encoding, + format: _serialization.PrivateFormat, + encryption_algorithm: _serialization.KeySerializationEncryption, + ) -> bytes: + """ + The serialized bytes of the private key. + + This method only returns the serialization of the seed form of the + private key, never the expanded one. + """ + + @abc.abstractmethod + def private_bytes_raw(self) -> bytes: + """ + The raw bytes of the private key. + Equivalent to private_bytes(Raw, Raw, NoEncryption()). + + This method only returns the seed form of the private key (32 bytes). + """ + + @abc.abstractmethod + def sign(self, data: Buffer, context: Buffer | None = None) -> bytes: + """ + Signs the data. + """ + + @abc.abstractmethod + def sign_mu(self, mu: Buffer) -> bytes: + """ + Signs a precomputed mu (message representative). + + mu must be 64 bytes and already incorporates the context, so no + context is accepted here. + """ + + @abc.abstractmethod + def __copy__(self) -> MLDSA65PrivateKey: + """ + Returns a copy. + """ + + @abc.abstractmethod + def __deepcopy__(self, memo: dict) -> MLDSA65PrivateKey: + """ + Returns a deep copy. + """ + + +if hasattr(rust_openssl, "mldsa"): + MLDSA65PrivateKey.register(rust_openssl.mldsa.MLDSA65PrivateKey) + + +class MLDSA87PublicKey(metaclass=abc.ABCMeta): + @classmethod + def from_public_bytes(cls, data: bytes) -> MLDSA87PublicKey: + from cryptography.hazmat.backends.openssl.backend import backend + + if not backend.mldsa_supported(): + raise UnsupportedAlgorithm( + "ML-DSA-87 is not supported by this backend.", + _Reasons.UNSUPPORTED_PUBLIC_KEY_ALGORITHM, + ) + + return rust_openssl.mldsa.from_mldsa87_public_bytes(data) + + @abc.abstractmethod + def public_bytes( + self, + encoding: _serialization.Encoding, + format: _serialization.PublicFormat, + ) -> bytes: + """ + The serialized bytes of the public key. + """ + + @abc.abstractmethod + def public_bytes_raw(self) -> bytes: + """ + The raw bytes of the public key. + Equivalent to public_bytes(Raw, Raw). + + The public key is 2,592 bytes for MLDSA-87. + """ + + @abc.abstractmethod + def verify( + self, + signature: Buffer, + data: Buffer, + context: Buffer | None = None, + ) -> None: + """ + Verify the signature. + """ + + @abc.abstractmethod + def verify_mu( + self, + signature: Buffer, + mu: Buffer, + ) -> None: + """ + Verify the signature over a precomputed mu (message representative). + + mu must be 64 bytes. + """ + + @abc.abstractmethod + def __eq__(self, other: object) -> bool: + """ + Checks equality. + """ + + @abc.abstractmethod + def __copy__(self) -> MLDSA87PublicKey: + """ + Returns a copy. + """ + + @abc.abstractmethod + def __deepcopy__(self, memo: dict) -> MLDSA87PublicKey: + """ + Returns a deep copy. + """ + + +if hasattr(rust_openssl, "mldsa"): + MLDSA87PublicKey.register(rust_openssl.mldsa.MLDSA87PublicKey) + + +class MLDSA87PrivateKey(metaclass=abc.ABCMeta): + @classmethod + def generate(cls) -> MLDSA87PrivateKey: + from cryptography.hazmat.backends.openssl.backend import backend + + if not backend.mldsa_supported(): + raise UnsupportedAlgorithm( + "ML-DSA-87 is not supported by this backend.", + _Reasons.UNSUPPORTED_PUBLIC_KEY_ALGORITHM, + ) + + return rust_openssl.mldsa.generate_mldsa87_key() + + @classmethod + def from_seed_bytes(cls, data: Buffer) -> MLDSA87PrivateKey: + from cryptography.hazmat.backends.openssl.backend import backend + + if not backend.mldsa_supported(): + raise UnsupportedAlgorithm( + "ML-DSA-87 is not supported by this backend.", + _Reasons.UNSUPPORTED_PUBLIC_KEY_ALGORITHM, + ) + + return rust_openssl.mldsa.from_mldsa87_seed_bytes(data) + + @abc.abstractmethod + def public_key(self) -> MLDSA87PublicKey: + """ + The MLDSA87PublicKey derived from the private key. + """ + + @abc.abstractmethod + def private_bytes( + self, + encoding: _serialization.Encoding, + format: _serialization.PrivateFormat, + encryption_algorithm: _serialization.KeySerializationEncryption, + ) -> bytes: + """ + The serialized bytes of the private key. + + This method only returns the serialization of the seed form of the + private key, never the expanded one. + """ + + @abc.abstractmethod + def private_bytes_raw(self) -> bytes: + """ + The raw bytes of the private key. + Equivalent to private_bytes(Raw, Raw, NoEncryption()). + + This method only returns the seed form of the private key (32 bytes). + """ + + @abc.abstractmethod + def sign(self, data: Buffer, context: Buffer | None = None) -> bytes: + """ + Signs the data. + """ + + @abc.abstractmethod + def sign_mu(self, mu: Buffer) -> bytes: + """ + Signs a precomputed mu (message representative). + + mu must be 64 bytes and already incorporates the context, so no + context is accepted here. + """ + + @abc.abstractmethod + def __copy__(self) -> MLDSA87PrivateKey: + """ + Returns a copy. + """ + + @abc.abstractmethod + def __deepcopy__(self, memo: dict) -> MLDSA87PrivateKey: + """ + Returns a deep copy. + """ + + +if hasattr(rust_openssl, "mldsa"): + MLDSA87PrivateKey.register(rust_openssl.mldsa.MLDSA87PrivateKey) + MLDSAMuHasher = rust_openssl.mldsa.MLDSAMuHasher diff --git a/py_modules/cryptography/hazmat/primitives/asymmetric/mlkem.py b/py_modules/cryptography/hazmat/primitives/asymmetric/mlkem.py new file mode 100644 index 0000000..64bca11 --- /dev/null +++ b/py_modules/cryptography/hazmat/primitives/asymmetric/mlkem.py @@ -0,0 +1,278 @@ +# This file is dual licensed under the terms of the Apache License, Version +# 2.0, and the BSD License. See the LICENSE file in the root of this repository +# for complete details. + +from __future__ import annotations + +import abc + +from cryptography.exceptions import UnsupportedAlgorithm, _Reasons +from cryptography.hazmat.bindings._rust import openssl as rust_openssl +from cryptography.hazmat.primitives import _serialization +from cryptography.utils import Buffer + + +class MLKEM768PublicKey(metaclass=abc.ABCMeta): + @classmethod + def from_public_bytes(cls, data: Buffer) -> MLKEM768PublicKey: + from cryptography.hazmat.backends.openssl.backend import backend + + if not backend.mlkem_supported(): + raise UnsupportedAlgorithm( + "ML-KEM-768 is not supported by this backend.", + _Reasons.UNSUPPORTED_PUBLIC_KEY_ALGORITHM, + ) + + return rust_openssl.mlkem.from_mlkem768_public_bytes(data) + + @abc.abstractmethod + def encapsulate(self) -> tuple[bytes, bytes]: + """ + Encapsulate: returns (shared_secret, ciphertext). + """ + + @abc.abstractmethod + def public_bytes( + self, + encoding: _serialization.Encoding, + format: _serialization.PublicFormat, + ) -> bytes: + """ + The serialized bytes of the public key. + """ + + @abc.abstractmethod + def public_bytes_raw(self) -> bytes: + """ + The raw bytes of the public key. + Equivalent to public_bytes(Raw, Raw). + + The public key is 1,184 bytes for ML-KEM-768. + """ + + @abc.abstractmethod + def __eq__(self, other: object) -> bool: + """ + Checks equality. + """ + + @abc.abstractmethod + def __copy__(self) -> MLKEM768PublicKey: + """ + Returns a copy. + """ + + @abc.abstractmethod + def __deepcopy__(self, memo: dict) -> MLKEM768PublicKey: + """ + Returns a deep copy. + """ + + +if hasattr(rust_openssl, "mlkem"): + MLKEM768PublicKey.register(rust_openssl.mlkem.MLKEM768PublicKey) + + +class MLKEM768PrivateKey(metaclass=abc.ABCMeta): + @classmethod + def generate(cls) -> MLKEM768PrivateKey: + from cryptography.hazmat.backends.openssl.backend import backend + + if not backend.mlkem_supported(): + raise UnsupportedAlgorithm( + "ML-KEM-768 is not supported by this backend.", + _Reasons.UNSUPPORTED_PUBLIC_KEY_ALGORITHM, + ) + + return rust_openssl.mlkem.generate_mlkem768_key() + + @classmethod + def from_seed_bytes(cls, data: Buffer) -> MLKEM768PrivateKey: + from cryptography.hazmat.backends.openssl.backend import backend + + if not backend.mlkem_supported(): + raise UnsupportedAlgorithm( + "ML-KEM-768 is not supported by this backend.", + _Reasons.UNSUPPORTED_PUBLIC_KEY_ALGORITHM, + ) + + return rust_openssl.mlkem.from_mlkem768_seed_bytes(data) + + @abc.abstractmethod + def decapsulate(self, ciphertext: Buffer) -> bytes: + """ + Decapsulate: returns shared_secret. + """ + + @abc.abstractmethod + def public_key(self) -> MLKEM768PublicKey: + """ + The MLKEM768PublicKey derived from this private key. + """ + + @abc.abstractmethod + def private_bytes( + self, + encoding: _serialization.Encoding, + format: _serialization.PrivateFormat, + encryption_algorithm: _serialization.KeySerializationEncryption, + ) -> bytes: + """ + The serialized bytes of the private key. + """ + + @abc.abstractmethod + def private_bytes_raw(self) -> bytes: + """ + The raw bytes of the private key (64-byte seed). + Equivalent to private_bytes(Raw, Raw, NoEncryption()). + """ + + @abc.abstractmethod + def __copy__(self) -> MLKEM768PrivateKey: + """ + Returns a copy. + """ + + @abc.abstractmethod + def __deepcopy__(self, memo: dict) -> MLKEM768PrivateKey: + """ + Returns a deep copy. + """ + + +if hasattr(rust_openssl, "mlkem"): + MLKEM768PrivateKey.register(rust_openssl.mlkem.MLKEM768PrivateKey) + + +class MLKEM1024PublicKey(metaclass=abc.ABCMeta): + @classmethod + def from_public_bytes(cls, data: Buffer) -> MLKEM1024PublicKey: + from cryptography.hazmat.backends.openssl.backend import backend + + if not backend.mlkem_supported(): + raise UnsupportedAlgorithm( + "ML-KEM-1024 is not supported by this backend.", + _Reasons.UNSUPPORTED_PUBLIC_KEY_ALGORITHM, + ) + + return rust_openssl.mlkem.from_mlkem1024_public_bytes(data) + + @abc.abstractmethod + def encapsulate(self) -> tuple[bytes, bytes]: + """ + Encapsulate: returns (shared_secret, ciphertext). + """ + + @abc.abstractmethod + def public_bytes( + self, + encoding: _serialization.Encoding, + format: _serialization.PublicFormat, + ) -> bytes: + """ + The serialized bytes of the public key. + """ + + @abc.abstractmethod + def public_bytes_raw(self) -> bytes: + """ + The raw bytes of the public key. + Equivalent to public_bytes(Raw, Raw). + + The public key is 1,568 bytes for ML-KEM-1024. + """ + + @abc.abstractmethod + def __eq__(self, other: object) -> bool: + """ + Checks equality. + """ + + @abc.abstractmethod + def __copy__(self) -> MLKEM1024PublicKey: + """ + Returns a copy. + """ + + @abc.abstractmethod + def __deepcopy__(self, memo: dict) -> MLKEM1024PublicKey: + """ + Returns a deep copy. + """ + + +if hasattr(rust_openssl, "mlkem"): + MLKEM1024PublicKey.register(rust_openssl.mlkem.MLKEM1024PublicKey) + + +class MLKEM1024PrivateKey(metaclass=abc.ABCMeta): + @classmethod + def generate(cls) -> MLKEM1024PrivateKey: + from cryptography.hazmat.backends.openssl.backend import backend + + if not backend.mlkem_supported(): + raise UnsupportedAlgorithm( + "ML-KEM-1024 is not supported by this backend.", + _Reasons.UNSUPPORTED_PUBLIC_KEY_ALGORITHM, + ) + + return rust_openssl.mlkem.generate_mlkem1024_key() + + @classmethod + def from_seed_bytes(cls, data: Buffer) -> MLKEM1024PrivateKey: + from cryptography.hazmat.backends.openssl.backend import backend + + if not backend.mlkem_supported(): + raise UnsupportedAlgorithm( + "ML-KEM-1024 is not supported by this backend.", + _Reasons.UNSUPPORTED_PUBLIC_KEY_ALGORITHM, + ) + + return rust_openssl.mlkem.from_mlkem1024_seed_bytes(data) + + @abc.abstractmethod + def decapsulate(self, ciphertext: Buffer) -> bytes: + """ + Decapsulate: returns shared_secret. + """ + + @abc.abstractmethod + def public_key(self) -> MLKEM1024PublicKey: + """ + The MLKEM1024PublicKey derived from this private key. + """ + + @abc.abstractmethod + def private_bytes( + self, + encoding: _serialization.Encoding, + format: _serialization.PrivateFormat, + encryption_algorithm: _serialization.KeySerializationEncryption, + ) -> bytes: + """ + The serialized bytes of the private key. + """ + + @abc.abstractmethod + def private_bytes_raw(self) -> bytes: + """ + The raw bytes of the private key (64-byte seed). + Equivalent to private_bytes(Raw, Raw, NoEncryption()). + """ + + @abc.abstractmethod + def __copy__(self) -> MLKEM1024PrivateKey: + """ + Returns a copy. + """ + + @abc.abstractmethod + def __deepcopy__(self, memo: dict) -> MLKEM1024PrivateKey: + """ + Returns a deep copy. + """ + + +if hasattr(rust_openssl, "mlkem"): + MLKEM1024PrivateKey.register(rust_openssl.mlkem.MLKEM1024PrivateKey) diff --git a/py_modules/cryptography/hazmat/primitives/asymmetric/padding.py b/py_modules/cryptography/hazmat/primitives/asymmetric/padding.py new file mode 100644 index 0000000..1fd2873 --- /dev/null +++ b/py_modules/cryptography/hazmat/primitives/asymmetric/padding.py @@ -0,0 +1,141 @@ +# This file is dual licensed under the terms of the Apache License, Version +# 2.0, and the BSD License. See the LICENSE file in the root of this repository +# for complete details. + +from __future__ import annotations + +import abc + +from cryptography.hazmat.primitives import hashes +from cryptography.hazmat.primitives._asymmetric import ( + AsymmetricPadding as AsymmetricPadding, +) +from cryptography.hazmat.primitives.asymmetric import rsa + + +class PKCS1v15(AsymmetricPadding): + name = "EMSA-PKCS1-v1_5" + + def __eq__(self, other: object) -> bool: + if not isinstance(other, PKCS1v15): + return NotImplemented + + return True + + +class _MaxLength: + "Sentinel value for `MAX_LENGTH`." + + +class _Auto: + "Sentinel value for `AUTO`." + + +class _DigestLength: + "Sentinel value for `DIGEST_LENGTH`." + + +class PSS(AsymmetricPadding): + MAX_LENGTH = _MaxLength() + AUTO = _Auto() + DIGEST_LENGTH = _DigestLength() + name = "EMSA-PSS" + _salt_length: int | _MaxLength | _Auto | _DigestLength + + def __init__( + self, + mgf: MGF, + salt_length: int | _MaxLength | _Auto | _DigestLength, + ) -> None: + self._mgf = mgf + + if not isinstance( + salt_length, (int, _MaxLength, _Auto, _DigestLength) + ): + raise TypeError( + "salt_length must be an integer, MAX_LENGTH, " + "DIGEST_LENGTH, or AUTO" + ) + + if isinstance(salt_length, int) and salt_length < 0: + raise ValueError("salt_length must be zero or greater.") + + self._salt_length = salt_length + + def __eq__(self, other: object) -> bool: + if not isinstance(other, PSS): + return NotImplemented + + return ( + self._mgf == other._mgf and self._salt_length == other._salt_length + ) + + @property + def mgf(self) -> MGF: + return self._mgf + + +class OAEP(AsymmetricPadding): + name = "EME-OAEP" + + def __init__( + self, + mgf: MGF, + algorithm: hashes.HashAlgorithm, + label: bytes | None, + ): + if not isinstance(algorithm, hashes.HashAlgorithm): + raise TypeError("Expected instance of hashes.HashAlgorithm.") + + self._mgf = mgf + self._algorithm = algorithm + self._label = label + + def __eq__(self, other: object) -> bool: + if not isinstance(other, OAEP): + return NotImplemented + + return ( + self._mgf == other._mgf + and self._algorithm == other._algorithm + and self._label == other._label + ) + + @property + def algorithm(self) -> hashes.HashAlgorithm: + return self._algorithm + + @property + def mgf(self) -> MGF: + return self._mgf + + +class MGF(metaclass=abc.ABCMeta): + _algorithm: hashes.HashAlgorithm + + +class MGF1(MGF): + def __init__(self, algorithm: hashes.HashAlgorithm): + if not isinstance(algorithm, hashes.HashAlgorithm): + raise TypeError("Expected instance of hashes.HashAlgorithm.") + + self._algorithm = algorithm + + def __eq__(self, other: object) -> bool: + if not isinstance(other, MGF1): + return NotImplemented + + return self._algorithm == other._algorithm + + +def calculate_max_pss_salt_length( + key: rsa.RSAPrivateKey | rsa.RSAPublicKey, + hash_algorithm: hashes.HashAlgorithm, +) -> int: + if not isinstance(key, (rsa.RSAPrivateKey, rsa.RSAPublicKey)): + raise TypeError("key must be an RSA public or private key") + # bit length - 1 per RFC 3447 + emlen = (key.key_size + 6) // 8 + salt_length = emlen - hash_algorithm.digest_size - 2 + assert salt_length >= 0 + return salt_length diff --git a/py_modules/cryptography/hazmat/primitives/asymmetric/rsa.py b/py_modules/cryptography/hazmat/primitives/asymmetric/rsa.py new file mode 100644 index 0000000..d730ceb --- /dev/null +++ b/py_modules/cryptography/hazmat/primitives/asymmetric/rsa.py @@ -0,0 +1,295 @@ +# This file is dual licensed under the terms of the Apache License, Version +# 2.0, and the BSD License. See the LICENSE file in the root of this repository +# for complete details. + +from __future__ import annotations + +import abc +import random +import typing +from math import gcd, lcm + +from cryptography.hazmat.bindings._rust import openssl as rust_openssl +from cryptography.hazmat.primitives import _serialization, hashes +from cryptography.hazmat.primitives._asymmetric import AsymmetricPadding +from cryptography.hazmat.primitives.asymmetric import utils as asym_utils + + +class RSAPrivateKey(metaclass=abc.ABCMeta): + @abc.abstractmethod + def decrypt(self, ciphertext: bytes, padding: AsymmetricPadding) -> bytes: + """ + Decrypts the provided ciphertext. + """ + + @property + @abc.abstractmethod + def key_size(self) -> int: + """ + The bit length of the public modulus. + """ + + @abc.abstractmethod + def public_key(self) -> RSAPublicKey: + """ + The RSAPublicKey associated with this private key. + """ + + @abc.abstractmethod + def sign( + self, + data: bytes, + padding: AsymmetricPadding, + algorithm: asym_utils.Prehashed + | hashes.HashAlgorithm + | asym_utils.NoDigestInfo, + ) -> bytes: + """ + Signs the data. + """ + + @abc.abstractmethod + def private_numbers(self) -> RSAPrivateNumbers: + """ + Returns an RSAPrivateNumbers. + """ + + @abc.abstractmethod + def private_bytes( + self, + encoding: _serialization.Encoding, + format: _serialization.PrivateFormat, + encryption_algorithm: _serialization.KeySerializationEncryption, + ) -> bytes: + """ + Returns the key serialized as bytes. + """ + + @abc.abstractmethod + def __copy__(self) -> RSAPrivateKey: + """ + Returns a copy. + """ + + @abc.abstractmethod + def __deepcopy__(self, memo: dict) -> RSAPrivateKey: + """ + Returns a deep copy. + """ + + +RSAPrivateKeyWithSerialization = RSAPrivateKey +RSAPrivateKey.register(rust_openssl.rsa.RSAPrivateKey) + + +class RSAPublicKey(metaclass=abc.ABCMeta): + @abc.abstractmethod + def encrypt(self, plaintext: bytes, padding: AsymmetricPadding) -> bytes: + """ + Encrypts the given plaintext. + """ + + @property + @abc.abstractmethod + def key_size(self) -> int: + """ + The bit length of the public modulus. + """ + + @abc.abstractmethod + def public_numbers(self) -> RSAPublicNumbers: + """ + Returns an RSAPublicNumbers + """ + + @abc.abstractmethod + def public_bytes( + self, + encoding: _serialization.Encoding, + format: _serialization.PublicFormat, + ) -> bytes: + """ + Returns the key serialized as bytes. + """ + + @abc.abstractmethod + def verify( + self, + signature: bytes, + data: bytes, + padding: AsymmetricPadding, + algorithm: asym_utils.Prehashed | hashes.HashAlgorithm, + ) -> None: + """ + Verifies the signature of the data. + """ + + @abc.abstractmethod + def recover_data_from_signature( + self, + signature: bytes, + padding: AsymmetricPadding, + algorithm: hashes.HashAlgorithm | asym_utils.NoDigestInfo | None, + ) -> bytes: + """ + Recovers the original data from the signature. + """ + + @abc.abstractmethod + def __eq__(self, other: object) -> bool: + """ + Checks equality. + """ + + @abc.abstractmethod + def __copy__(self) -> RSAPublicKey: + """ + Returns a copy. + """ + + @abc.abstractmethod + def __deepcopy__(self, memo: dict) -> RSAPublicKey: + """ + Returns a deep copy. + """ + + +RSAPublicKeyWithSerialization = RSAPublicKey +RSAPublicKey.register(rust_openssl.rsa.RSAPublicKey) + +RSAPrivateNumbers = rust_openssl.rsa.RSAPrivateNumbers +RSAPublicNumbers = rust_openssl.rsa.RSAPublicNumbers + + +def generate_private_key( + public_exponent: int, + key_size: int, + backend: typing.Any = None, +) -> RSAPrivateKey: + _verify_rsa_parameters(public_exponent, key_size) + return rust_openssl.rsa.generate_private_key(public_exponent, key_size) + + +def _verify_rsa_parameters(public_exponent: int, key_size: int) -> None: + if public_exponent not in (3, 65537): + raise ValueError( + "public_exponent must be either 3 (for legacy compatibility) or " + "65537. Almost everyone should choose 65537 here!" + ) + + if key_size < 1024: + raise ValueError("key_size must be at least 1024-bits.") + + +def _modinv(e: int, m: int) -> int: + """ + Modular Multiplicative Inverse. Returns x such that: (x*e) mod m == 1 + """ + x1, x2 = 1, 0 + a, b = e, m + while b > 0: + q, r = divmod(a, b) + xn = x1 - q * x2 + a, b, x1, x2 = b, r, x2, xn + return x1 % m + + +def rsa_crt_iqmp(p: int, q: int) -> int: + """ + Compute the CRT (q ** -1) % p value from RSA primes p and q. + """ + if p <= 1 or q <= 1: + raise ValueError("Values can't be <= 1") + return _modinv(q, p) + + +def rsa_crt_dmp1(private_exponent: int, p: int) -> int: + """ + Compute the CRT private_exponent % (p - 1) value from the RSA + private_exponent (d) and p. + """ + if private_exponent <= 1 or p <= 1: + raise ValueError("Values can't be <= 1") + return private_exponent % (p - 1) + + +def rsa_crt_dmq1(private_exponent: int, q: int) -> int: + """ + Compute the CRT private_exponent % (q - 1) value from the RSA + private_exponent (d) and q. + """ + if private_exponent <= 1 or q <= 1: + raise ValueError("Values can't be <= 1") + return private_exponent % (q - 1) + + +def rsa_recover_private_exponent(e: int, p: int, q: int) -> int: + """ + Compute the RSA private_exponent (d) given the public exponent (e) + and the RSA primes p and q. + + This uses the Carmichael totient function to generate the + smallest possible working value of the private exponent. + """ + # This lambda_n is the Carmichael totient function. + # The original RSA paper uses the Euler totient function + # here: phi_n = (p - 1) * (q - 1) + # Either version of the private exponent will work, but the + # one generated by the older formulation may be larger + # than necessary. (lambda_n always divides phi_n) + if e <= 1 or p <= 1 or q <= 1: + raise ValueError("Values can't be <= 1") + return _modinv(e, lcm(p - 1, q - 1)) + + +# Controls the number of iterations rsa_recover_prime_factors will perform +# to obtain the prime factors. +_MAX_RECOVERY_ATTEMPTS = 500 + + +def rsa_recover_prime_factors(n: int, e: int, d: int) -> tuple[int, int]: + """ + Compute factors p and q from the private exponent d. We assume that n has + no more than two factors. This function is adapted from code in PyCrypto. + """ + # reject invalid values early + if d <= 1 or e <= 1: + raise ValueError("d, e can't be <= 1") + if 17 != pow(17, e * d, n): + raise ValueError("n, d, e don't match") + # See 8.2.2(i) in Handbook of Applied Cryptography. + ktot = d * e - 1 + # The quantity d*e-1 is a multiple of phi(n), even, + # and can be represented as t*2^s. + t = ktot + while t % 2 == 0: + t = t // 2 + # Cycle through all multiplicative inverses in Zn. + # The algorithm is non-deterministic, but there is a 50% chance + # any candidate a leads to successful factoring. + # See "Digitalized Signatures and Public Key Functions as Intractable + # as Factorization", M. Rabin, 1979 + spotted = False + tries = 0 + while not spotted and tries < _MAX_RECOVERY_ATTEMPTS: + a = random.randint(2, n - 1) + tries += 1 + k = t + # Cycle through all values a^{t*2^i}=a^k + while k < ktot: + cand = pow(a, k, n) + # Check if a^k is a non-trivial root of unity (mod n) + if cand != 1 and cand != (n - 1) and pow(cand, 2, n) == 1: + # We have found a number such that (cand-1)(cand+1)=0 (mod n). + # Either of the terms divides n. + p = gcd(cand + 1, n) + spotted = True + break + k *= 2 + if not spotted: + raise ValueError("Unable to compute factors p and q from exponent d.") + # Found ! + q, r = divmod(n, p) + assert r == 0 + p, q = sorted((p, q), reverse=True) + return (p, q) diff --git a/py_modules/cryptography/hazmat/primitives/asymmetric/types.py b/py_modules/cryptography/hazmat/primitives/asymmetric/types.py new file mode 100644 index 0000000..dbf4b74 --- /dev/null +++ b/py_modules/cryptography/hazmat/primitives/asymmetric/types.py @@ -0,0 +1,94 @@ +# This file is dual licensed under the terms of the Apache License, Version +# 2.0, and the BSD License. See the LICENSE file in the root of this repository +# for complete details. + +from __future__ import annotations + +import typing + +from cryptography.hazmat.primitives.asymmetric import ( + dh, + dsa, + ec, + ed448, + ed25519, + mldsa, + mlkem, + rsa, + x448, + x25519, +) + +# Every asymmetric key type. These use the private DH aliases so that +# importing this module doesn't trigger the FFDH deprecation warning. +PublicKeyTypes = typing.Union[ + dh._DHPublicKey, + dsa.DSAPublicKey, + rsa.RSAPublicKey, + ec.EllipticCurvePublicKey, + ed25519.Ed25519PublicKey, + ed448.Ed448PublicKey, + mldsa.MLDSA44PublicKey, + mldsa.MLDSA65PublicKey, + mldsa.MLDSA87PublicKey, + mlkem.MLKEM768PublicKey, + mlkem.MLKEM1024PublicKey, + x25519.X25519PublicKey, + x448.X448PublicKey, +] +# Every asymmetric key type +PrivateKeyTypes = typing.Union[ + dh._DHPrivateKey, + ed25519.Ed25519PrivateKey, + ed448.Ed448PrivateKey, + mldsa.MLDSA44PrivateKey, + mldsa.MLDSA65PrivateKey, + mldsa.MLDSA87PrivateKey, + mlkem.MLKEM768PrivateKey, + mlkem.MLKEM1024PrivateKey, + rsa.RSAPrivateKey, + dsa.DSAPrivateKey, + ec.EllipticCurvePrivateKey, + x25519.X25519PrivateKey, + x448.X448PrivateKey, +] +# Just the key types we allow to be used for x509 signing. This mirrors +# the certificate public key types +CertificateIssuerPrivateKeyTypes = typing.Union[ + ed25519.Ed25519PrivateKey, + ed448.Ed448PrivateKey, + rsa.RSAPrivateKey, + dsa.DSAPrivateKey, + ec.EllipticCurvePrivateKey, + mldsa.MLDSA44PrivateKey, + mldsa.MLDSA65PrivateKey, + mldsa.MLDSA87PrivateKey, +] +# Just the key types we allow to be used for x509 signing. This mirrors +# the certificate private key types +CertificateIssuerPublicKeyTypes = typing.Union[ + dsa.DSAPublicKey, + rsa.RSAPublicKey, + ec.EllipticCurvePublicKey, + ed25519.Ed25519PublicKey, + ed448.Ed448PublicKey, + mldsa.MLDSA44PublicKey, + mldsa.MLDSA65PublicKey, + mldsa.MLDSA87PublicKey, +] +# This type removes DHPublicKey. x448/x25519/mlkem can be a public key +# but cannot be used in signing so they are allowed here. +CertificatePublicKeyTypes = typing.Union[ + dsa.DSAPublicKey, + rsa.RSAPublicKey, + ec.EllipticCurvePublicKey, + ed25519.Ed25519PublicKey, + ed448.Ed448PublicKey, + mldsa.MLDSA44PublicKey, + mldsa.MLDSA65PublicKey, + mldsa.MLDSA87PublicKey, + mlkem.MLKEM768PublicKey, + mlkem.MLKEM1024PublicKey, + x25519.X25519PublicKey, + x448.X448PublicKey, +] diff --git a/py_modules/cryptography/hazmat/primitives/asymmetric/utils.py b/py_modules/cryptography/hazmat/primitives/asymmetric/utils.py new file mode 100644 index 0000000..c01c342 --- /dev/null +++ b/py_modules/cryptography/hazmat/primitives/asymmetric/utils.py @@ -0,0 +1,28 @@ +# This file is dual licensed under the terms of the Apache License, Version +# 2.0, and the BSD License. See the LICENSE file in the root of this repository +# for complete details. + +from __future__ import annotations + +from cryptography.hazmat.bindings._rust import asn1 +from cryptography.hazmat.primitives import hashes + +decode_dss_signature = asn1.decode_dss_signature +encode_dss_signature = asn1.encode_dss_signature + + +class NoDigestInfo: + pass + + +class Prehashed: + def __init__(self, algorithm: hashes.HashAlgorithm): + if not isinstance(algorithm, hashes.HashAlgorithm): + raise TypeError("Expected instance of HashAlgorithm.") + + self._algorithm = algorithm + self._digest_size = algorithm.digest_size + + @property + def digest_size(self) -> int: + return self._digest_size diff --git a/py_modules/cryptography/hazmat/primitives/asymmetric/x25519.py b/py_modules/cryptography/hazmat/primitives/asymmetric/x25519.py new file mode 100644 index 0000000..7498998 --- /dev/null +++ b/py_modules/cryptography/hazmat/primitives/asymmetric/x25519.py @@ -0,0 +1,134 @@ +# This file is dual licensed under the terms of the Apache License, Version +# 2.0, and the BSD License. See the LICENSE file in the root of this repository +# for complete details. + +from __future__ import annotations + +import abc + +from cryptography.exceptions import UnsupportedAlgorithm, _Reasons +from cryptography.hazmat.bindings._rust import openssl as rust_openssl +from cryptography.hazmat.primitives import _serialization +from cryptography.utils import Buffer + + +class X25519PublicKey(metaclass=abc.ABCMeta): + @classmethod + def from_public_bytes(cls, data: bytes) -> X25519PublicKey: + from cryptography.hazmat.backends.openssl.backend import backend + + if not backend.x25519_supported(): + raise UnsupportedAlgorithm( + "X25519 is not supported by this version of OpenSSL.", + _Reasons.UNSUPPORTED_EXCHANGE_ALGORITHM, + ) + + return rust_openssl.x25519.from_public_bytes(data) + + @abc.abstractmethod + def public_bytes( + self, + encoding: _serialization.Encoding, + format: _serialization.PublicFormat, + ) -> bytes: + """ + The serialized bytes of the public key. + """ + + @abc.abstractmethod + def public_bytes_raw(self) -> bytes: + """ + The raw bytes of the public key. + Equivalent to public_bytes(Raw, Raw). + """ + + @abc.abstractmethod + def __eq__(self, other: object) -> bool: + """ + Checks equality. + """ + + @abc.abstractmethod + def __copy__(self) -> X25519PublicKey: + """ + Returns a copy. + """ + + @abc.abstractmethod + def __deepcopy__(self, memo: dict) -> X25519PublicKey: + """ + Returns a deep copy. + """ + + +X25519PublicKey.register(rust_openssl.x25519.X25519PublicKey) + + +class X25519PrivateKey(metaclass=abc.ABCMeta): + @classmethod + def generate(cls) -> X25519PrivateKey: + from cryptography.hazmat.backends.openssl.backend import backend + + if not backend.x25519_supported(): + raise UnsupportedAlgorithm( + "X25519 is not supported by this version of OpenSSL.", + _Reasons.UNSUPPORTED_EXCHANGE_ALGORITHM, + ) + return rust_openssl.x25519.generate_key() + + @classmethod + def from_private_bytes(cls, data: Buffer) -> X25519PrivateKey: + from cryptography.hazmat.backends.openssl.backend import backend + + if not backend.x25519_supported(): + raise UnsupportedAlgorithm( + "X25519 is not supported by this version of OpenSSL.", + _Reasons.UNSUPPORTED_EXCHANGE_ALGORITHM, + ) + + return rust_openssl.x25519.from_private_bytes(data) + + @abc.abstractmethod + def public_key(self) -> X25519PublicKey: + """ + Returns the public key associated with this private key + """ + + @abc.abstractmethod + def private_bytes( + self, + encoding: _serialization.Encoding, + format: _serialization.PrivateFormat, + encryption_algorithm: _serialization.KeySerializationEncryption, + ) -> bytes: + """ + The serialized bytes of the private key. + """ + + @abc.abstractmethod + def private_bytes_raw(self) -> bytes: + """ + The raw bytes of the private key. + Equivalent to private_bytes(Raw, Raw, NoEncryption()). + """ + + @abc.abstractmethod + def exchange(self, peer_public_key: X25519PublicKey) -> bytes: + """ + Performs a key exchange operation using the provided peer's public key. + """ + + @abc.abstractmethod + def __copy__(self) -> X25519PrivateKey: + """ + Returns a copy. + """ + + @abc.abstractmethod + def __deepcopy__(self, memo: dict) -> X25519PrivateKey: + """ + Returns a deep copy. + """ + + +X25519PrivateKey.register(rust_openssl.x25519.X25519PrivateKey) diff --git a/py_modules/cryptography/hazmat/primitives/asymmetric/x448.py b/py_modules/cryptography/hazmat/primitives/asymmetric/x448.py new file mode 100644 index 0000000..b9dc826 --- /dev/null +++ b/py_modules/cryptography/hazmat/primitives/asymmetric/x448.py @@ -0,0 +1,137 @@ +# This file is dual licensed under the terms of the Apache License, Version +# 2.0, and the BSD License. See the LICENSE file in the root of this repository +# for complete details. + +from __future__ import annotations + +import abc + +from cryptography.exceptions import UnsupportedAlgorithm, _Reasons +from cryptography.hazmat.bindings._rust import openssl as rust_openssl +from cryptography.hazmat.primitives import _serialization +from cryptography.utils import Buffer + + +class X448PublicKey(metaclass=abc.ABCMeta): + @classmethod + def from_public_bytes(cls, data: bytes) -> X448PublicKey: + from cryptography.hazmat.backends.openssl.backend import backend + + if not backend.x448_supported(): + raise UnsupportedAlgorithm( + "X448 is not supported by this version of OpenSSL.", + _Reasons.UNSUPPORTED_EXCHANGE_ALGORITHM, + ) + + return rust_openssl.x448.from_public_bytes(data) + + @abc.abstractmethod + def public_bytes( + self, + encoding: _serialization.Encoding, + format: _serialization.PublicFormat, + ) -> bytes: + """ + The serialized bytes of the public key. + """ + + @abc.abstractmethod + def public_bytes_raw(self) -> bytes: + """ + The raw bytes of the public key. + Equivalent to public_bytes(Raw, Raw). + """ + + @abc.abstractmethod + def __eq__(self, other: object) -> bool: + """ + Checks equality. + """ + + @abc.abstractmethod + def __copy__(self) -> X448PublicKey: + """ + Returns a copy. + """ + + @abc.abstractmethod + def __deepcopy__(self, memo: dict) -> X448PublicKey: + """ + Returns a deep copy. + """ + + +if hasattr(rust_openssl, "x448"): + X448PublicKey.register(rust_openssl.x448.X448PublicKey) + + +class X448PrivateKey(metaclass=abc.ABCMeta): + @classmethod + def generate(cls) -> X448PrivateKey: + from cryptography.hazmat.backends.openssl.backend import backend + + if not backend.x448_supported(): + raise UnsupportedAlgorithm( + "X448 is not supported by this version of OpenSSL.", + _Reasons.UNSUPPORTED_EXCHANGE_ALGORITHM, + ) + + return rust_openssl.x448.generate_key() + + @classmethod + def from_private_bytes(cls, data: Buffer) -> X448PrivateKey: + from cryptography.hazmat.backends.openssl.backend import backend + + if not backend.x448_supported(): + raise UnsupportedAlgorithm( + "X448 is not supported by this version of OpenSSL.", + _Reasons.UNSUPPORTED_EXCHANGE_ALGORITHM, + ) + + return rust_openssl.x448.from_private_bytes(data) + + @abc.abstractmethod + def public_key(self) -> X448PublicKey: + """ + Returns the public key associated with this private key + """ + + @abc.abstractmethod + def private_bytes( + self, + encoding: _serialization.Encoding, + format: _serialization.PrivateFormat, + encryption_algorithm: _serialization.KeySerializationEncryption, + ) -> bytes: + """ + The serialized bytes of the private key. + """ + + @abc.abstractmethod + def private_bytes_raw(self) -> bytes: + """ + The raw bytes of the private key. + Equivalent to private_bytes(Raw, Raw, NoEncryption()). + """ + + @abc.abstractmethod + def exchange(self, peer_public_key: X448PublicKey) -> bytes: + """ + Performs a key exchange operation using the provided peer's public key. + """ + + @abc.abstractmethod + def __copy__(self) -> X448PrivateKey: + """ + Returns a copy. + """ + + @abc.abstractmethod + def __deepcopy__(self, memo: dict) -> X448PrivateKey: + """ + Returns a deep copy. + """ + + +if hasattr(rust_openssl, "x448"): + X448PrivateKey.register(rust_openssl.x448.X448PrivateKey) diff --git a/py_modules/cryptography/hazmat/primitives/ciphers/__init__.py b/py_modules/cryptography/hazmat/primitives/ciphers/__init__.py new file mode 100644 index 0000000..10c15d0 --- /dev/null +++ b/py_modules/cryptography/hazmat/primitives/ciphers/__init__.py @@ -0,0 +1,27 @@ +# This file is dual licensed under the terms of the Apache License, Version +# 2.0, and the BSD License. See the LICENSE file in the root of this repository +# for complete details. + +from __future__ import annotations + +from cryptography.hazmat.primitives._cipheralgorithm import ( + BlockCipherAlgorithm, + CipherAlgorithm, +) +from cryptography.hazmat.primitives.ciphers.base import ( + AEADCipherContext, + AEADDecryptionContext, + AEADEncryptionContext, + Cipher, + CipherContext, +) + +__all__ = [ + "AEADCipherContext", + "AEADDecryptionContext", + "AEADEncryptionContext", + "BlockCipherAlgorithm", + "Cipher", + "CipherAlgorithm", + "CipherContext", +] diff --git a/py_modules/cryptography/hazmat/primitives/ciphers/aead.py b/py_modules/cryptography/hazmat/primitives/ciphers/aead.py new file mode 100644 index 0000000..c8a582d --- /dev/null +++ b/py_modules/cryptography/hazmat/primitives/ciphers/aead.py @@ -0,0 +1,23 @@ +# This file is dual licensed under the terms of the Apache License, Version +# 2.0, and the BSD License. See the LICENSE file in the root of this repository +# for complete details. + +from __future__ import annotations + +from cryptography.hazmat.bindings._rust import openssl as rust_openssl + +__all__ = [ + "AESCCM", + "AESGCM", + "AESGCMSIV", + "AESOCB3", + "AESSIV", + "ChaCha20Poly1305", +] + +AESGCM = rust_openssl.aead.AESGCM +ChaCha20Poly1305 = rust_openssl.aead.ChaCha20Poly1305 +AESCCM = rust_openssl.aead.AESCCM +AESSIV = rust_openssl.aead.AESSIV +AESOCB3 = rust_openssl.aead.AESOCB3 +AESGCMSIV = rust_openssl.aead.AESGCMSIV diff --git a/py_modules/cryptography/hazmat/primitives/ciphers/algorithms.py b/py_modules/cryptography/hazmat/primitives/ciphers/algorithms.py new file mode 100644 index 0000000..6b20dd5 --- /dev/null +++ b/py_modules/cryptography/hazmat/primitives/ciphers/algorithms.py @@ -0,0 +1,138 @@ +# This file is dual licensed under the terms of the Apache License, Version +# 2.0, and the BSD License. See the LICENSE file in the root of this repository +# for complete details. + +from __future__ import annotations + +from cryptography import utils +from cryptography.hazmat.decrepit.ciphers.algorithms import ( + ARC4 as ARC4, +) +from cryptography.hazmat.decrepit.ciphers.algorithms import ( + CAST5 as CAST5, +) +from cryptography.hazmat.decrepit.ciphers.algorithms import ( + IDEA as IDEA, +) +from cryptography.hazmat.decrepit.ciphers.algorithms import ( + SEED as SEED, +) +from cryptography.hazmat.decrepit.ciphers.algorithms import ( + Blowfish as Blowfish, +) +from cryptography.hazmat.decrepit.ciphers.algorithms import ( + Camellia as Camellia, +) +from cryptography.hazmat.decrepit.ciphers.algorithms import ( + TripleDES as TripleDES, +) +from cryptography.hazmat.primitives._cipheralgorithm import _verify_key_size +from cryptography.hazmat.primitives.ciphers import ( + BlockCipherAlgorithm, + CipherAlgorithm, +) + + +class AES(BlockCipherAlgorithm): + name = "AES" + block_size = 128 + # 512 added to support AES-256-XTS, which uses 512-bit keys + key_sizes = frozenset([128, 192, 256, 512]) + + def __init__(self, key: utils.Buffer): + self.key = _verify_key_size(self, key) + + @property + def key_size(self) -> int: + return len(self.key) * 8 + + +class AES128(BlockCipherAlgorithm): + name = "AES" + block_size = 128 + key_sizes = frozenset([128]) + key_size = 128 + + def __init__(self, key: utils.Buffer): + self.key = _verify_key_size(self, key) + + +class AES256(BlockCipherAlgorithm): + name = "AES" + block_size = 128 + key_sizes = frozenset([256]) + key_size = 256 + + def __init__(self, key: utils.Buffer): + self.key = _verify_key_size(self, key) + + +utils.deprecated( + Camellia, + __name__, + "Camellia has been moved to " + "cryptography.hazmat.decrepit.ciphers.algorithms.Camellia and " + "will be removed from " + "cryptography.hazmat.primitives.ciphers.algorithms in 49.0.0.", + utils.DeprecatedIn43, + name="Camellia", +) + + +utils.deprecated( + ARC4, + __name__, + "ARC4 has been moved to " + "cryptography.hazmat.decrepit.ciphers.algorithms.ARC4 and " + "will be removed from " + "cryptography.hazmat.primitives.ciphers.algorithms in 48.0.0.", + utils.DeprecatedIn43, + name="ARC4", +) + + +utils.deprecated( + TripleDES, + __name__, + "TripleDES has been moved to " + "cryptography.hazmat.decrepit.ciphers.algorithms.TripleDES and " + "will be removed from " + "cryptography.hazmat.primitives.ciphers.algorithms in 48.0.0.", + utils.DeprecatedIn43, + name="TripleDES", +) + + +class ChaCha20(CipherAlgorithm): + name = "ChaCha20" + key_sizes = frozenset([256]) + + def __init__(self, key: utils.Buffer, nonce: utils.Buffer): + self.key = _verify_key_size(self, key) + utils._check_byteslike("nonce", nonce) + + if len(nonce) != 16: + raise ValueError("nonce must be 128-bits (16 bytes)") + + self._nonce = nonce + + @property + def nonce(self) -> utils.Buffer: + return self._nonce + + @property + def key_size(self) -> int: + return len(self.key) * 8 + + +class SM4(BlockCipherAlgorithm): + name = "SM4" + block_size = 128 + key_sizes = frozenset([128]) + + def __init__(self, key: bytes): + self.key = _verify_key_size(self, key) + + @property + def key_size(self) -> int: + return len(self.key) * 8 diff --git a/py_modules/cryptography/hazmat/primitives/ciphers/base.py b/py_modules/cryptography/hazmat/primitives/ciphers/base.py new file mode 100644 index 0000000..24fceea --- /dev/null +++ b/py_modules/cryptography/hazmat/primitives/ciphers/base.py @@ -0,0 +1,146 @@ +# This file is dual licensed under the terms of the Apache License, Version +# 2.0, and the BSD License. See the LICENSE file in the root of this repository +# for complete details. + +from __future__ import annotations + +import abc +import typing + +from cryptography.hazmat.bindings._rust import openssl as rust_openssl +from cryptography.hazmat.primitives._cipheralgorithm import CipherAlgorithm +from cryptography.hazmat.primitives.ciphers import modes +from cryptography.utils import Buffer + + +class CipherContext(metaclass=abc.ABCMeta): + @abc.abstractmethod + def update(self, data: Buffer) -> bytes: + """ + Processes the provided bytes through the cipher and returns the results + as bytes. + """ + + @abc.abstractmethod + def update_into(self, data: Buffer, buf: Buffer) -> int: + """ + Processes the provided bytes and writes the resulting data into the + provided buffer. Returns the number of bytes written. + """ + + @abc.abstractmethod + def finalize(self) -> bytes: + """ + Returns the results of processing the final block as bytes. + """ + + @abc.abstractmethod + def reset_nonce(self, nonce: bytes) -> None: + """ + Resets the nonce for the cipher context to the provided value. + Raises an exception if it does not support reset or if the + provided nonce does not have a valid length. + """ + + +class AEADCipherContext(CipherContext, metaclass=abc.ABCMeta): + @abc.abstractmethod + def authenticate_additional_data(self, data: Buffer) -> None: + """ + Authenticates the provided bytes. + """ + + +class AEADDecryptionContext(AEADCipherContext, metaclass=abc.ABCMeta): + @abc.abstractmethod + def finalize_with_tag(self, tag: bytes) -> bytes: + """ + Returns the results of processing the final block as bytes and allows + delayed passing of the authentication tag. + """ + + +class AEADEncryptionContext(AEADCipherContext, metaclass=abc.ABCMeta): + @property + @abc.abstractmethod + def tag(self) -> bytes: + """ + Returns tag bytes. This is only available after encryption is + finalized. + """ + + +Mode = typing.TypeVar( + "Mode", bound=typing.Optional[modes.Mode], covariant=True +) + + +class Cipher(typing.Generic[Mode]): + def __init__( + self, + algorithm: CipherAlgorithm, + mode: Mode, + backend: typing.Any = None, + ) -> None: + if not isinstance(algorithm, CipherAlgorithm): + raise TypeError("Expected interface of CipherAlgorithm.") + + if mode is not None: + # mypy needs this assert to narrow the type from our generic + # type. Maybe it won't some time in the future. + assert isinstance(mode, modes.Mode) + mode.validate_for_algorithm(algorithm) + + self.algorithm = algorithm + self.mode = mode + + @typing.overload + def encryptor( + self: Cipher[modes.ModeWithAuthenticationTag], + ) -> AEADEncryptionContext: ... + + @typing.overload + def encryptor( + self: _CIPHER_TYPE, + ) -> CipherContext: ... + + def encryptor(self): + if isinstance(self.mode, modes.ModeWithAuthenticationTag): + if self.mode.tag is not None: + raise ValueError( + "Authentication tag must be None when encrypting." + ) + + return rust_openssl.ciphers.create_encryption_ctx( + self.algorithm, self.mode + ) + + @typing.overload + def decryptor( + self: Cipher[modes.ModeWithAuthenticationTag], + ) -> AEADDecryptionContext: ... + + @typing.overload + def decryptor( + self: _CIPHER_TYPE, + ) -> CipherContext: ... + + def decryptor(self): + return rust_openssl.ciphers.create_decryption_ctx( + self.algorithm, self.mode + ) + + +_CIPHER_TYPE = Cipher[ + typing.Union[ + modes.ModeWithNonce, + modes.ModeWithTweak, + modes.ECB, + modes.ModeWithInitializationVector, + None, + ] +] + +CipherContext.register(rust_openssl.ciphers.CipherContext) +AEADEncryptionContext.register(rust_openssl.ciphers.AEADEncryptionContext) +AEADDecryptionContext.register(rust_openssl.ciphers.AEADDecryptionContext) diff --git a/py_modules/cryptography/hazmat/primitives/ciphers/modes.py b/py_modules/cryptography/hazmat/primitives/ciphers/modes.py new file mode 100644 index 0000000..c2008e9 --- /dev/null +++ b/py_modules/cryptography/hazmat/primitives/ciphers/modes.py @@ -0,0 +1,192 @@ +# This file is dual licensed under the terms of the Apache License, Version +# 2.0, and the BSD License. See the LICENSE file in the root of this repository +# for complete details. + +from __future__ import annotations + +from cryptography import utils +from cryptography.exceptions import UnsupportedAlgorithm, _Reasons +from cryptography.hazmat.decrepit.ciphers.modes import CFB as CFB +from cryptography.hazmat.decrepit.ciphers.modes import CFB8 as CFB8 +from cryptography.hazmat.decrepit.ciphers.modes import OFB as OFB +from cryptography.hazmat.primitives._cipheralgorithm import ( + BlockCipherAlgorithm, + CipherAlgorithm, +) +from cryptography.hazmat.primitives._modes import ( + Mode as Mode, +) +from cryptography.hazmat.primitives._modes import ( + ModeWithAuthenticationTag as ModeWithAuthenticationTag, +) +from cryptography.hazmat.primitives._modes import ( + ModeWithInitializationVector as ModeWithInitializationVector, +) +from cryptography.hazmat.primitives._modes import ( + ModeWithNonce as ModeWithNonce, +) +from cryptography.hazmat.primitives._modes import ( + ModeWithTweak as ModeWithTweak, +) +from cryptography.hazmat.primitives._modes import ( + _check_aes_key_length, + _check_iv_and_key_length, + _check_nonce_length, +) +from cryptography.hazmat.primitives.ciphers import algorithms + + +class CBC(ModeWithInitializationVector): + name = "CBC" + + def __init__(self, initialization_vector: utils.Buffer): + utils._check_byteslike("initialization_vector", initialization_vector) + self._initialization_vector = initialization_vector + + @property + def initialization_vector(self) -> utils.Buffer: + return self._initialization_vector + + validate_for_algorithm = _check_iv_and_key_length + + +class XTS(ModeWithTweak): + name = "XTS" + + def __init__(self, tweak: utils.Buffer): + utils._check_byteslike("tweak", tweak) + + if len(tweak) != 16: + raise ValueError("tweak must be 128-bits (16 bytes)") + + self._tweak = tweak + + @property + def tweak(self) -> utils.Buffer: + return self._tweak + + def validate_for_algorithm(self, algorithm: CipherAlgorithm) -> None: + if isinstance(algorithm, (algorithms.AES128, algorithms.AES256)): + raise TypeError( + "The AES128 and AES256 classes do not support XTS, please use " + "the standard AES class instead." + ) + + if algorithm.key_size not in (256, 512): + raise ValueError( + "The XTS specification requires a 256-bit key for AES-128-XTS" + " and 512-bit key for AES-256-XTS" + ) + + +class ECB(Mode): + name = "ECB" + + validate_for_algorithm = _check_aes_key_length + + +class CTR(ModeWithNonce): + name = "CTR" + + def __init__(self, nonce: utils.Buffer): + utils._check_byteslike("nonce", nonce) + self._nonce = nonce + + @property + def nonce(self) -> utils.Buffer: + return self._nonce + + def validate_for_algorithm(self, algorithm: CipherAlgorithm) -> None: + _check_aes_key_length(self, algorithm) + _check_nonce_length(self.nonce, self.name, algorithm) + + +class GCM(ModeWithInitializationVector, ModeWithAuthenticationTag): + name = "GCM" + _MAX_ENCRYPTED_BYTES = (2**39 - 256) // 8 + _MAX_AAD_BYTES = (2**64 - 1) // 8 + + def __init__( + self, + initialization_vector: utils.Buffer, + tag: bytes | None = None, + min_tag_length: int = 16, + ): + # OpenSSL 3.0.0 constrains GCM IVs to [64, 1024] bits inclusive + # This is a sane limit anyway so we'll enforce it here. + utils._check_byteslike("initialization_vector", initialization_vector) + if len(initialization_vector) < 8 or len(initialization_vector) > 128: + raise ValueError( + "initialization_vector must be between 8 and 128 bytes (64 " + "and 1024 bits)." + ) + self._initialization_vector = initialization_vector + if min_tag_length < 4: + raise ValueError("min_tag_length must be >= 4") + if tag is not None: + utils._check_bytes("tag", tag) + if len(tag) < min_tag_length: + raise ValueError( + f"Authentication tag must be {min_tag_length} bytes or " + "longer." + ) + self._tag = tag + self._min_tag_length = min_tag_length + + @property + def tag(self) -> bytes | None: + return self._tag + + @property + def initialization_vector(self) -> utils.Buffer: + return self._initialization_vector + + def validate_for_algorithm(self, algorithm: CipherAlgorithm) -> None: + _check_aes_key_length(self, algorithm) + if not isinstance(algorithm, BlockCipherAlgorithm): + raise UnsupportedAlgorithm( + "GCM requires a block cipher algorithm", + _Reasons.UNSUPPORTED_CIPHER, + ) + block_size_bytes = algorithm.block_size // 8 + if self._tag is not None and len(self._tag) > block_size_bytes: + raise ValueError( + f"Authentication tag cannot be more than {block_size_bytes} " + "bytes." + ) + + +utils.deprecated( + OFB, + __name__, + "OFB has been moved to " + "cryptography.hazmat.decrepit.ciphers.modes.OFB and " + "will be removed from " + "cryptography.hazmat.primitives.ciphers.modes in 49.0.0.", + utils.DeprecatedIn47, + name="OFB", +) + + +utils.deprecated( + CFB, + __name__, + "CFB has been moved to " + "cryptography.hazmat.decrepit.ciphers.modes.CFB and " + "will be removed from " + "cryptography.hazmat.primitives.ciphers.modes in 49.0.0.", + utils.DeprecatedIn47, + name="CFB", +) + + +utils.deprecated( + CFB8, + __name__, + "CFB8 has been moved to " + "cryptography.hazmat.decrepit.ciphers.modes.CFB8 and " + "will be removed from " + "cryptography.hazmat.primitives.ciphers.modes in 49.0.0.", + utils.DeprecatedIn47, + name="CFB8", +) diff --git a/py_modules/cryptography/hazmat/primitives/cmac.py b/py_modules/cryptography/hazmat/primitives/cmac.py new file mode 100644 index 0000000..2c67ce2 --- /dev/null +++ b/py_modules/cryptography/hazmat/primitives/cmac.py @@ -0,0 +1,10 @@ +# This file is dual licensed under the terms of the Apache License, Version +# 2.0, and the BSD License. See the LICENSE file in the root of this repository +# for complete details. + +from __future__ import annotations + +from cryptography.hazmat.bindings._rust import openssl as rust_openssl + +__all__ = ["CMAC"] +CMAC = rust_openssl.cmac.CMAC diff --git a/py_modules/cryptography/hazmat/primitives/constant_time.py b/py_modules/cryptography/hazmat/primitives/constant_time.py new file mode 100644 index 0000000..3975c71 --- /dev/null +++ b/py_modules/cryptography/hazmat/primitives/constant_time.py @@ -0,0 +1,14 @@ +# This file is dual licensed under the terms of the Apache License, Version +# 2.0, and the BSD License. See the LICENSE file in the root of this repository +# for complete details. + +from __future__ import annotations + +import hmac + + +def bytes_eq(a: bytes, b: bytes) -> bool: + if not isinstance(a, bytes) or not isinstance(b, bytes): + raise TypeError("a and b must be bytes.") + + return hmac.compare_digest(a, b) diff --git a/py_modules/cryptography/hazmat/primitives/hashes.py b/py_modules/cryptography/hazmat/primitives/hashes.py new file mode 100644 index 0000000..50ed355 --- /dev/null +++ b/py_modules/cryptography/hazmat/primitives/hashes.py @@ -0,0 +1,357 @@ +# This file is dual licensed under the terms of the Apache License, Version +# 2.0, and the BSD License. See the LICENSE file in the root of this repository +# for complete details. + +from __future__ import annotations + +import abc +import sys + +from cryptography.hazmat.bindings._rust import openssl as rust_openssl +from cryptography.utils import Buffer + +__all__ = [ + "MD5", + "SHA1", + "SHA3_224", + "SHA3_256", + "SHA3_384", + "SHA3_512", + "SHA224", + "SHA256", + "SHA384", + "SHA512", + "SHA512_224", + "SHA512_256", + "SHAKE128", + "SHAKE256", + "SM3", + "BLAKE2b", + "BLAKE2s", + "ExtendableOutputFunction", + "Hash", + "HashAlgorithm", + "HashContext", + "XOFHash", +] + + +class HashAlgorithm(metaclass=abc.ABCMeta): + @property + @abc.abstractmethod + def name(self) -> str: + """ + A string naming this algorithm (e.g. "sha256", "md5"). + """ + + @property + @abc.abstractmethod + def digest_size(self) -> int: + """ + The size of the resulting digest in bytes. + """ + + @property + @abc.abstractmethod + def block_size(self) -> int | None: + """ + The internal block size of the hash function, or None if the hash + function does not use blocks internally (e.g. SHA3). + """ + + +class HashContext(metaclass=abc.ABCMeta): + @property + @abc.abstractmethod + def algorithm(self) -> HashAlgorithm: + """ + A HashAlgorithm that will be used by this context. + """ + + @abc.abstractmethod + def update(self, data: Buffer) -> None: + """ + Processes the provided bytes through the hash. + """ + + @abc.abstractmethod + def finalize(self) -> bytes: + """ + Finalizes the hash context and returns the hash digest as bytes. + """ + + @abc.abstractmethod + def copy(self) -> HashContext: + """ + Return a HashContext that is a copy of the current context. + """ + + +Hash = rust_openssl.hashes.Hash +HashContext.register(Hash) + +XOFHash = rust_openssl.hashes.XOFHash + + +class ExtendableOutputFunction(metaclass=abc.ABCMeta): + """ + An interface for extendable output functions. + """ + + +class SHA1(HashAlgorithm): + name = "sha1" + digest_size = 20 + block_size = 64 + + def __eq__(self, other: object) -> bool: + if not isinstance(other, SHA1): + return NotImplemented + + return True + + +class SHA512_224(HashAlgorithm): # noqa: N801 + name = "sha512-224" + digest_size = 28 + block_size = 128 + + def __eq__(self, other: object) -> bool: + if not isinstance(other, SHA512_224): + return NotImplemented + + return True + + +class SHA512_256(HashAlgorithm): # noqa: N801 + name = "sha512-256" + digest_size = 32 + block_size = 128 + + def __eq__(self, other: object) -> bool: + if not isinstance(other, SHA512_256): + return NotImplemented + + return True + + +class SHA224(HashAlgorithm): + name = "sha224" + digest_size = 28 + block_size = 64 + + def __eq__(self, other: object) -> bool: + if not isinstance(other, SHA224): + return NotImplemented + + return True + + +class SHA256(HashAlgorithm): + name = "sha256" + digest_size = 32 + block_size = 64 + + def __eq__(self, other: object) -> bool: + if not isinstance(other, SHA256): + return NotImplemented + + return True + + +class SHA384(HashAlgorithm): + name = "sha384" + digest_size = 48 + block_size = 128 + + def __eq__(self, other: object) -> bool: + if not isinstance(other, SHA384): + return NotImplemented + + return True + + +class SHA512(HashAlgorithm): + name = "sha512" + digest_size = 64 + block_size = 128 + + def __eq__(self, other: object) -> bool: + if not isinstance(other, SHA512): + return NotImplemented + + return True + + +class SHA3_224(HashAlgorithm): # noqa: N801 + name = "sha3-224" + digest_size = 28 + block_size = None + + def __eq__(self, other: object) -> bool: + if not isinstance(other, SHA3_224): + return NotImplemented + + return True + + +class SHA3_256(HashAlgorithm): # noqa: N801 + name = "sha3-256" + digest_size = 32 + block_size = None + + def __eq__(self, other: object) -> bool: + if not isinstance(other, SHA3_256): + return NotImplemented + + return True + + +class SHA3_384(HashAlgorithm): # noqa: N801 + name = "sha3-384" + digest_size = 48 + block_size = None + + def __eq__(self, other: object) -> bool: + if not isinstance(other, SHA3_384): + return NotImplemented + + return True + + +class SHA3_512(HashAlgorithm): # noqa: N801 + name = "sha3-512" + digest_size = 64 + block_size = None + + def __eq__(self, other: object) -> bool: + if not isinstance(other, SHA3_512): + return NotImplemented + + return True + + +class SHAKE128(HashAlgorithm, ExtendableOutputFunction): + name = "shake128" + block_size = None + + def __init__(self, digest_size: int): + if not isinstance(digest_size, int): + raise TypeError("digest_size must be an integer") + + if digest_size < 1: + raise ValueError("digest_size must be a positive integer") + + self._digest_size = digest_size + + def __eq__(self, other: object) -> bool: + if not isinstance(other, SHAKE128): + return NotImplemented + + return self._digest_size == other._digest_size + + @property + def digest_size(self) -> int: + return self._digest_size + + @classmethod + def xof(cls): + return cls(sys.maxsize) + + +class SHAKE256(HashAlgorithm, ExtendableOutputFunction): + name = "shake256" + block_size = None + + def __init__(self, digest_size: int): + if not isinstance(digest_size, int): + raise TypeError("digest_size must be an integer") + + if digest_size < 1: + raise ValueError("digest_size must be a positive integer") + + self._digest_size = digest_size + + def __eq__(self, other: object) -> bool: + if not isinstance(other, SHAKE256): + return NotImplemented + + return self._digest_size == other._digest_size + + @property + def digest_size(self) -> int: + return self._digest_size + + @classmethod + def xof(cls): + return cls(sys.maxsize) + + +class MD5(HashAlgorithm): + name = "md5" + digest_size = 16 + block_size = 64 + + def __eq__(self, other: object) -> bool: + if not isinstance(other, MD5): + return NotImplemented + + return True + + +class BLAKE2b(HashAlgorithm): + name = "blake2b" + _max_digest_size = 64 + _min_digest_size = 1 + block_size = 128 + + def __init__(self, digest_size: int): + if digest_size != 64: + raise ValueError("Digest size must be 64") + + self._digest_size = digest_size + + def __eq__(self, other: object) -> bool: + if not isinstance(other, BLAKE2b): + return NotImplemented + + return self._digest_size == other._digest_size + + @property + def digest_size(self) -> int: + return self._digest_size + + +class BLAKE2s(HashAlgorithm): + name = "blake2s" + block_size = 64 + _max_digest_size = 32 + _min_digest_size = 1 + + def __init__(self, digest_size: int): + if digest_size != 32: + raise ValueError("Digest size must be 32") + + self._digest_size = digest_size + + def __eq__(self, other: object) -> bool: + if not isinstance(other, BLAKE2s): + return NotImplemented + + return self._digest_size == other._digest_size + + @property + def digest_size(self) -> int: + return self._digest_size + + +class SM3(HashAlgorithm): + name = "sm3" + digest_size = 32 + block_size = 64 + + def __eq__(self, other: object) -> bool: + if not isinstance(other, SM3): + return NotImplemented + + return True diff --git a/py_modules/cryptography/hazmat/primitives/hmac.py b/py_modules/cryptography/hazmat/primitives/hmac.py new file mode 100644 index 0000000..a9442d5 --- /dev/null +++ b/py_modules/cryptography/hazmat/primitives/hmac.py @@ -0,0 +1,13 @@ +# This file is dual licensed under the terms of the Apache License, Version +# 2.0, and the BSD License. See the LICENSE file in the root of this repository +# for complete details. + +from __future__ import annotations + +from cryptography.hazmat.bindings._rust import openssl as rust_openssl +from cryptography.hazmat.primitives import hashes + +__all__ = ["HMAC"] + +HMAC = rust_openssl.hmac.HMAC +hashes.HashContext.register(HMAC) diff --git a/py_modules/cryptography/hazmat/primitives/hpke.py b/py_modules/cryptography/hazmat/primitives/hpke.py new file mode 100644 index 0000000..c802808 --- /dev/null +++ b/py_modules/cryptography/hazmat/primitives/hpke.py @@ -0,0 +1,27 @@ +# This file is dual licensed under the terms of the Apache License, Version +# 2.0, and the BSD License. See the LICENSE file in the root of this repository +# for complete details. + +from __future__ import annotations + +from cryptography.hazmat.bindings._rust import openssl as rust_openssl + +AEAD = rust_openssl.hpke.AEAD +KDF = rust_openssl.hpke.KDF +KEM = rust_openssl.hpke.KEM +MLKEM768X25519PrivateKey = rust_openssl.hpke.MLKEM768X25519PrivateKey +MLKEM768X25519PublicKey = rust_openssl.hpke.MLKEM768X25519PublicKey +MLKEM1024P384PrivateKey = rust_openssl.hpke.MLKEM1024P384PrivateKey +MLKEM1024P384PublicKey = rust_openssl.hpke.MLKEM1024P384PublicKey +Suite = rust_openssl.hpke.Suite + +__all__ = [ + "AEAD", + "KDF", + "KEM", + "MLKEM768X25519PrivateKey", + "MLKEM768X25519PublicKey", + "MLKEM1024P384PrivateKey", + "MLKEM1024P384PublicKey", + "Suite", +] diff --git a/py_modules/cryptography/hazmat/primitives/kdf/__init__.py b/py_modules/cryptography/hazmat/primitives/kdf/__init__.py new file mode 100644 index 0000000..26c45bd --- /dev/null +++ b/py_modules/cryptography/hazmat/primitives/kdf/__init__.py @@ -0,0 +1,32 @@ +# This file is dual licensed under the terms of the Apache License, Version +# 2.0, and the BSD License. See the LICENSE file in the root of this repository +# for complete details. + +from __future__ import annotations + +import abc + +from cryptography.utils import Buffer + + +class KeyDerivationFunction(metaclass=abc.ABCMeta): + @abc.abstractmethod + def derive(self, key_material: bytes) -> bytes: + """ + Deterministically generates and returns a new key based on the existing + key material. + """ + + @abc.abstractmethod + def derive_into(self, key_material: bytes, buffer: Buffer) -> None: + """ + Deterministically generates a new key based on the existing key + material and stores it in the provided buffer. + """ + + @abc.abstractmethod + def verify(self, key_material: bytes, expected_key: bytes) -> None: + """ + Checks whether the key generated by the key material matches the + expected derived key. Raises an exception if they do not match. + """ diff --git a/py_modules/cryptography/hazmat/primitives/kdf/argon2.py b/py_modules/cryptography/hazmat/primitives/kdf/argon2.py new file mode 100644 index 0000000..03e84d4 --- /dev/null +++ b/py_modules/cryptography/hazmat/primitives/kdf/argon2.py @@ -0,0 +1,17 @@ +# This file is dual licensed under the terms of the Apache License, Version +# 2.0, and the BSD License. See the LICENSE file in the root of this repository +# for complete details. + +from __future__ import annotations + +from cryptography.hazmat.bindings._rust import openssl as rust_openssl +from cryptography.hazmat.primitives.kdf import KeyDerivationFunction + +Argon2d = rust_openssl.kdf.Argon2d +Argon2i = rust_openssl.kdf.Argon2i +Argon2id = rust_openssl.kdf.Argon2id +KeyDerivationFunction.register(Argon2d) +KeyDerivationFunction.register(Argon2i) +KeyDerivationFunction.register(Argon2id) + +__all__ = ["Argon2d", "Argon2i", "Argon2id"] diff --git a/py_modules/cryptography/hazmat/primitives/kdf/concatkdf.py b/py_modules/cryptography/hazmat/primitives/kdf/concatkdf.py new file mode 100644 index 0000000..398dc5d --- /dev/null +++ b/py_modules/cryptography/hazmat/primitives/kdf/concatkdf.py @@ -0,0 +1,16 @@ +# This file is dual licensed under the terms of the Apache License, Version +# 2.0, and the BSD License. See the LICENSE file in the root of this repository +# for complete details. + +from __future__ import annotations + +from cryptography.hazmat.bindings._rust import openssl as rust_openssl +from cryptography.hazmat.primitives.kdf import KeyDerivationFunction + +ConcatKDFHash = rust_openssl.kdf.ConcatKDFHash +ConcatKDFHMAC = rust_openssl.kdf.ConcatKDFHMAC + +KeyDerivationFunction.register(ConcatKDFHash) +KeyDerivationFunction.register(ConcatKDFHMAC) + +__all__ = ["ConcatKDFHMAC", "ConcatKDFHash"] diff --git a/py_modules/cryptography/hazmat/primitives/kdf/hkdf.py b/py_modules/cryptography/hazmat/primitives/kdf/hkdf.py new file mode 100644 index 0000000..1e162d9 --- /dev/null +++ b/py_modules/cryptography/hazmat/primitives/kdf/hkdf.py @@ -0,0 +1,16 @@ +# This file is dual licensed under the terms of the Apache License, Version +# 2.0, and the BSD License. See the LICENSE file in the root of this repository +# for complete details. + +from __future__ import annotations + +from cryptography.hazmat.bindings._rust import openssl as rust_openssl +from cryptography.hazmat.primitives.kdf import KeyDerivationFunction + +HKDF = rust_openssl.kdf.HKDF +HKDFExpand = rust_openssl.kdf.HKDFExpand + +KeyDerivationFunction.register(HKDF) +KeyDerivationFunction.register(HKDFExpand) + +__all__ = ["HKDF", "HKDFExpand"] diff --git a/py_modules/cryptography/hazmat/primitives/kdf/kbkdf.py b/py_modules/cryptography/hazmat/primitives/kdf/kbkdf.py new file mode 100644 index 0000000..e559df8 --- /dev/null +++ b/py_modules/cryptography/hazmat/primitives/kdf/kbkdf.py @@ -0,0 +1,26 @@ +# This file is dual licensed under the terms of the Apache License, Version +# 2.0, and the BSD License. See the LICENSE file in the root of this repository +# for complete details. + +from __future__ import annotations + +from cryptography import utils +from cryptography.hazmat.bindings._rust import openssl as rust_openssl +from cryptography.hazmat.primitives.kdf import KeyDerivationFunction + + +class Mode(utils.Enum): + CounterMode = "ctr" + + +class CounterLocation(utils.Enum): + BeforeFixed = "before_fixed" + AfterFixed = "after_fixed" + MiddleFixed = "middle_fixed" + + +KBKDFHMAC = rust_openssl.kdf.KBKDFHMAC +KeyDerivationFunction.register(KBKDFHMAC) + +KBKDFCMAC = rust_openssl.kdf.KBKDFCMAC +KeyDerivationFunction.register(KBKDFCMAC) diff --git a/py_modules/cryptography/hazmat/primitives/kdf/pbkdf2.py b/py_modules/cryptography/hazmat/primitives/kdf/pbkdf2.py new file mode 100644 index 0000000..771d80d --- /dev/null +++ b/py_modules/cryptography/hazmat/primitives/kdf/pbkdf2.py @@ -0,0 +1,13 @@ +# This file is dual licensed under the terms of the Apache License, Version +# 2.0, and the BSD License. See the LICENSE file in the root of this repository +# for complete details. + +from __future__ import annotations + +from cryptography.hazmat.bindings._rust import openssl as rust_openssl +from cryptography.hazmat.primitives.kdf import KeyDerivationFunction + +PBKDF2HMAC = rust_openssl.kdf.PBKDF2HMAC +KeyDerivationFunction.register(PBKDF2HMAC) + +__all__ = ["PBKDF2HMAC"] diff --git a/py_modules/cryptography/hazmat/primitives/kdf/scrypt.py b/py_modules/cryptography/hazmat/primitives/kdf/scrypt.py new file mode 100644 index 0000000..f791cee --- /dev/null +++ b/py_modules/cryptography/hazmat/primitives/kdf/scrypt.py @@ -0,0 +1,19 @@ +# This file is dual licensed under the terms of the Apache License, Version +# 2.0, and the BSD License. See the LICENSE file in the root of this repository +# for complete details. + +from __future__ import annotations + +import sys + +from cryptography.hazmat.bindings._rust import openssl as rust_openssl +from cryptography.hazmat.primitives.kdf import KeyDerivationFunction + +# This is used by the scrypt tests to skip tests that require more memory +# than the MEM_LIMIT +_MEM_LIMIT = sys.maxsize // 2 + +Scrypt = rust_openssl.kdf.Scrypt +KeyDerivationFunction.register(Scrypt) + +__all__ = ["Scrypt"] diff --git a/py_modules/cryptography/hazmat/primitives/kdf/x963kdf.py b/py_modules/cryptography/hazmat/primitives/kdf/x963kdf.py new file mode 100644 index 0000000..8c4e2d3 --- /dev/null +++ b/py_modules/cryptography/hazmat/primitives/kdf/x963kdf.py @@ -0,0 +1,13 @@ +# This file is dual licensed under the terms of the Apache License, Version +# 2.0, and the BSD License. See the LICENSE file in the root of this repository +# for complete details. + +from __future__ import annotations + +from cryptography.hazmat.bindings._rust import openssl as rust_openssl +from cryptography.hazmat.primitives.kdf import KeyDerivationFunction + +X963KDF = rust_openssl.kdf.X963KDF +KeyDerivationFunction.register(X963KDF) + +__all__ = ["X963KDF"] diff --git a/py_modules/cryptography/hazmat/primitives/keywrap.py b/py_modules/cryptography/hazmat/primitives/keywrap.py new file mode 100644 index 0000000..123d97b --- /dev/null +++ b/py_modules/cryptography/hazmat/primitives/keywrap.py @@ -0,0 +1,25 @@ +# This file is dual licensed under the terms of the Apache License, Version +# 2.0, and the BSD License. See the LICENSE file in the root of this repository +# for complete details. + +from __future__ import annotations + +from cryptography.hazmat.bindings._rust import openssl as rust_openssl + +__all__ = [ + "InvalidUnwrap", + "aes_key_unwrap", + "aes_key_unwrap_with_padding", + "aes_key_wrap", + "aes_key_wrap_with_padding", +] + + +class InvalidUnwrap(Exception): + pass + + +aes_key_wrap = rust_openssl.keywrap.aes_key_wrap +aes_key_unwrap = rust_openssl.keywrap.aes_key_unwrap +aes_key_wrap_with_padding = rust_openssl.keywrap.aes_key_wrap_with_padding +aes_key_unwrap_with_padding = rust_openssl.keywrap.aes_key_unwrap_with_padding diff --git a/py_modules/cryptography/hazmat/primitives/padding.py b/py_modules/cryptography/hazmat/primitives/padding.py new file mode 100644 index 0000000..f9cd1f1 --- /dev/null +++ b/py_modules/cryptography/hazmat/primitives/padding.py @@ -0,0 +1,69 @@ +# This file is dual licensed under the terms of the Apache License, Version +# 2.0, and the BSD License. See the LICENSE file in the root of this repository +# for complete details. + +from __future__ import annotations + +import abc + +from cryptography import utils +from cryptography.hazmat.bindings._rust import ( + ANSIX923PaddingContext, + ANSIX923UnpaddingContext, + PKCS7PaddingContext, + PKCS7UnpaddingContext, +) + + +class PaddingContext(metaclass=abc.ABCMeta): + @abc.abstractmethod + def update(self, data: utils.Buffer) -> bytes: + """ + Pads the provided bytes and returns any available data as bytes. + """ + + @abc.abstractmethod + def finalize(self) -> bytes: + """ + Finalize the padding, returns bytes. + """ + + +def _byte_padding_check(block_size: int) -> None: + if not (0 <= block_size <= 2040): + raise ValueError("block_size must be in range(0, 2041).") + + if block_size % 8 != 0: + raise ValueError("block_size must be a multiple of 8.") + + +class PKCS7: + def __init__(self, block_size: int): + _byte_padding_check(block_size) + self.block_size = block_size + + def padder(self) -> PaddingContext: + return PKCS7PaddingContext(self.block_size) + + def unpadder(self) -> PaddingContext: + return PKCS7UnpaddingContext(self.block_size) + + +PaddingContext.register(PKCS7PaddingContext) +PaddingContext.register(PKCS7UnpaddingContext) + + +class ANSIX923: + def __init__(self, block_size: int): + _byte_padding_check(block_size) + self.block_size = block_size + + def padder(self) -> PaddingContext: + return ANSIX923PaddingContext(self.block_size) + + def unpadder(self) -> PaddingContext: + return ANSIX923UnpaddingContext(self.block_size) + + +PaddingContext.register(ANSIX923PaddingContext) +PaddingContext.register(ANSIX923UnpaddingContext) diff --git a/py_modules/cryptography/hazmat/primitives/poly1305.py b/py_modules/cryptography/hazmat/primitives/poly1305.py new file mode 100644 index 0000000..7f5a77a --- /dev/null +++ b/py_modules/cryptography/hazmat/primitives/poly1305.py @@ -0,0 +1,11 @@ +# This file is dual licensed under the terms of the Apache License, Version +# 2.0, and the BSD License. See the LICENSE file in the root of this repository +# for complete details. + +from __future__ import annotations + +from cryptography.hazmat.bindings._rust import openssl as rust_openssl + +__all__ = ["Poly1305"] + +Poly1305 = rust_openssl.poly1305.Poly1305 diff --git a/py_modules/cryptography/hazmat/primitives/serialization/__init__.py b/py_modules/cryptography/hazmat/primitives/serialization/__init__.py new file mode 100644 index 0000000..40beb21 --- /dev/null +++ b/py_modules/cryptography/hazmat/primitives/serialization/__init__.py @@ -0,0 +1,85 @@ +# This file is dual licensed under the terms of the Apache License, Version +# 2.0, and the BSD License. See the LICENSE file in the root of this repository +# for complete details. + +from __future__ import annotations + +from cryptography import utils +from cryptography.hazmat.primitives._serialization import ( + BestAvailableEncryption, + Encoding, + KeySerializationEncryption, + NoEncryption, + ParameterFormat, + PrivateFormat, + PublicFormat, + _KeySerializationEncryption, +) +from cryptography.hazmat.primitives.asymmetric.dh import _FFDH_DEPRECATION_MSG +from cryptography.hazmat.primitives.serialization.base import ( + load_der_parameters, + load_der_private_key, + load_der_public_key, + load_pem_parameters, + load_pem_private_key, + load_pem_public_key, +) +from cryptography.hazmat.primitives.serialization.ssh import ( + SSHCertificate, + SSHCertificateBuilder, + SSHCertificateType, + SSHCertPrivateKeyTypes, + SSHCertPublicKeyTypes, + SSHPrivateKeyTypes, + SSHPublicKeyTypes, + load_ssh_private_key, + load_ssh_public_identity, + load_ssh_public_key, + ssh_key_fingerprint, +) + +__all__ = [ + "BestAvailableEncryption", + "Encoding", + "KeySerializationEncryption", + "NoEncryption", + "ParameterFormat", + "PrivateFormat", + "PublicFormat", + "SSHCertPrivateKeyTypes", + "SSHCertPublicKeyTypes", + "SSHCertificate", + "SSHCertificateBuilder", + "SSHCertificateType", + "SSHPrivateKeyTypes", + "SSHPublicKeyTypes", + "_KeySerializationEncryption", + "load_der_parameters", + "load_der_private_key", + "load_der_public_key", + "load_pem_parameters", + "load_pem_private_key", + "load_pem_public_key", + "load_ssh_private_key", + "load_ssh_public_identity", + "load_ssh_public_key", + "ssh_key_fingerprint", +] + +# These can only load FFDH parameters, so the functions themselves are +# deprecated alongside the rest of FFDH. +utils.deprecated( + load_pem_parameters, + __name__, + _FFDH_DEPRECATION_MSG, + utils.DeprecatedIn50, + name="load_pem_parameters", +) + +utils.deprecated( + load_der_parameters, + __name__, + _FFDH_DEPRECATION_MSG, + utils.DeprecatedIn50, + name="load_der_parameters", +) diff --git a/py_modules/cryptography/hazmat/primitives/serialization/base.py b/py_modules/cryptography/hazmat/primitives/serialization/base.py new file mode 100644 index 0000000..f2b503b --- /dev/null +++ b/py_modules/cryptography/hazmat/primitives/serialization/base.py @@ -0,0 +1,16 @@ +# This file is dual licensed under the terms of the Apache License, Version +# 2.0, and the BSD License. See the LICENSE file in the root of this repository +# for complete details. + +from __future__ import annotations + +from cryptography.hazmat.bindings._rust import openssl as rust_openssl + +load_pem_private_key = rust_openssl.keys.load_pem_private_key +load_der_private_key = rust_openssl.keys.load_der_private_key + +load_pem_public_key = rust_openssl.keys.load_pem_public_key +load_der_public_key = rust_openssl.keys.load_der_public_key + +load_pem_parameters = rust_openssl.dh.from_pem_parameters +load_der_parameters = rust_openssl.dh.from_der_parameters diff --git a/py_modules/cryptography/hazmat/primitives/serialization/pkcs12.py b/py_modules/cryptography/hazmat/primitives/serialization/pkcs12.py new file mode 100644 index 0000000..58884ff --- /dev/null +++ b/py_modules/cryptography/hazmat/primitives/serialization/pkcs12.py @@ -0,0 +1,176 @@ +# This file is dual licensed under the terms of the Apache License, Version +# 2.0, and the BSD License. See the LICENSE file in the root of this repository +# for complete details. + +from __future__ import annotations + +import typing +from collections.abc import Iterable + +from cryptography import x509 +from cryptography.hazmat.bindings._rust import pkcs12 as rust_pkcs12 +from cryptography.hazmat.primitives import serialization +from cryptography.hazmat.primitives._serialization import PBES as PBES +from cryptography.hazmat.primitives.asymmetric import ( + dsa, + ec, + ed448, + ed25519, + rsa, +) +from cryptography.hazmat.primitives.asymmetric.types import PrivateKeyTypes + +__all__ = [ + "PBES", + "PKCS12Certificate", + "PKCS12KeyAndCertificates", + "PKCS12PrivateKeyTypes", + "load_key_and_certificates", + "load_pkcs12", + "serialize_java_truststore", + "serialize_key_and_certificates", +] + +PKCS12PrivateKeyTypes = typing.Union[ + rsa.RSAPrivateKey, + dsa.DSAPrivateKey, + ec.EllipticCurvePrivateKey, + ed25519.Ed25519PrivateKey, + ed448.Ed448PrivateKey, +] + + +PKCS12Certificate = rust_pkcs12.PKCS12Certificate + + +class PKCS12KeyAndCertificates: + def __init__( + self, + key: PrivateKeyTypes | None, + cert: PKCS12Certificate | None, + additional_certs: list[PKCS12Certificate], + ): + if key is not None and not isinstance( + key, + ( + rsa.RSAPrivateKey, + dsa.DSAPrivateKey, + ec.EllipticCurvePrivateKey, + ed25519.Ed25519PrivateKey, + ed448.Ed448PrivateKey, + ), + ): + raise TypeError( + "Key must be RSA, DSA, EllipticCurve, ED25519, or ED448" + " private key, or None." + ) + if cert is not None and not isinstance(cert, PKCS12Certificate): + raise TypeError("cert must be a PKCS12Certificate object or None") + if not all( + isinstance(add_cert, PKCS12Certificate) + for add_cert in additional_certs + ): + raise TypeError( + "all values in additional_certs must be PKCS12Certificate" + " objects" + ) + self._key = key + self._cert = cert + self._additional_certs = additional_certs + + @property + def key(self) -> PrivateKeyTypes | None: + return self._key + + @property + def cert(self) -> PKCS12Certificate | None: + return self._cert + + @property + def additional_certs(self) -> list[PKCS12Certificate]: + return self._additional_certs + + def __eq__(self, other: object) -> bool: + if not isinstance(other, PKCS12KeyAndCertificates): + return NotImplemented + + return ( + self.key == other.key + and self.cert == other.cert + and self.additional_certs == other.additional_certs + ) + + def __hash__(self) -> int: + return hash((self.key, self.cert, tuple(self.additional_certs))) + + def __repr__(self) -> str: + fmt = ( + "" + ) + return fmt.format(self.key, self.cert, self.additional_certs) + + +load_key_and_certificates = rust_pkcs12.load_key_and_certificates +load_pkcs12 = rust_pkcs12.load_pkcs12 + + +_PKCS12CATypes = typing.Union[ + x509.Certificate, + PKCS12Certificate, +] + + +def serialize_java_truststore( + certs: Iterable[PKCS12Certificate], + encryption_algorithm: serialization.KeySerializationEncryption, +) -> bytes: + if not certs: + raise ValueError("You must supply at least one cert") + + if not isinstance( + encryption_algorithm, serialization.KeySerializationEncryption + ): + raise TypeError( + "Key encryption algorithm must be a " + "KeySerializationEncryption instance" + ) + + return rust_pkcs12.serialize_java_truststore(certs, encryption_algorithm) + + +def serialize_key_and_certificates( + name: bytes | None, + key: PKCS12PrivateKeyTypes | None, + cert: x509.Certificate | None, + cas: Iterable[_PKCS12CATypes] | None, + encryption_algorithm: serialization.KeySerializationEncryption, +) -> bytes: + if key is not None and not isinstance( + key, + ( + rsa.RSAPrivateKey, + dsa.DSAPrivateKey, + ec.EllipticCurvePrivateKey, + ed25519.Ed25519PrivateKey, + ed448.Ed448PrivateKey, + ), + ): + raise TypeError( + "Key must be RSA, DSA, EllipticCurve, ED25519, or ED448" + " private key, or None." + ) + + if not isinstance( + encryption_algorithm, serialization.KeySerializationEncryption + ): + raise TypeError( + "Key encryption algorithm must be a " + "KeySerializationEncryption instance" + ) + + if key is None and cert is None and not cas: + raise ValueError("You must supply at least one of key, cert, or cas") + + return rust_pkcs12.serialize_key_and_certificates( + name, key, cert, cas, encryption_algorithm + ) diff --git a/py_modules/cryptography/hazmat/primitives/serialization/pkcs7.py b/py_modules/cryptography/hazmat/primitives/serialization/pkcs7.py new file mode 100644 index 0000000..76b667a --- /dev/null +++ b/py_modules/cryptography/hazmat/primitives/serialization/pkcs7.py @@ -0,0 +1,412 @@ +# This file is dual licensed under the terms of the Apache License, Version +# 2.0, and the BSD License. See the LICENSE file in the root of this repository +# for complete details. + +from __future__ import annotations + +import email.base64mime +import email.generator +import email.message +import email.policy +import io +import typing +from collections.abc import Iterable + +from cryptography import utils, x509 +from cryptography.exceptions import UnsupportedAlgorithm, _Reasons +from cryptography.hazmat.bindings._rust import pkcs7 as rust_pkcs7 +from cryptography.hazmat.primitives import hashes, serialization +from cryptography.hazmat.primitives.asymmetric import ec, padding, rsa +from cryptography.hazmat.primitives.ciphers import ( + algorithms, +) +from cryptography.utils import _check_byteslike + +load_pem_pkcs7_certificates = rust_pkcs7.load_pem_pkcs7_certificates + +load_der_pkcs7_certificates = rust_pkcs7.load_der_pkcs7_certificates + +serialize_certificates = rust_pkcs7.serialize_certificates + +PKCS7HashTypes = typing.Union[ + hashes.SHA224, + hashes.SHA256, + hashes.SHA384, + hashes.SHA512, +] + +PKCS7PrivateKeyTypes = typing.Union[ + rsa.RSAPrivateKey, ec.EllipticCurvePrivateKey +] + +ContentEncryptionAlgorithm = typing.Union[ + type[algorithms.AES128], type[algorithms.AES256] +] + + +class PKCS7Options(utils.Enum): + Text = "Add text/plain MIME type" + Binary = "Don't translate input data into canonical MIME format" + DetachedSignature = "Don't embed data in the PKCS7 structure" + NoCapabilities = "Don't embed SMIME capabilities" + NoAttributes = "Don't embed authenticatedAttributes" + NoCerts = "Don't embed signer certificate" + + +class PKCS7SignatureBuilder: + def __init__( + self, + data: utils.Buffer | None = None, + signers: list[ + tuple[ + x509.Certificate, + PKCS7PrivateKeyTypes, + PKCS7HashTypes, + padding.PSS | padding.PKCS1v15 | None, + ] + ] = [], + additional_certs: list[x509.Certificate] = [], + ): + self._data = data + self._signers = signers + self._additional_certs = additional_certs + + def set_data(self, data: utils.Buffer) -> PKCS7SignatureBuilder: + _check_byteslike("data", data) + if self._data is not None: + raise ValueError("data may only be set once") + + return PKCS7SignatureBuilder(data, self._signers) + + def add_signer( + self, + certificate: x509.Certificate, + private_key: PKCS7PrivateKeyTypes, + hash_algorithm: PKCS7HashTypes, + *, + rsa_padding: padding.PSS | padding.PKCS1v15 | None = None, + ) -> PKCS7SignatureBuilder: + if not isinstance( + hash_algorithm, + ( + hashes.SHA224, + hashes.SHA256, + hashes.SHA384, + hashes.SHA512, + ), + ): + raise TypeError( + "hash_algorithm must be one of hashes.SHA224, " + "SHA256, SHA384, or SHA512" + ) + if not isinstance(certificate, x509.Certificate): + raise TypeError("certificate must be a x509.Certificate") + + if not isinstance( + private_key, (rsa.RSAPrivateKey, ec.EllipticCurvePrivateKey) + ): + raise TypeError("Only RSA & EC keys are supported at this time.") + + if rsa_padding is not None: + if not isinstance(rsa_padding, (padding.PSS, padding.PKCS1v15)): + raise TypeError("Padding must be PSS or PKCS1v15") + if not isinstance(private_key, rsa.RSAPrivateKey): + raise TypeError("Padding is only supported for RSA keys") + + return PKCS7SignatureBuilder( + self._data, + [ + *self._signers, + (certificate, private_key, hash_algorithm, rsa_padding), + ], + ) + + def add_certificate( + self, certificate: x509.Certificate + ) -> PKCS7SignatureBuilder: + if not isinstance(certificate, x509.Certificate): + raise TypeError("certificate must be a x509.Certificate") + + return PKCS7SignatureBuilder( + self._data, self._signers, [*self._additional_certs, certificate] + ) + + def sign( + self, + encoding: serialization.Encoding, + options: Iterable[PKCS7Options], + backend: typing.Any = None, + ) -> bytes: + if len(self._signers) == 0: + raise ValueError("Must have at least one signer") + if self._data is None: + raise ValueError("You must add data to sign") + options = list(options) + if not all(isinstance(x, PKCS7Options) for x in options): + raise ValueError("options must be from the PKCS7Options enum") + if encoding not in ( + serialization.Encoding.PEM, + serialization.Encoding.DER, + serialization.Encoding.SMIME, + ): + raise ValueError( + "Must be PEM, DER, or SMIME from the Encoding enum" + ) + + # Text is a meaningless option unless it is accompanied by + # DetachedSignature + if ( + PKCS7Options.Text in options + and PKCS7Options.DetachedSignature not in options + ): + raise ValueError( + "When passing the Text option you must also pass " + "DetachedSignature" + ) + + if PKCS7Options.Text in options and encoding in ( + serialization.Encoding.DER, + serialization.Encoding.PEM, + ): + raise ValueError( + "The Text option is only available for SMIME serialization" + ) + + # No attributes implies no capabilities so we'll error if you try to + # pass both. + if ( + PKCS7Options.NoAttributes in options + and PKCS7Options.NoCapabilities in options + ): + raise ValueError( + "NoAttributes is a superset of NoCapabilities. Do not pass " + "both values." + ) + + return rust_pkcs7.sign_and_serialize(self, encoding, options) + + +class PKCS7EnvelopeBuilder: + def __init__( + self, + *, + _data: bytes | None = None, + _recipients: list[x509.Certificate] | None = None, + _content_encryption_algorithm: ContentEncryptionAlgorithm + | None = None, + ): + from cryptography.hazmat.backends.openssl.backend import ( + backend as ossl, + ) + + if not ossl.rsa_encryption_supported(padding=padding.PKCS1v15()): + raise UnsupportedAlgorithm( + "RSA with PKCS1 v1.5 padding is not supported by this version" + " of OpenSSL.", + _Reasons.UNSUPPORTED_PADDING, + ) + self._data = _data + self._recipients = _recipients if _recipients is not None else [] + self._content_encryption_algorithm = _content_encryption_algorithm + + def set_data(self, data: bytes) -> PKCS7EnvelopeBuilder: + _check_byteslike("data", data) + if self._data is not None: + raise ValueError("data may only be set once") + + return PKCS7EnvelopeBuilder( + _data=data, + _recipients=self._recipients, + _content_encryption_algorithm=self._content_encryption_algorithm, + ) + + def add_recipient( + self, + certificate: x509.Certificate, + ) -> PKCS7EnvelopeBuilder: + if not isinstance(certificate, x509.Certificate): + raise TypeError("certificate must be a x509.Certificate") + + if not isinstance(certificate.public_key(), rsa.RSAPublicKey): + raise TypeError("Only RSA keys are supported at this time.") + + return PKCS7EnvelopeBuilder( + _data=self._data, + _recipients=[ + *self._recipients, + certificate, + ], + _content_encryption_algorithm=self._content_encryption_algorithm, + ) + + def set_content_encryption_algorithm( + self, content_encryption_algorithm: ContentEncryptionAlgorithm + ) -> PKCS7EnvelopeBuilder: + if self._content_encryption_algorithm is not None: + raise ValueError("Content encryption algo may only be set once") + if content_encryption_algorithm not in { + algorithms.AES128, + algorithms.AES256, + }: + raise TypeError("Only AES128 and AES256 are supported") + + return PKCS7EnvelopeBuilder( + _data=self._data, + _recipients=self._recipients, + _content_encryption_algorithm=content_encryption_algorithm, + ) + + def encrypt( + self, + encoding: serialization.Encoding, + options: Iterable[PKCS7Options], + ) -> bytes: + if len(self._recipients) == 0: + raise ValueError("Must have at least one recipient") + if self._data is None: + raise ValueError("You must add data to encrypt") + + # The default content encryption algorithm is AES-128-CBC, which the + # S/MIME v3.2 RFC specifies as MUST support (https://datatracker.ietf.org/doc/html/rfc5751#section-2.7) + # however rest of S/MIME v3.2 is not currently supported + content_encryption_algorithm = ( + self._content_encryption_algorithm or algorithms.AES128 + ) + + options = list(options) + if not all(isinstance(x, PKCS7Options) for x in options): + raise ValueError("options must be from the PKCS7Options enum") + if encoding not in ( + serialization.Encoding.PEM, + serialization.Encoding.DER, + serialization.Encoding.SMIME, + ): + raise ValueError( + "Must be PEM, DER, or SMIME from the Encoding enum" + ) + + # Only allow options that make sense for encryption + if any( + opt not in [PKCS7Options.Text, PKCS7Options.Binary] + for opt in options + ): + raise ValueError( + "Only the following options are supported for encryption: " + "Text, Binary" + ) + elif PKCS7Options.Text in options and PKCS7Options.Binary in options: + # OpenSSL accepts both options at the same time, but ignores Text. + # We fail defensively to avoid unexpected outputs. + raise ValueError( + "Cannot use Binary and Text options at the same time" + ) + + return rust_pkcs7.encrypt_and_serialize( + self, content_encryption_algorithm, encoding, options + ) + + +pkcs7_decrypt_der = rust_pkcs7.decrypt_der +pkcs7_decrypt_pem = rust_pkcs7.decrypt_pem +pkcs7_decrypt_smime = rust_pkcs7.decrypt_smime + + +def _smime_signed_encode( + data: bytes, signature: bytes, micalg: str, text_mode: bool +) -> bytes: + # This function works pretty hard to replicate what OpenSSL does + # precisely. For good and for ill. + + m = email.message.Message() + m.add_header("MIME-Version", "1.0") + m.add_header( + "Content-Type", + "multipart/signed", + protocol="application/x-pkcs7-signature", + micalg=micalg, + ) + + m.preamble = "This is an S/MIME signed message\n" + + msg_part = OpenSSLMimePart() + msg_part.set_payload(data) + if text_mode: + msg_part.add_header("Content-Type", "text/plain") + m.attach(msg_part) + + sig_part = email.message.MIMEPart() + sig_part.add_header( + "Content-Type", "application/x-pkcs7-signature", name="smime.p7s" + ) + sig_part.add_header("Content-Transfer-Encoding", "base64") + sig_part.add_header( + "Content-Disposition", "attachment", filename="smime.p7s" + ) + sig_part.set_payload( + email.base64mime.body_encode(signature, maxlinelen=65) + ) + del sig_part["MIME-Version"] + m.attach(sig_part) + + fp = io.BytesIO() + g = email.generator.BytesGenerator( + fp, + maxheaderlen=0, + mangle_from_=False, + policy=m.policy.clone(linesep="\r\n"), + ) + g.flatten(m) + return fp.getvalue() + + +def _smime_enveloped_encode(data: bytes) -> bytes: + m = email.message.Message() + m.add_header("MIME-Version", "1.0") + m.add_header("Content-Disposition", "attachment", filename="smime.p7m") + m.add_header( + "Content-Type", + "application/pkcs7-mime", + smime_type="enveloped-data", + name="smime.p7m", + ) + m.add_header("Content-Transfer-Encoding", "base64") + + m.set_payload(email.base64mime.body_encode(data, maxlinelen=65)) + + return m.as_bytes(policy=m.policy.clone(linesep="\n", max_line_length=0)) + + +def _smime_enveloped_decode(data: bytes) -> bytes: + m = email.message_from_bytes(data) + if m.get_content_type() not in { + "application/x-pkcs7-mime", + "application/pkcs7-mime", + }: + raise ValueError("Not an S/MIME enveloped message") + return bytes(m.get_payload(decode=True)) + + +def _smime_remove_text_headers(data: bytes) -> bytes: + m = email.message_from_bytes(data) + # Using get() instead of get_content_type() since it has None as default, + # where the latter has "text/plain". Both methods are case-insensitive. + content_type = m.get("content-type") + if content_type is None: + raise ValueError( + "Decrypted MIME data has no 'Content-Type' header. " + "Please remove the 'Text' option to parse it manually." + ) + if "text/plain" not in content_type: + raise ValueError( + f"Decrypted MIME data content type is '{content_type}', not " + "'text/plain'. Remove the 'Text' option to parse it manually." + ) + return bytes(m.get_payload(decode=True)) + + +class OpenSSLMimePart(email.message.MIMEPart): + # A MIMEPart subclass that replicates OpenSSL's behavior of not including + # a newline if there are no headers. + def _write_headers(self, generator) -> None: + if list(self.raw_items()): + generator._write_headers(self) diff --git a/py_modules/cryptography/hazmat/primitives/serialization/ssh.py b/py_modules/cryptography/hazmat/primitives/serialization/ssh.py new file mode 100644 index 0000000..230e147 --- /dev/null +++ b/py_modules/cryptography/hazmat/primitives/serialization/ssh.py @@ -0,0 +1,1615 @@ +# This file is dual licensed under the terms of the Apache License, Version +# 2.0, and the BSD License. See the LICENSE file in the root of this repository +# for complete details. + +from __future__ import annotations + +import binascii +import enum +import os +import re +import typing +import warnings +from dataclasses import dataclass + +from cryptography import utils +from cryptography.exceptions import UnsupportedAlgorithm +from cryptography.hazmat.primitives import hashes +from cryptography.hazmat.primitives.asymmetric import ( + dsa, + ec, + ed25519, + padding, + rsa, +) +from cryptography.hazmat.primitives.asymmetric import utils as asym_utils +from cryptography.hazmat.primitives.ciphers import ( + AEADDecryptionContext, + Cipher, + algorithms, + modes, +) +from cryptography.hazmat.primitives.serialization import ( + Encoding, + KeySerializationEncryption, + NoEncryption, + PrivateFormat, + PublicFormat, + _KeySerializationEncryption, +) + +try: + from bcrypt import kdf as _bcrypt_kdf + + _bcrypt_supported = True +except ImportError: + _bcrypt_supported = False + + def _bcrypt_kdf( + password: bytes, + salt: bytes, + desired_key_bytes: int, + rounds: int, + ignore_few_rounds: bool = False, + ) -> bytes: + raise UnsupportedAlgorithm("Need bcrypt module") + + +_SSH_ED25519 = b"ssh-ed25519" +_SSH_RSA = b"ssh-rsa" +_SSH_DSA = b"ssh-dss" +_ECDSA_NISTP256 = b"ecdsa-sha2-nistp256" +_ECDSA_NISTP384 = b"ecdsa-sha2-nistp384" +_ECDSA_NISTP521 = b"ecdsa-sha2-nistp521" +_CERT_SUFFIX = b"-cert-v01@openssh.com" + +# U2F application string suffixed pubkey +_SK_SSH_ED25519 = b"sk-ssh-ed25519@openssh.com" +_SK_SSH_ECDSA_NISTP256 = b"sk-ecdsa-sha2-nistp256@openssh.com" + +# These are not key types, only algorithms, so they cannot appear +# as a public key type +_SSH_RSA_SHA256 = b"rsa-sha2-256" +_SSH_RSA_SHA512 = b"rsa-sha2-512" + +_SSH_PUBKEY_RC = re.compile(rb"\A(\S+)[ \t]+(\S+)") +_SK_MAGIC = b"openssh-key-v1\0" +_SK_START = b"-----BEGIN OPENSSH PRIVATE KEY-----" +_SK_END = b"-----END OPENSSH PRIVATE KEY-----" +_BCRYPT = b"bcrypt" +_NONE = b"none" +_DEFAULT_CIPHER = b"aes256-ctr" +_DEFAULT_ROUNDS = 16 + +# re is only way to work on bytes-like data +_PEM_RC = re.compile(_SK_START + b"(.*?)" + _SK_END, re.DOTALL) + +# padding for max blocksize +_PADDING = memoryview(bytearray(range(1, 1 + 16))) + + +@dataclass +class _SSHCipher: + alg: type[algorithms.AES] + key_len: int + mode: type[modes.CTR] | type[modes.CBC] | type[modes.GCM] + block_len: int + iv_len: int + tag_len: int | None + is_aead: bool + + +# ciphers that are actually used in key wrapping +_SSH_CIPHERS: dict[bytes, _SSHCipher] = { + b"aes256-ctr": _SSHCipher( + alg=algorithms.AES, + key_len=32, + mode=modes.CTR, + block_len=16, + iv_len=16, + tag_len=None, + is_aead=False, + ), + b"aes256-cbc": _SSHCipher( + alg=algorithms.AES, + key_len=32, + mode=modes.CBC, + block_len=16, + iv_len=16, + tag_len=None, + is_aead=False, + ), + b"aes256-gcm@openssh.com": _SSHCipher( + alg=algorithms.AES, + key_len=32, + mode=modes.GCM, + block_len=16, + iv_len=12, + tag_len=16, + is_aead=True, + ), +} + +# map local curve name to key type +_ECDSA_KEY_TYPE = { + "secp256r1": _ECDSA_NISTP256, + "secp384r1": _ECDSA_NISTP384, + "secp521r1": _ECDSA_NISTP521, +} + + +def _get_ssh_key_type(key: SSHPrivateKeyTypes | SSHPublicKeyTypes) -> bytes: + if isinstance(key, ec.EllipticCurvePrivateKey): + key_type = _ecdsa_key_type(key.public_key()) + elif isinstance(key, ec.EllipticCurvePublicKey): + key_type = _ecdsa_key_type(key) + elif isinstance(key, (rsa.RSAPrivateKey, rsa.RSAPublicKey)): + key_type = _SSH_RSA + elif isinstance(key, (dsa.DSAPrivateKey, dsa.DSAPublicKey)): + key_type = _SSH_DSA + elif isinstance( + key, (ed25519.Ed25519PrivateKey, ed25519.Ed25519PublicKey) + ): + key_type = _SSH_ED25519 + else: + raise ValueError("Unsupported key type") + + return key_type + + +def _ecdsa_key_type(public_key: ec.EllipticCurvePublicKey) -> bytes: + """Return SSH key_type and curve_name for private key.""" + curve = public_key.curve + if curve.name not in _ECDSA_KEY_TYPE: + raise ValueError( + f"Unsupported curve for ssh private key: {curve.name!r}" + ) + return _ECDSA_KEY_TYPE[curve.name] + + +def _check_block_size(data: utils.Buffer, block_len: int) -> None: + """Require data to be full blocks""" + if not data or len(data) % block_len != 0: + raise ValueError("Corrupt data: missing padding") + + +def _check_empty(data: utils.Buffer) -> None: + """All data should have been parsed.""" + if data: + raise ValueError("Corrupt data: unparsed data") + + +def _init_cipher( + ciphername: bytes, + password: bytes | None, + salt: bytes, + rounds: int, +) -> Cipher[modes.CBC | modes.CTR | modes.GCM]: + """Generate key + iv and return cipher.""" + if not password: + raise TypeError( + "Key is password-protected, but password was not provided." + ) + + ciph = _SSH_CIPHERS[ciphername] + seed = _bcrypt_kdf( + password, salt, ciph.key_len + ciph.iv_len, rounds, True + ) + return Cipher( + ciph.alg(seed[: ciph.key_len]), + ciph.mode(seed[ciph.key_len :]), + ) + + +def _get_u32(data: memoryview) -> tuple[int, memoryview]: + """Uint32""" + if len(data) < 4: + raise ValueError("Invalid data") + return int.from_bytes(data[:4], byteorder="big"), data[4:] + + +def _get_u64(data: memoryview) -> tuple[int, memoryview]: + """Uint64""" + if len(data) < 8: + raise ValueError("Invalid data") + return int.from_bytes(data[:8], byteorder="big"), data[8:] + + +def _get_sshstr(data: memoryview) -> tuple[memoryview, memoryview]: + """Bytes with u32 length prefix""" + n, data = _get_u32(data) + if n > len(data): + raise ValueError("Invalid data") + return data[:n], data[n:] + + +def _get_mpint(data: memoryview) -> tuple[int, memoryview]: + """Big integer.""" + val, data = _get_sshstr(data) + if val and val[0] > 0x7F: + raise ValueError("Invalid data") + return int.from_bytes(val, "big"), data + + +def _to_mpint(val: int) -> bytes: + """Storage format for signed bigint.""" + if val < 0: + raise ValueError("negative mpint not allowed") + if not val: + return b"" + nbytes = (val.bit_length() + 8) // 8 + return utils.int_to_bytes(val, nbytes) + + +class _FragList: + """Build recursive structure without data copy.""" + + flist: list[utils.Buffer] + + def __init__(self, init: list[utils.Buffer] | None = None) -> None: + self.flist = [] + if init: + self.flist.extend(init) + + def put_raw(self, val: utils.Buffer) -> None: + """Add plain bytes""" + self.flist.append(val) + + def put_u32(self, val: int) -> None: + """Big-endian uint32""" + self.flist.append(val.to_bytes(length=4, byteorder="big")) + + def put_u64(self, val: int) -> None: + """Big-endian uint64""" + self.flist.append(val.to_bytes(length=8, byteorder="big")) + + def put_sshstr(self, val: bytes | _FragList) -> None: + """Bytes prefixed with u32 length""" + if isinstance(val, (bytes, memoryview, bytearray)): + self.put_u32(len(val)) + self.flist.append(val) + else: + self.put_u32(val.size()) + self.flist.extend(val.flist) + + def put_mpint(self, val: int) -> None: + """Big-endian bigint prefixed with u32 length""" + self.put_sshstr(_to_mpint(val)) + + def size(self) -> int: + """Current number of bytes""" + return sum(map(len, self.flist)) + + def render(self, dstbuf: memoryview, pos: int = 0) -> int: + """Write into bytearray""" + for frag in self.flist: + flen = len(frag) + start, pos = pos, pos + flen + dstbuf[start:pos] = frag + return pos + + def tobytes(self) -> bytes: + """Return as bytes""" + buf = memoryview(bytearray(self.size())) + self.render(buf) + return buf.tobytes() + + +class _SSHFormatRSA: + """Format for RSA keys. + + Public: + mpint e, n + Private: + mpint n, e, d, iqmp, p, q + """ + + def get_public( + self, data: memoryview + ) -> tuple[tuple[int, int], memoryview]: + """RSA public fields""" + e, data = _get_mpint(data) + n, data = _get_mpint(data) + return (e, n), data + + def load_public( + self, data: memoryview + ) -> tuple[rsa.RSAPublicKey, memoryview]: + """Make RSA public key from data.""" + (e, n), data = self.get_public(data) + public_numbers = rsa.RSAPublicNumbers(e, n) + public_key = public_numbers.public_key() + return public_key, data + + def load_private( + self, data: memoryview, pubfields, unsafe_skip_rsa_key_validation: bool + ) -> tuple[rsa.RSAPrivateKey, memoryview]: + """Make RSA private key from data.""" + n, data = _get_mpint(data) + e, data = _get_mpint(data) + d, data = _get_mpint(data) + iqmp, data = _get_mpint(data) + p, data = _get_mpint(data) + q, data = _get_mpint(data) + + if (e, n) != pubfields: + raise ValueError("Corrupt data: rsa field mismatch") + dmp1 = rsa.rsa_crt_dmp1(d, p) + dmq1 = rsa.rsa_crt_dmq1(d, q) + public_numbers = rsa.RSAPublicNumbers(e, n) + private_numbers = rsa.RSAPrivateNumbers( + p, q, d, dmp1, dmq1, iqmp, public_numbers + ) + private_key = private_numbers.private_key( + unsafe_skip_rsa_key_validation=unsafe_skip_rsa_key_validation + ) + return private_key, data + + def encode_public( + self, public_key: rsa.RSAPublicKey, f_pub: _FragList + ) -> None: + """Write RSA public key""" + pubn = public_key.public_numbers() + f_pub.put_mpint(pubn.e) + f_pub.put_mpint(pubn.n) + + def encode_private( + self, private_key: rsa.RSAPrivateKey, f_priv: _FragList + ) -> None: + """Write RSA private key""" + private_numbers = private_key.private_numbers() + public_numbers = private_numbers.public_numbers + + f_priv.put_mpint(public_numbers.n) + f_priv.put_mpint(public_numbers.e) + + f_priv.put_mpint(private_numbers.d) + f_priv.put_mpint(private_numbers.iqmp) + f_priv.put_mpint(private_numbers.p) + f_priv.put_mpint(private_numbers.q) + + +class _SSHFormatDSA: + """Format for DSA keys. + + Public: + mpint p, q, g, y + Private: + mpint p, q, g, y, x + """ + + def get_public(self, data: memoryview) -> tuple[tuple, memoryview]: + """DSA public fields""" + p, data = _get_mpint(data) + q, data = _get_mpint(data) + g, data = _get_mpint(data) + y, data = _get_mpint(data) + return (p, q, g, y), data + + def load_public( + self, data: memoryview + ) -> tuple[dsa.DSAPublicKey, memoryview]: + """Make DSA public key from data.""" + (p, q, g, y), data = self.get_public(data) + parameter_numbers = dsa.DSAParameterNumbers(p, q, g) + public_numbers = dsa.DSAPublicNumbers(y, parameter_numbers) + self._validate(public_numbers) + public_key = public_numbers.public_key() + return public_key, data + + def load_private( + self, data: memoryview, pubfields, unsafe_skip_rsa_key_validation: bool + ) -> tuple[dsa.DSAPrivateKey, memoryview]: + """Make DSA private key from data.""" + (p, q, g, y), data = self.get_public(data) + x, data = _get_mpint(data) + + if (p, q, g, y) != pubfields: + raise ValueError("Corrupt data: dsa field mismatch") + parameter_numbers = dsa.DSAParameterNumbers(p, q, g) + public_numbers = dsa.DSAPublicNumbers(y, parameter_numbers) + self._validate(public_numbers) + private_numbers = dsa.DSAPrivateNumbers(x, public_numbers) + private_key = private_numbers.private_key() + return private_key, data + + def encode_public( + self, public_key: dsa.DSAPublicKey, f_pub: _FragList + ) -> None: + """Write DSA public key""" + public_numbers = public_key.public_numbers() + parameter_numbers = public_numbers.parameter_numbers + self._validate(public_numbers) + + f_pub.put_mpint(parameter_numbers.p) + f_pub.put_mpint(parameter_numbers.q) + f_pub.put_mpint(parameter_numbers.g) + f_pub.put_mpint(public_numbers.y) + + def encode_private( + self, private_key: dsa.DSAPrivateKey, f_priv: _FragList + ) -> None: + """Write DSA private key""" + self.encode_public(private_key.public_key(), f_priv) + f_priv.put_mpint(private_key.private_numbers().x) + + def _validate(self, public_numbers: dsa.DSAPublicNumbers) -> None: + parameter_numbers = public_numbers.parameter_numbers + if parameter_numbers.p.bit_length() != 1024: + raise ValueError("SSH supports only 1024 bit DSA keys") + + +class _SSHFormatECDSA: + """Format for ECDSA keys. + + Public: + str curve + bytes point + Private: + str curve + bytes point + mpint secret + """ + + def __init__(self, ssh_curve_name: bytes, curve: ec.EllipticCurve): + self.ssh_curve_name = ssh_curve_name + self.curve = curve + + def get_public( + self, data: memoryview + ) -> tuple[tuple[memoryview, memoryview], memoryview]: + """ECDSA public fields""" + curve, data = _get_sshstr(data) + point, data = _get_sshstr(data) + if curve != self.ssh_curve_name: + raise ValueError("Curve name mismatch") + if len(point) == 0: + raise ValueError("Invalid EC point: empty data") + if point[0] != 4: + raise NotImplementedError("Need uncompressed point") + return (curve, point), data + + def load_public( + self, data: memoryview + ) -> tuple[ec.EllipticCurvePublicKey, memoryview]: + """Make ECDSA public key from data.""" + (_, point), data = self.get_public(data) + public_key = ec.EllipticCurvePublicKey.from_encoded_point( + self.curve, point.tobytes() + ) + return public_key, data + + def load_private( + self, data: memoryview, pubfields, unsafe_skip_rsa_key_validation: bool + ) -> tuple[ec.EllipticCurvePrivateKey, memoryview]: + """Make ECDSA private key from data.""" + (curve_name, point), data = self.get_public(data) + secret, data = _get_mpint(data) + + if (curve_name, point) != pubfields: + raise ValueError("Corrupt data: ecdsa field mismatch") + private_key = ec.derive_private_key(secret, self.curve) + return private_key, data + + def encode_public( + self, public_key: ec.EllipticCurvePublicKey, f_pub: _FragList + ) -> None: + """Write ECDSA public key""" + point = public_key.public_bytes( + Encoding.X962, PublicFormat.UncompressedPoint + ) + f_pub.put_sshstr(self.ssh_curve_name) + f_pub.put_sshstr(point) + + def encode_private( + self, private_key: ec.EllipticCurvePrivateKey, f_priv: _FragList + ) -> None: + """Write ECDSA private key""" + public_key = private_key.public_key() + private_numbers = private_key.private_numbers() + + self.encode_public(public_key, f_priv) + f_priv.put_mpint(private_numbers.private_value) + + +class _SSHFormatEd25519: + """Format for Ed25519 keys. + + Public: + bytes point + Private: + bytes point + bytes secret_and_point + """ + + def get_public( + self, data: memoryview + ) -> tuple[tuple[memoryview], memoryview]: + """Ed25519 public fields""" + point, data = _get_sshstr(data) + return (point,), data + + def load_public( + self, data: memoryview + ) -> tuple[ed25519.Ed25519PublicKey, memoryview]: + """Make Ed25519 public key from data.""" + (point,), data = self.get_public(data) + public_key = ed25519.Ed25519PublicKey.from_public_bytes( + point.tobytes() + ) + return public_key, data + + def load_private( + self, data: memoryview, pubfields, unsafe_skip_rsa_key_validation: bool + ) -> tuple[ed25519.Ed25519PrivateKey, memoryview]: + """Make Ed25519 private key from data.""" + (point,), data = self.get_public(data) + keypair, data = _get_sshstr(data) + + secret = keypair[:32] + point2 = keypair[32:] + if point != point2 or (point,) != pubfields: + raise ValueError("Corrupt data: ed25519 field mismatch") + private_key = ed25519.Ed25519PrivateKey.from_private_bytes(secret) + return private_key, data + + def encode_public( + self, public_key: ed25519.Ed25519PublicKey, f_pub: _FragList + ) -> None: + """Write Ed25519 public key""" + raw_public_key = public_key.public_bytes( + Encoding.Raw, PublicFormat.Raw + ) + f_pub.put_sshstr(raw_public_key) + + def encode_private( + self, private_key: ed25519.Ed25519PrivateKey, f_priv: _FragList + ) -> None: + """Write Ed25519 private key""" + public_key = private_key.public_key() + raw_private_key = private_key.private_bytes( + Encoding.Raw, PrivateFormat.Raw, NoEncryption() + ) + raw_public_key = public_key.public_bytes( + Encoding.Raw, PublicFormat.Raw + ) + f_keypair = _FragList([raw_private_key, raw_public_key]) + + self.encode_public(public_key, f_priv) + f_priv.put_sshstr(f_keypair) + + +def load_application(data) -> tuple[memoryview, memoryview]: + """ + U2F application strings + """ + application, data = _get_sshstr(data) + if not application.tobytes().startswith(b"ssh:"): + raise ValueError( + "U2F application string does not start with b'ssh:' " + f"({application})" + ) + return application, data + + +class _SSHFormatSKEd25519: + """ + The format of a sk-ssh-ed25519@openssh.com public key is: + + string "sk-ssh-ed25519@openssh.com" + string public key + string application (user-specified, but typically "ssh:") + """ + + def load_public( + self, data: memoryview + ) -> tuple[ed25519.Ed25519PublicKey, memoryview]: + """Make Ed25519 public key from data.""" + public_key, data = _lookup_kformat(_SSH_ED25519).load_public(data) + _, data = load_application(data) + return public_key, data + + def get_public(self, data: memoryview) -> typing.NoReturn: + # Confusingly `get_public` is an entry point used by private key + # loading. + raise UnsupportedAlgorithm( + "sk-ssh-ed25519 private keys cannot be loaded" + ) + + +class _SSHFormatSKECDSA: + """ + The format of a sk-ecdsa-sha2-nistp256@openssh.com public key is: + + string "sk-ecdsa-sha2-nistp256@openssh.com" + string curve name + ec_point Q + string application (user-specified, but typically "ssh:") + """ + + def load_public( + self, data: memoryview + ) -> tuple[ec.EllipticCurvePublicKey, memoryview]: + """Make ECDSA public key from data.""" + public_key, data = _lookup_kformat(_ECDSA_NISTP256).load_public(data) + _, data = load_application(data) + return public_key, data + + def get_public(self, data: memoryview) -> typing.NoReturn: + # Confusingly `get_public` is an entry point used by private key + # loading. + raise UnsupportedAlgorithm( + "sk-ecdsa-sha2-nistp256 private keys cannot be loaded" + ) + + +_KEY_FORMATS = { + _SSH_RSA: _SSHFormatRSA(), + _SSH_DSA: _SSHFormatDSA(), + _SSH_ED25519: _SSHFormatEd25519(), + _ECDSA_NISTP256: _SSHFormatECDSA(b"nistp256", ec.SECP256R1()), + _ECDSA_NISTP384: _SSHFormatECDSA(b"nistp384", ec.SECP384R1()), + _ECDSA_NISTP521: _SSHFormatECDSA(b"nistp521", ec.SECP521R1()), + _SK_SSH_ED25519: _SSHFormatSKEd25519(), + _SK_SSH_ECDSA_NISTP256: _SSHFormatSKECDSA(), +} + + +def _lookup_kformat(key_type: utils.Buffer): + """Return valid format or throw error""" + if not isinstance(key_type, bytes): + key_type = memoryview(key_type).tobytes() + if key_type in _KEY_FORMATS: + return _KEY_FORMATS[key_type] + raise UnsupportedAlgorithm(f"Unsupported key type: {key_type!r}") + + +SSHPrivateKeyTypes = typing.Union[ + ec.EllipticCurvePrivateKey, + rsa.RSAPrivateKey, + dsa.DSAPrivateKey, + ed25519.Ed25519PrivateKey, +] + + +def load_ssh_private_key( + data: utils.Buffer, + password: bytes | None, + backend: typing.Any = None, + *, + unsafe_skip_rsa_key_validation: bool = False, +) -> SSHPrivateKeyTypes: + """Load private key from OpenSSH custom encoding.""" + utils._check_byteslike("data", data) + if password is not None: + utils._check_bytes("password", password) + + m = _PEM_RC.search(data) + if not m: + raise ValueError("Not OpenSSH private key format") + p1 = m.start(1) + p2 = m.end(1) + data = binascii.a2b_base64(memoryview(data)[p1:p2]) + if not data.startswith(_SK_MAGIC): + raise ValueError("Not OpenSSH private key format") + data = memoryview(data)[len(_SK_MAGIC) :] + + # parse header + ciphername, data = _get_sshstr(data) + kdfname, data = _get_sshstr(data) + kdfoptions, data = _get_sshstr(data) + nkeys, data = _get_u32(data) + if nkeys != 1: + raise ValueError("Only one key supported") + + # load public key data + pubdata, data = _get_sshstr(data) + pub_key_type, pubdata = _get_sshstr(pubdata) + kformat = _lookup_kformat(pub_key_type) + pubfields, pubdata = kformat.get_public(pubdata) + _check_empty(pubdata) + + if ciphername != _NONE or kdfname != _NONE: + ciphername_bytes = ciphername.tobytes() + if ciphername_bytes not in _SSH_CIPHERS: + raise UnsupportedAlgorithm( + f"Unsupported cipher: {ciphername_bytes!r}" + ) + if kdfname != _BCRYPT: + raise UnsupportedAlgorithm(f"Unsupported KDF: {kdfname!r}") + blklen = _SSH_CIPHERS[ciphername_bytes].block_len + tag_len = _SSH_CIPHERS[ciphername_bytes].tag_len + # load secret data + edata, data = _get_sshstr(data) + # see https://bugzilla.mindrot.org/show_bug.cgi?id=3553 for + # information about how OpenSSH handles AEAD tags + if _SSH_CIPHERS[ciphername_bytes].is_aead: + tag = bytes(data) + if len(tag) != tag_len: + raise ValueError("Corrupt data: invalid tag length for cipher") + else: + _check_empty(data) + _check_block_size(edata, blklen) + salt, kbuf = _get_sshstr(kdfoptions) + rounds, kbuf = _get_u32(kbuf) + _check_empty(kbuf) + ciph = _init_cipher(ciphername_bytes, password, salt.tobytes(), rounds) + dec = ciph.decryptor() + edata = memoryview(dec.update(edata)) + if _SSH_CIPHERS[ciphername_bytes].is_aead: + assert isinstance(dec, AEADDecryptionContext) + _check_empty(dec.finalize_with_tag(tag)) + else: + # _check_block_size requires data to be a full block so there + # should be no output from finalize + _check_empty(dec.finalize()) + else: + if password: + raise TypeError( + "Password was given but private key is not encrypted." + ) + # load secret data + edata, data = _get_sshstr(data) + _check_empty(data) + blklen = 8 + _check_block_size(edata, blklen) + ck1, edata = _get_u32(edata) + ck2, edata = _get_u32(edata) + if ck1 != ck2: + raise ValueError("Corrupt data: broken checksum") + + # load per-key struct + key_type, edata = _get_sshstr(edata) + if key_type != pub_key_type: + raise ValueError("Corrupt data: key type mismatch") + private_key, edata = kformat.load_private( + edata, + pubfields, + unsafe_skip_rsa_key_validation=unsafe_skip_rsa_key_validation, + ) + # We don't use the comment + _, edata = _get_sshstr(edata) + + # yes, SSH does padding check *after* all other parsing is done. + # need to follow as it writes zero-byte padding too. + if edata != _PADDING[: len(edata)]: + raise ValueError("Corrupt data: invalid padding") + + if isinstance(private_key, dsa.DSAPrivateKey): + warnings.warn( + "SSH DSA keys are deprecated and will be removed in a future " + "release.", + utils.DeprecatedIn40, + stacklevel=2, + ) + + return private_key + + +def _serialize_ssh_private_key( + private_key: SSHPrivateKeyTypes, + password: bytes, + encryption_algorithm: KeySerializationEncryption, +) -> bytes: + """Serialize private key with OpenSSH custom encoding.""" + utils._check_bytes("password", password) + if isinstance(private_key, dsa.DSAPrivateKey): + warnings.warn( + "SSH DSA key support is deprecated and will be " + "removed in a future release", + utils.DeprecatedIn40, + stacklevel=4, + ) + + key_type = _get_ssh_key_type(private_key) + kformat = _lookup_kformat(key_type) + + # setup parameters + f_kdfoptions = _FragList() + if password: + ciphername = _DEFAULT_CIPHER + blklen = _SSH_CIPHERS[ciphername].block_len + kdfname = _BCRYPT + rounds = _DEFAULT_ROUNDS + if ( + isinstance(encryption_algorithm, _KeySerializationEncryption) + and encryption_algorithm._kdf_rounds is not None + ): + rounds = encryption_algorithm._kdf_rounds + salt = os.urandom(16) + f_kdfoptions.put_sshstr(salt) + f_kdfoptions.put_u32(rounds) + ciph = _init_cipher(ciphername, password, salt, rounds) + else: + ciphername = kdfname = _NONE + blklen = 8 + ciph = None + nkeys = 1 + checkval = os.urandom(4) + comment = b"" + + # encode public and private parts together + f_public_key = _FragList() + f_public_key.put_sshstr(key_type) + kformat.encode_public(private_key.public_key(), f_public_key) + + f_secrets = _FragList([checkval, checkval]) + f_secrets.put_sshstr(key_type) + kformat.encode_private(private_key, f_secrets) + f_secrets.put_sshstr(comment) + f_secrets.put_raw(_PADDING[: blklen - (f_secrets.size() % blklen)]) + + # top-level structure + f_main = _FragList() + f_main.put_raw(_SK_MAGIC) + f_main.put_sshstr(ciphername) + f_main.put_sshstr(kdfname) + f_main.put_sshstr(f_kdfoptions) + f_main.put_u32(nkeys) + f_main.put_sshstr(f_public_key) + f_main.put_sshstr(f_secrets) + + # copy result info bytearray + slen = f_secrets.size() + mlen = f_main.size() + buf = memoryview(bytearray(mlen + blklen)) + f_main.render(buf) + ofs = mlen - slen + + # encrypt in-place + if ciph is not None: + ciph.encryptor().update_into(buf[ofs:mlen], buf[ofs:]) + + return bytes(buf[:mlen]) + + +SSHPublicKeyTypes = typing.Union[ + ec.EllipticCurvePublicKey, + rsa.RSAPublicKey, + dsa.DSAPublicKey, + ed25519.Ed25519PublicKey, +] + +SSHCertPublicKeyTypes = typing.Union[ + ec.EllipticCurvePublicKey, + rsa.RSAPublicKey, + ed25519.Ed25519PublicKey, +] + + +class SSHCertificateType(enum.Enum): + USER = 1 + HOST = 2 + + +class SSHCertificate: + def __init__( + self, + _nonce: memoryview, + _public_key: SSHPublicKeyTypes, + _serial: int, + _cctype: int, + _key_id: memoryview, + _valid_principals: list[bytes], + _valid_after: int, + _valid_before: int, + _critical_options: dict[bytes, bytes], + _extensions: dict[bytes, bytes], + _sig_type: memoryview, + _sig_key: memoryview, + _inner_sig_type: memoryview, + _signature: memoryview, + _tbs_cert_body: memoryview, + _cert_key_type: bytes, + _cert_body: memoryview, + ): + self._nonce = _nonce + self._public_key = _public_key + self._serial = _serial + try: + self._type = SSHCertificateType(_cctype) + except ValueError: + raise ValueError("Invalid certificate type") + self._key_id = _key_id + self._valid_principals = _valid_principals + self._valid_after = _valid_after + self._valid_before = _valid_before + self._critical_options = _critical_options + self._extensions = _extensions + self._sig_type = _sig_type + self._sig_key = _sig_key + self._inner_sig_type = _inner_sig_type + self._signature = _signature + self._cert_key_type = _cert_key_type + self._cert_body = _cert_body + self._tbs_cert_body = _tbs_cert_body + + @property + def nonce(self) -> bytes: + return bytes(self._nonce) + + def public_key(self) -> SSHCertPublicKeyTypes: + # make mypy happy until we remove DSA support entirely and + # the underlying union won't have a disallowed type + return typing.cast(SSHCertPublicKeyTypes, self._public_key) + + @property + def serial(self) -> int: + return self._serial + + @property + def type(self) -> SSHCertificateType: + return self._type + + @property + def key_id(self) -> bytes: + return bytes(self._key_id) + + @property + def valid_principals(self) -> list[bytes]: + return self._valid_principals + + @property + def valid_before(self) -> int: + return self._valid_before + + @property + def valid_after(self) -> int: + return self._valid_after + + @property + def critical_options(self) -> dict[bytes, bytes]: + return self._critical_options + + @property + def extensions(self) -> dict[bytes, bytes]: + return self._extensions + + def signature_key(self) -> SSHCertPublicKeyTypes: + sigformat = _lookup_kformat(self._sig_type) + signature_key, sigkey_rest = sigformat.load_public(self._sig_key) + _check_empty(sigkey_rest) + return signature_key + + def public_bytes(self) -> bytes: + return ( + bytes(self._cert_key_type) + + b" " + + binascii.b2a_base64(bytes(self._cert_body), newline=False) + ) + + def verify_cert_signature(self) -> None: + signature_key = self.signature_key() + if isinstance(signature_key, ed25519.Ed25519PublicKey): + signature_key.verify( + bytes(self._signature), bytes(self._tbs_cert_body) + ) + elif isinstance(signature_key, ec.EllipticCurvePublicKey): + # The signature is encoded as a pair of big-endian integers + r, data = _get_mpint(self._signature) + s, data = _get_mpint(data) + _check_empty(data) + computed_sig = asym_utils.encode_dss_signature(r, s) + hash_alg = _get_ec_hash_alg(signature_key.curve) + signature_key.verify( + computed_sig, bytes(self._tbs_cert_body), ec.ECDSA(hash_alg) + ) + else: + assert isinstance(signature_key, rsa.RSAPublicKey) + if self._inner_sig_type == _SSH_RSA: + hash_alg = hashes.SHA1() + elif self._inner_sig_type == _SSH_RSA_SHA256: + hash_alg = hashes.SHA256() + else: + assert self._inner_sig_type == _SSH_RSA_SHA512 + hash_alg = hashes.SHA512() + signature_key.verify( + bytes(self._signature), + bytes(self._tbs_cert_body), + padding.PKCS1v15(), + hash_alg, + ) + + +def _get_ec_hash_alg(curve: ec.EllipticCurve) -> hashes.HashAlgorithm: + if isinstance(curve, ec.SECP256R1): + return hashes.SHA256() + elif isinstance(curve, ec.SECP384R1): + return hashes.SHA384() + else: + assert isinstance(curve, ec.SECP521R1) + return hashes.SHA512() + + +def _load_ssh_public_identity( + data: utils.Buffer, + _legacy_dsa_allowed=False, +) -> SSHCertificate | SSHPublicKeyTypes: + utils._check_byteslike("data", data) + + m = _SSH_PUBKEY_RC.match(data) + if not m: + raise ValueError("Invalid line format") + key_type = orig_key_type = m.group(1) + key_body = m.group(2) + with_cert = False + if key_type.endswith(_CERT_SUFFIX): + with_cert = True + key_type = key_type[: -len(_CERT_SUFFIX)] + if key_type == _SSH_DSA and not _legacy_dsa_allowed: + raise UnsupportedAlgorithm( + "DSA keys aren't supported in SSH certificates" + ) + kformat = _lookup_kformat(key_type) + + try: + rest = memoryview(binascii.a2b_base64(key_body)) + except (TypeError, binascii.Error): + raise ValueError("Invalid format") + + if with_cert: + cert_body = rest + inner_key_type, rest = _get_sshstr(rest) + if inner_key_type != orig_key_type: + raise ValueError("Invalid key format") + if with_cert: + nonce, rest = _get_sshstr(rest) + public_key, rest = kformat.load_public(rest) + if with_cert: + serial, rest = _get_u64(rest) + cctype, rest = _get_u32(rest) + key_id, rest = _get_sshstr(rest) + principals, rest = _get_sshstr(rest) + valid_principals = [] + while principals: + principal, principals = _get_sshstr(principals) + valid_principals.append(bytes(principal)) + valid_after, rest = _get_u64(rest) + valid_before, rest = _get_u64(rest) + crit_options, rest = _get_sshstr(rest) + critical_options = _parse_exts_opts(crit_options) + exts, rest = _get_sshstr(rest) + extensions = _parse_exts_opts(exts) + # Get the reserved field, which is unused. + _, rest = _get_sshstr(rest) + sig_key_raw, rest = _get_sshstr(rest) + sig_type, sig_key = _get_sshstr(sig_key_raw) + if sig_type == _SSH_DSA and not _legacy_dsa_allowed: + raise UnsupportedAlgorithm( + "DSA signatures aren't supported in SSH certificates" + ) + # Get the entire cert body and subtract the signature + tbs_cert_body = cert_body[: -len(rest)] + signature_raw, rest = _get_sshstr(rest) + _check_empty(rest) + inner_sig_type, sig_rest = _get_sshstr(signature_raw) + # RSA certs can have multiple algorithm types + if ( + sig_type == _SSH_RSA + and inner_sig_type + not in [_SSH_RSA_SHA256, _SSH_RSA_SHA512, _SSH_RSA] + ) or (sig_type != _SSH_RSA and inner_sig_type != sig_type): + raise ValueError("Signature key type does not match") + signature, sig_rest = _get_sshstr(sig_rest) + _check_empty(sig_rest) + return SSHCertificate( + nonce, + public_key, + serial, + cctype, + key_id, + valid_principals, + valid_after, + valid_before, + critical_options, + extensions, + sig_type, + sig_key, + inner_sig_type, + signature, + tbs_cert_body, + orig_key_type, + cert_body, + ) + else: + _check_empty(rest) + return public_key + + +def load_ssh_public_identity( + data: utils.Buffer, +) -> SSHCertificate | SSHPublicKeyTypes: + return _load_ssh_public_identity(data) + + +def _parse_exts_opts(exts_opts: memoryview) -> dict[bytes, bytes]: + result: dict[bytes, bytes] = {} + last_name = None + while exts_opts: + name, exts_opts = _get_sshstr(exts_opts) + bname: bytes = bytes(name) + if bname in result: + raise ValueError("Duplicate name") + if last_name is not None and bname < last_name: + raise ValueError("Fields not lexically sorted") + value, exts_opts = _get_sshstr(exts_opts) + if len(value) > 0: + value, extra = _get_sshstr(value) + if len(extra) > 0: + raise ValueError("Unexpected extra data after value") + result[bname] = bytes(value) + last_name = bname + return result + + +def ssh_key_fingerprint( + key: SSHPublicKeyTypes, + hash_algorithm: hashes.MD5 | hashes.SHA1 | hashes.SHA256, +) -> bytes: + if not isinstance( + hash_algorithm, + (hashes.MD5, hashes.SHA1, hashes.SHA256), + ): + raise TypeError("hash_algorithm must be either MD5, SHA1, or SHA256") + + key_type = _get_ssh_key_type(key) + kformat = _lookup_kformat(key_type) + + f_pub = _FragList() + f_pub.put_sshstr(key_type) + kformat.encode_public(key, f_pub) + + ssh_binary_data = f_pub.tobytes() + + # Hash the binary data + hash_obj = hashes.Hash(hash_algorithm) + hash_obj.update(ssh_binary_data) + return hash_obj.finalize() + + +def load_ssh_public_key( + data: utils.Buffer, backend: typing.Any = None +) -> SSHPublicKeyTypes: + cert_or_key = _load_ssh_public_identity(data, _legacy_dsa_allowed=True) + public_key: SSHPublicKeyTypes + if isinstance(cert_or_key, SSHCertificate): + public_key = cert_or_key.public_key() + else: + public_key = cert_or_key + + if isinstance(public_key, dsa.DSAPublicKey): + warnings.warn( + "SSH DSA keys are deprecated and will be removed in a future " + "release.", + utils.DeprecatedIn40, + stacklevel=2, + ) + return public_key + + +def serialize_ssh_public_key(public_key: SSHPublicKeyTypes) -> bytes: + """One-line public key format for OpenSSH""" + if isinstance(public_key, dsa.DSAPublicKey): + warnings.warn( + "SSH DSA key support is deprecated and will be " + "removed in a future release", + utils.DeprecatedIn40, + stacklevel=4, + ) + key_type = _get_ssh_key_type(public_key) + kformat = _lookup_kformat(key_type) + + f_pub = _FragList() + f_pub.put_sshstr(key_type) + kformat.encode_public(public_key, f_pub) + + pub = binascii.b2a_base64(f_pub.tobytes()).strip() + return b"".join([key_type, b" ", pub]) + + +SSHCertPrivateKeyTypes = typing.Union[ + ec.EllipticCurvePrivateKey, + rsa.RSAPrivateKey, + ed25519.Ed25519PrivateKey, +] + + +# This is an undocumented limit enforced in the openssh codebase for sshd and +# ssh-keygen, but it is undefined in the ssh certificates spec. +_SSHKEY_CERT_MAX_PRINCIPALS = 256 + + +class SSHCertificateBuilder: + def __init__( + self, + _public_key: SSHCertPublicKeyTypes | None = None, + _serial: int | None = None, + _type: SSHCertificateType | None = None, + _key_id: bytes | None = None, + _valid_principals: list[bytes] = [], + _valid_for_all_principals: bool = False, + _valid_before: int | None = None, + _valid_after: int | None = None, + _critical_options: list[tuple[bytes, bytes]] = [], + _extensions: list[tuple[bytes, bytes]] = [], + ): + self._public_key = _public_key + self._serial = _serial + self._type = _type + self._key_id = _key_id + self._valid_principals = _valid_principals + self._valid_for_all_principals = _valid_for_all_principals + self._valid_before = _valid_before + self._valid_after = _valid_after + self._critical_options = _critical_options + self._extensions = _extensions + + def public_key( + self, public_key: SSHCertPublicKeyTypes + ) -> SSHCertificateBuilder: + if not isinstance( + public_key, + ( + ec.EllipticCurvePublicKey, + rsa.RSAPublicKey, + ed25519.Ed25519PublicKey, + ), + ): + raise TypeError("Unsupported key type") + if self._public_key is not None: + raise ValueError("public_key already set") + + return SSHCertificateBuilder( + _public_key=public_key, + _serial=self._serial, + _type=self._type, + _key_id=self._key_id, + _valid_principals=self._valid_principals, + _valid_for_all_principals=self._valid_for_all_principals, + _valid_before=self._valid_before, + _valid_after=self._valid_after, + _critical_options=self._critical_options, + _extensions=self._extensions, + ) + + def serial(self, serial: int) -> SSHCertificateBuilder: + if not isinstance(serial, int): + raise TypeError("serial must be an integer") + if not 0 <= serial < 2**64: + raise ValueError("serial must be between 0 and 2**64") + if self._serial is not None: + raise ValueError("serial already set") + + return SSHCertificateBuilder( + _public_key=self._public_key, + _serial=serial, + _type=self._type, + _key_id=self._key_id, + _valid_principals=self._valid_principals, + _valid_for_all_principals=self._valid_for_all_principals, + _valid_before=self._valid_before, + _valid_after=self._valid_after, + _critical_options=self._critical_options, + _extensions=self._extensions, + ) + + def type(self, type: SSHCertificateType) -> SSHCertificateBuilder: + if not isinstance(type, SSHCertificateType): + raise TypeError("type must be an SSHCertificateType") + if self._type is not None: + raise ValueError("type already set") + + return SSHCertificateBuilder( + _public_key=self._public_key, + _serial=self._serial, + _type=type, + _key_id=self._key_id, + _valid_principals=self._valid_principals, + _valid_for_all_principals=self._valid_for_all_principals, + _valid_before=self._valid_before, + _valid_after=self._valid_after, + _critical_options=self._critical_options, + _extensions=self._extensions, + ) + + def key_id(self, key_id: bytes) -> SSHCertificateBuilder: + if not isinstance(key_id, bytes): + raise TypeError("key_id must be bytes") + if self._key_id is not None: + raise ValueError("key_id already set") + + return SSHCertificateBuilder( + _public_key=self._public_key, + _serial=self._serial, + _type=self._type, + _key_id=key_id, + _valid_principals=self._valid_principals, + _valid_for_all_principals=self._valid_for_all_principals, + _valid_before=self._valid_before, + _valid_after=self._valid_after, + _critical_options=self._critical_options, + _extensions=self._extensions, + ) + + def valid_principals( + self, valid_principals: list[bytes] + ) -> SSHCertificateBuilder: + if self._valid_for_all_principals: + raise ValueError( + "Principals can't be set because the cert is valid " + "for all principals" + ) + if ( + not all(isinstance(x, bytes) for x in valid_principals) + or not valid_principals + ): + raise TypeError( + "principals must be a list of bytes and can't be empty" + ) + if self._valid_principals: + raise ValueError("valid_principals already set") + + if len(valid_principals) > _SSHKEY_CERT_MAX_PRINCIPALS: + raise ValueError( + "Reached or exceeded the maximum number of valid_principals" + ) + + return SSHCertificateBuilder( + _public_key=self._public_key, + _serial=self._serial, + _type=self._type, + _key_id=self._key_id, + _valid_principals=valid_principals, + _valid_for_all_principals=self._valid_for_all_principals, + _valid_before=self._valid_before, + _valid_after=self._valid_after, + _critical_options=self._critical_options, + _extensions=self._extensions, + ) + + def valid_for_all_principals(self): + if self._valid_principals: + raise ValueError( + "valid_principals already set, can't set " + "valid_for_all_principals" + ) + if self._valid_for_all_principals: + raise ValueError("valid_for_all_principals already set") + + return SSHCertificateBuilder( + _public_key=self._public_key, + _serial=self._serial, + _type=self._type, + _key_id=self._key_id, + _valid_principals=self._valid_principals, + _valid_for_all_principals=True, + _valid_before=self._valid_before, + _valid_after=self._valid_after, + _critical_options=self._critical_options, + _extensions=self._extensions, + ) + + def valid_before(self, valid_before: int | float) -> SSHCertificateBuilder: + if not isinstance(valid_before, (int, float)): + raise TypeError("valid_before must be an int or float") + valid_before = int(valid_before) + if valid_before < 0 or valid_before >= 2**64: + raise ValueError("valid_before must [0, 2**64)") + if self._valid_before is not None: + raise ValueError("valid_before already set") + + return SSHCertificateBuilder( + _public_key=self._public_key, + _serial=self._serial, + _type=self._type, + _key_id=self._key_id, + _valid_principals=self._valid_principals, + _valid_for_all_principals=self._valid_for_all_principals, + _valid_before=valid_before, + _valid_after=self._valid_after, + _critical_options=self._critical_options, + _extensions=self._extensions, + ) + + def valid_after(self, valid_after: int | float) -> SSHCertificateBuilder: + if not isinstance(valid_after, (int, float)): + raise TypeError("valid_after must be an int or float") + valid_after = int(valid_after) + if valid_after < 0 or valid_after >= 2**64: + raise ValueError("valid_after must [0, 2**64)") + if self._valid_after is not None: + raise ValueError("valid_after already set") + + return SSHCertificateBuilder( + _public_key=self._public_key, + _serial=self._serial, + _type=self._type, + _key_id=self._key_id, + _valid_principals=self._valid_principals, + _valid_for_all_principals=self._valid_for_all_principals, + _valid_before=self._valid_before, + _valid_after=valid_after, + _critical_options=self._critical_options, + _extensions=self._extensions, + ) + + def add_critical_option( + self, name: bytes, value: bytes + ) -> SSHCertificateBuilder: + if not isinstance(name, bytes) or not isinstance(value, bytes): + raise TypeError("name and value must be bytes") + # This is O(n**2) + if name in [name for name, _ in self._critical_options]: + raise ValueError("Duplicate critical option name") + + return SSHCertificateBuilder( + _public_key=self._public_key, + _serial=self._serial, + _type=self._type, + _key_id=self._key_id, + _valid_principals=self._valid_principals, + _valid_for_all_principals=self._valid_for_all_principals, + _valid_before=self._valid_before, + _valid_after=self._valid_after, + _critical_options=[*self._critical_options, (name, value)], + _extensions=self._extensions, + ) + + def add_extension( + self, name: bytes, value: bytes + ) -> SSHCertificateBuilder: + if not isinstance(name, bytes) or not isinstance(value, bytes): + raise TypeError("name and value must be bytes") + # This is O(n**2) + if name in [name for name, _ in self._extensions]: + raise ValueError("Duplicate extension name") + + return SSHCertificateBuilder( + _public_key=self._public_key, + _serial=self._serial, + _type=self._type, + _key_id=self._key_id, + _valid_principals=self._valid_principals, + _valid_for_all_principals=self._valid_for_all_principals, + _valid_before=self._valid_before, + _valid_after=self._valid_after, + _critical_options=self._critical_options, + _extensions=[*self._extensions, (name, value)], + ) + + def sign(self, private_key: SSHCertPrivateKeyTypes) -> SSHCertificate: + if not isinstance( + private_key, + ( + ec.EllipticCurvePrivateKey, + rsa.RSAPrivateKey, + ed25519.Ed25519PrivateKey, + ), + ): + raise TypeError("Unsupported private key type") + + if self._public_key is None: + raise ValueError("public_key must be set") + + # Not required + serial = 0 if self._serial is None else self._serial + + if self._type is None: + raise ValueError("type must be set") + + # Not required + key_id = b"" if self._key_id is None else self._key_id + + # A zero length list is valid, but means the certificate + # is valid for any principal of the specified type. We require + # the user to explicitly set valid_for_all_principals to get + # that behavior. + if not self._valid_principals and not self._valid_for_all_principals: + raise ValueError( + "valid_principals must be set if valid_for_all_principals " + "is False" + ) + + if self._valid_before is None: + raise ValueError("valid_before must be set") + + if self._valid_after is None: + raise ValueError("valid_after must be set") + + if self._valid_after > self._valid_before: + raise ValueError("valid_after must be earlier than valid_before") + + # lexically sort our byte strings + self._critical_options.sort(key=lambda x: x[0]) + self._extensions.sort(key=lambda x: x[0]) + + key_type = _get_ssh_key_type(self._public_key) + cert_prefix = key_type + _CERT_SUFFIX + + # Marshal the bytes to be signed + nonce = os.urandom(32) + kformat = _lookup_kformat(key_type) + f = _FragList() + f.put_sshstr(cert_prefix) + f.put_sshstr(nonce) + kformat.encode_public(self._public_key, f) + f.put_u64(serial) + f.put_u32(self._type.value) + f.put_sshstr(key_id) + fprincipals = _FragList() + for p in self._valid_principals: + fprincipals.put_sshstr(p) + f.put_sshstr(fprincipals.tobytes()) + f.put_u64(self._valid_after) + f.put_u64(self._valid_before) + fcrit = _FragList() + for name, value in self._critical_options: + fcrit.put_sshstr(name) + if len(value) > 0: + foptval = _FragList() + foptval.put_sshstr(value) + fcrit.put_sshstr(foptval.tobytes()) + else: + fcrit.put_sshstr(value) + f.put_sshstr(fcrit.tobytes()) + fext = _FragList() + for name, value in self._extensions: + fext.put_sshstr(name) + if len(value) > 0: + fextval = _FragList() + fextval.put_sshstr(value) + fext.put_sshstr(fextval.tobytes()) + else: + fext.put_sshstr(value) + f.put_sshstr(fext.tobytes()) + f.put_sshstr(b"") # RESERVED FIELD + # encode CA public key + ca_type = _get_ssh_key_type(private_key) + caformat = _lookup_kformat(ca_type) + caf = _FragList() + caf.put_sshstr(ca_type) + caformat.encode_public(private_key.public_key(), caf) + f.put_sshstr(caf.tobytes()) + # Sigs according to the rules defined for the CA's public key + # (RFC4253 section 6.6 for ssh-rsa, RFC5656 for ECDSA, + # and RFC8032 for Ed25519). + if isinstance(private_key, ed25519.Ed25519PrivateKey): + signature = private_key.sign(f.tobytes()) + fsig = _FragList() + fsig.put_sshstr(ca_type) + fsig.put_sshstr(signature) + f.put_sshstr(fsig.tobytes()) + elif isinstance(private_key, ec.EllipticCurvePrivateKey): + hash_alg = _get_ec_hash_alg(private_key.curve) + signature = private_key.sign(f.tobytes(), ec.ECDSA(hash_alg)) + r, s = asym_utils.decode_dss_signature(signature) + fsig = _FragList() + fsig.put_sshstr(ca_type) + fsigblob = _FragList() + fsigblob.put_mpint(r) + fsigblob.put_mpint(s) + fsig.put_sshstr(fsigblob.tobytes()) + f.put_sshstr(fsig.tobytes()) + + else: + assert isinstance(private_key, rsa.RSAPrivateKey) + # Just like Golang, we're going to use SHA512 for RSA + # https://cs.opensource.google/go/x/crypto/+/refs/tags/ + # v0.4.0:ssh/certs.go;l=445 + # RFC 8332 defines SHA256 and 512 as options + fsig = _FragList() + fsig.put_sshstr(_SSH_RSA_SHA512) + signature = private_key.sign( + f.tobytes(), padding.PKCS1v15(), hashes.SHA512() + ) + fsig.put_sshstr(signature) + f.put_sshstr(fsig.tobytes()) + + cert_data = binascii.b2a_base64(f.tobytes()).strip() + # load_ssh_public_identity returns a union, but this is + # guaranteed to be an SSHCertificate, so we cast to make + # mypy happy. + return typing.cast( + SSHCertificate, + load_ssh_public_identity(b"".join([cert_prefix, b" ", cert_data])), + ) diff --git a/py_modules/cryptography/hazmat/primitives/twofactor/__init__.py b/py_modules/cryptography/hazmat/primitives/twofactor/__init__.py new file mode 100644 index 0000000..c1af423 --- /dev/null +++ b/py_modules/cryptography/hazmat/primitives/twofactor/__init__.py @@ -0,0 +1,9 @@ +# This file is dual licensed under the terms of the Apache License, Version +# 2.0, and the BSD License. See the LICENSE file in the root of this repository +# for complete details. + +from __future__ import annotations + + +class InvalidToken(Exception): + pass diff --git a/py_modules/cryptography/hazmat/primitives/twofactor/hotp.py b/py_modules/cryptography/hazmat/primitives/twofactor/hotp.py new file mode 100644 index 0000000..21fb000 --- /dev/null +++ b/py_modules/cryptography/hazmat/primitives/twofactor/hotp.py @@ -0,0 +1,101 @@ +# This file is dual licensed under the terms of the Apache License, Version +# 2.0, and the BSD License. See the LICENSE file in the root of this repository +# for complete details. + +from __future__ import annotations + +import base64 +import typing +from urllib.parse import quote, urlencode + +from cryptography.hazmat.primitives import constant_time, hmac +from cryptography.hazmat.primitives.hashes import SHA1, SHA256, SHA512 +from cryptography.hazmat.primitives.twofactor import InvalidToken +from cryptography.utils import Buffer + +HOTPHashTypes = typing.Union[SHA1, SHA256, SHA512] + + +def _generate_uri( + hotp: HOTP, + type_name: str, + account_name: str, + issuer: str | None, + extra_parameters: list[tuple[str, int]], +) -> str: + parameters = [ + ("digits", hotp._length), + ("secret", base64.b32encode(hotp._key)), + ("algorithm", hotp._algorithm.name.upper()), + ] + + if issuer is not None: + parameters.append(("issuer", issuer)) + + parameters.extend(extra_parameters) + + label = ( + f"{quote(issuer)}:{quote(account_name)}" + if issuer + else quote(account_name) + ) + return f"otpauth://{type_name}/{label}?{urlencode(parameters)}" + + +class HOTP: + def __init__( + self, + key: Buffer, + length: int, + algorithm: HOTPHashTypes, + backend: typing.Any = None, + enforce_key_length: bool = True, + ) -> None: + if len(key) < 16 and enforce_key_length is True: + raise ValueError("Key length has to be at least 128 bits.") + + if not isinstance(length, int): + raise TypeError("Length parameter must be an integer type.") + + if length < 6 or length > 8: + raise ValueError("Length of HOTP has to be between 6 and 8.") + + if not isinstance(algorithm, (SHA1, SHA256, SHA512)): + raise TypeError("Algorithm must be SHA1, SHA256 or SHA512.") + + self._key = key + self._length = length + self._algorithm = algorithm + + def generate(self, counter: int) -> bytes: + if not isinstance(counter, int): + raise TypeError("Counter parameter must be an integer type.") + + truncated_value = self._dynamic_truncate(counter) + hotp = truncated_value % (10**self._length) + return "{0:0{1}}".format(hotp, self._length).encode() + + def verify(self, hotp: bytes, counter: int) -> None: + if not constant_time.bytes_eq(self.generate(counter), hotp): + raise InvalidToken("Supplied HOTP value does not match.") + + def _dynamic_truncate(self, counter: int) -> int: + ctx = hmac.HMAC(self._key, self._algorithm) + + try: + ctx.update(counter.to_bytes(length=8, byteorder="big")) + except OverflowError: + raise ValueError(f"Counter must be between 0 and {2**64 - 1}.") + + hmac_value = ctx.finalize() + + offset = hmac_value[len(hmac_value) - 1] & 0b1111 + p = hmac_value[offset : offset + 4] + return int.from_bytes(p, byteorder="big") & 0x7FFFFFFF + + def get_provisioning_uri( + self, account_name: str, counter: int, issuer: str | None + ) -> str: + return _generate_uri( + self, "hotp", account_name, issuer, [("counter", int(counter))] + ) diff --git a/py_modules/cryptography/hazmat/primitives/twofactor/totp.py b/py_modules/cryptography/hazmat/primitives/twofactor/totp.py new file mode 100644 index 0000000..10c725c --- /dev/null +++ b/py_modules/cryptography/hazmat/primitives/twofactor/totp.py @@ -0,0 +1,56 @@ +# This file is dual licensed under the terms of the Apache License, Version +# 2.0, and the BSD License. See the LICENSE file in the root of this repository +# for complete details. + +from __future__ import annotations + +import typing + +from cryptography.hazmat.primitives import constant_time +from cryptography.hazmat.primitives.twofactor import InvalidToken +from cryptography.hazmat.primitives.twofactor.hotp import ( + HOTP, + HOTPHashTypes, + _generate_uri, +) +from cryptography.utils import Buffer + + +class TOTP: + def __init__( + self, + key: Buffer, + length: int, + algorithm: HOTPHashTypes, + time_step: int, + backend: typing.Any = None, + enforce_key_length: bool = True, + ): + self._time_step = time_step + self._hotp = HOTP( + key, length, algorithm, enforce_key_length=enforce_key_length + ) + + def generate(self, time: int | float) -> bytes: + if not isinstance(time, (int, float)): + raise TypeError( + "Time parameter must be an integer type or float type." + ) + + counter = int(time / self._time_step) + return self._hotp.generate(counter) + + def verify(self, totp: bytes, time: int) -> None: + if not constant_time.bytes_eq(self.generate(time), totp): + raise InvalidToken("Supplied TOTP value does not match.") + + def get_provisioning_uri( + self, account_name: str, issuer: str | None + ) -> str: + return _generate_uri( + self._hotp, + "totp", + account_name, + issuer, + [("period", int(self._time_step))], + ) diff --git a/py_modules/cryptography/py.typed b/py_modules/cryptography/py.typed new file mode 100644 index 0000000..e69de29 diff --git a/py_modules/cryptography/utils.py b/py_modules/cryptography/utils.py new file mode 100644 index 0000000..90bd42d --- /dev/null +++ b/py_modules/cryptography/utils.py @@ -0,0 +1,136 @@ +# This file is dual licensed under the terms of the Apache License, Version +# 2.0, and the BSD License. See the LICENSE file in the root of this repository +# for complete details. + +from __future__ import annotations + +import enum +import sys +import types +import typing +import warnings +from collections.abc import Sequence + + +# We use a UserWarning subclass, instead of DeprecationWarning, because CPython +# decided deprecation warnings should be invisible by default. +class CryptographyDeprecationWarning(UserWarning): + pass + + +# Several APIs were deprecated with no specific end-of-life date because of the +# ubiquity of their use. They should not be removed until we agree on when that +# cycle ends. +DeprecatedIn36 = CryptographyDeprecationWarning +DeprecatedIn40 = CryptographyDeprecationWarning +DeprecatedIn41 = CryptographyDeprecationWarning +DeprecatedIn42 = CryptographyDeprecationWarning +DeprecatedIn43 = CryptographyDeprecationWarning +DeprecatedIn47 = CryptographyDeprecationWarning +DeprecatedIn50 = CryptographyDeprecationWarning + + +# If you're wondering why we don't use `Buffer`, it's because `Buffer` would +# be more accurately named: Bufferable. It means something which has an +# `__buffer__`. Which means you can't actually treat the result as a buffer +# (and do things like take a `len()`). +Buffer = typing.Union[bytes, bytearray, memoryview] + + +def _check_bytes(name: str, value: bytes) -> None: + if not isinstance(value, bytes): + raise TypeError(f"{name} must be bytes") + + +def _check_byteslike(name: str, value: Buffer) -> None: + if isinstance(value, (bytes, bytearray, memoryview)): + return + try: + memoryview(value) + except TypeError: + raise TypeError(f"{name} must be bytes-like") + + +def int_to_bytes(integer: int, length: int | None = None) -> bytes: + if length == 0: + raise ValueError("length argument can't be 0") + return integer.to_bytes( + length or (integer.bit_length() + 7) // 8 or 1, "big" + ) + + +class InterfaceNotImplemented(Exception): + pass + + +class _DeprecatedValue: + def __init__(self, value: object, message: str, warning_class): + self.value = value + self.message = message + self.warning_class = warning_class + + +class _ModuleWithDeprecations(types.ModuleType): + def __init__(self, module: types.ModuleType): + super().__init__(module.__name__) + self.__dict__["_module"] = module + + def __getattr__(self, name: str) -> typing.Any: + obj = getattr(self._module, name) + if isinstance(obj, _DeprecatedValue): + warnings.warn(obj.message, obj.warning_class, stacklevel=2) + obj = obj.value + else: + # Cache non-deprecated attributes in our own `__dict__` so that + # subsequent lookups are ordinary module attribute accesses and + # don't pay for this `__getattr__` (which would otherwise defeat + # CPython's LOAD_ATTR module caching for every attribute of the + # module). `__setattr__` and `__delattr__` keep the cache + # coherent. + self.__dict__[name] = obj + return obj + + def __setattr__(self, attr: str, value: object) -> None: + if isinstance(value, _DeprecatedValue): + self.__dict__.pop(attr, None) + else: + self.__dict__[attr] = value + setattr(self._module, attr, value) + + def __delattr__(self, attr: str) -> None: + obj = getattr(self._module, attr) + if isinstance(obj, _DeprecatedValue): + warnings.warn(obj.message, obj.warning_class, stacklevel=2) + + self.__dict__.pop(attr, None) + delattr(self._module, attr) + + def __dir__(self) -> Sequence[str]: + return ["_module", *dir(self._module)] + + +def deprecated( + value: object, + module_name: str, + message: str, + warning_class: type[Warning], + name: str | None = None, +) -> _DeprecatedValue: + module = sys.modules[module_name] + if not isinstance(module, _ModuleWithDeprecations): + sys.modules[module_name] = module = _ModuleWithDeprecations(module) + dv = _DeprecatedValue(value, message, warning_class) + # Maintain backwards compatibility with `name is None` for pyOpenSSL. + if name is not None: + setattr(module, name, dv) + return dv + + +# Python 3.10 changed representation of enums. We use well-defined object +# representation and string representation from Python 3.9. +class Enum(enum.Enum): + def __repr__(self) -> str: + return f"<{self.__class__.__name__}.{self._name_}: {self._value_!r}>" + + def __str__(self) -> str: + return f"{self.__class__.__name__}.{self._name_}" diff --git a/py_modules/cryptography/x509/__init__.py b/py_modules/cryptography/x509/__init__.py new file mode 100644 index 0000000..6442b00 --- /dev/null +++ b/py_modules/cryptography/x509/__init__.py @@ -0,0 +1,270 @@ +# This file is dual licensed under the terms of the Apache License, Version +# 2.0, and the BSD License. See the LICENSE file in the root of this repository +# for complete details. + +from __future__ import annotations + +from cryptography.x509 import certificate_transparency, oid, verification +from cryptography.x509.base import ( + Attribute, + AttributeNotFound, + Attributes, + Certificate, + CertificateBuilder, + CertificateRevocationList, + CertificateRevocationListBuilder, + CertificateSigningRequest, + CertificateSigningRequestBuilder, + InvalidVersion, + RevokedCertificate, + RevokedCertificateBuilder, + Version, + load_der_x509_certificate, + load_der_x509_crl, + load_der_x509_csr, + load_pem_x509_certificate, + load_pem_x509_certificates, + load_pem_x509_crl, + load_pem_x509_csr, + random_serial_number, +) +from cryptography.x509.extensions import ( + AccessDescription, + Admission, + Admissions, + AuthorityInformationAccess, + AuthorityKeyIdentifier, + BasicConstraints, + CertificateIssuer, + CertificatePolicies, + CRLDistributionPoints, + CRLNumber, + CRLReason, + DeltaCRLIndicator, + DistributionPoint, + DuplicateExtension, + ExtendedKeyUsage, + Extension, + ExtensionNotFound, + Extensions, + ExtensionType, + FreshestCRL, + GeneralNames, + InhibitAnyPolicy, + InvalidityDate, + IssuerAlternativeName, + IssuingDistributionPoint, + KeyUsage, + MSCertificateTemplate, + NameConstraints, + NamingAuthority, + NoticeReference, + OCSPAcceptableResponses, + OCSPNoCheck, + OCSPNonce, + PolicyConstraints, + PolicyInformation, + PrecertificateSignedCertificateTimestamps, + PrecertPoison, + PrivateKeyUsagePeriod, + ProfessionInfo, + ReasonFlags, + SignedCertificateTimestamps, + SubjectAlternativeName, + SubjectInformationAccess, + SubjectKeyIdentifier, + TLSFeature, + TLSFeatureType, + UnrecognizedExtension, + UserNotice, +) +from cryptography.x509.general_name import ( + DirectoryName, + DNSName, + GeneralName, + IPAddress, + OtherName, + RegisteredID, + RFC822Name, + UniformResourceIdentifier, + UnsupportedGeneralNameType, +) +from cryptography.x509.name import ( + Name, + NameAttribute, + RelativeDistinguishedName, +) +from cryptography.x509.oid import ( + AuthorityInformationAccessOID, + CertificatePoliciesOID, + CRLEntryExtensionOID, + ExtendedKeyUsageOID, + ExtensionOID, + NameOID, + ObjectIdentifier, + PublicKeyAlgorithmOID, + SignatureAlgorithmOID, +) + +OID_AUTHORITY_INFORMATION_ACCESS = ExtensionOID.AUTHORITY_INFORMATION_ACCESS +OID_AUTHORITY_KEY_IDENTIFIER = ExtensionOID.AUTHORITY_KEY_IDENTIFIER +OID_BASIC_CONSTRAINTS = ExtensionOID.BASIC_CONSTRAINTS +OID_CERTIFICATE_POLICIES = ExtensionOID.CERTIFICATE_POLICIES +OID_CRL_DISTRIBUTION_POINTS = ExtensionOID.CRL_DISTRIBUTION_POINTS +OID_EXTENDED_KEY_USAGE = ExtensionOID.EXTENDED_KEY_USAGE +OID_FRESHEST_CRL = ExtensionOID.FRESHEST_CRL +OID_INHIBIT_ANY_POLICY = ExtensionOID.INHIBIT_ANY_POLICY +OID_ISSUER_ALTERNATIVE_NAME = ExtensionOID.ISSUER_ALTERNATIVE_NAME +OID_KEY_USAGE = ExtensionOID.KEY_USAGE +OID_PRIVATE_KEY_USAGE_PERIOD = ExtensionOID.PRIVATE_KEY_USAGE_PERIOD +OID_NAME_CONSTRAINTS = ExtensionOID.NAME_CONSTRAINTS +OID_OCSP_NO_CHECK = ExtensionOID.OCSP_NO_CHECK +OID_POLICY_CONSTRAINTS = ExtensionOID.POLICY_CONSTRAINTS +OID_POLICY_MAPPINGS = ExtensionOID.POLICY_MAPPINGS +OID_SUBJECT_ALTERNATIVE_NAME = ExtensionOID.SUBJECT_ALTERNATIVE_NAME +OID_SUBJECT_DIRECTORY_ATTRIBUTES = ExtensionOID.SUBJECT_DIRECTORY_ATTRIBUTES +OID_SUBJECT_INFORMATION_ACCESS = ExtensionOID.SUBJECT_INFORMATION_ACCESS +OID_SUBJECT_KEY_IDENTIFIER = ExtensionOID.SUBJECT_KEY_IDENTIFIER + +OID_DSA_WITH_SHA1 = SignatureAlgorithmOID.DSA_WITH_SHA1 +OID_DSA_WITH_SHA224 = SignatureAlgorithmOID.DSA_WITH_SHA224 +OID_DSA_WITH_SHA256 = SignatureAlgorithmOID.DSA_WITH_SHA256 +OID_ECDSA_WITH_SHA1 = SignatureAlgorithmOID.ECDSA_WITH_SHA1 +OID_ECDSA_WITH_SHA224 = SignatureAlgorithmOID.ECDSA_WITH_SHA224 +OID_ECDSA_WITH_SHA256 = SignatureAlgorithmOID.ECDSA_WITH_SHA256 +OID_ECDSA_WITH_SHA384 = SignatureAlgorithmOID.ECDSA_WITH_SHA384 +OID_ECDSA_WITH_SHA512 = SignatureAlgorithmOID.ECDSA_WITH_SHA512 +OID_RSA_WITH_MD5 = SignatureAlgorithmOID.RSA_WITH_MD5 +OID_RSA_WITH_SHA1 = SignatureAlgorithmOID.RSA_WITH_SHA1 +OID_RSA_WITH_SHA224 = SignatureAlgorithmOID.RSA_WITH_SHA224 +OID_RSA_WITH_SHA256 = SignatureAlgorithmOID.RSA_WITH_SHA256 +OID_RSA_WITH_SHA384 = SignatureAlgorithmOID.RSA_WITH_SHA384 +OID_RSA_WITH_SHA512 = SignatureAlgorithmOID.RSA_WITH_SHA512 +OID_RSASSA_PSS = SignatureAlgorithmOID.RSASSA_PSS + +OID_COMMON_NAME = NameOID.COMMON_NAME +OID_COUNTRY_NAME = NameOID.COUNTRY_NAME +OID_DOMAIN_COMPONENT = NameOID.DOMAIN_COMPONENT +OID_DN_QUALIFIER = NameOID.DN_QUALIFIER +OID_EMAIL_ADDRESS = NameOID.EMAIL_ADDRESS +OID_GENERATION_QUALIFIER = NameOID.GENERATION_QUALIFIER +OID_GIVEN_NAME = NameOID.GIVEN_NAME +OID_LOCALITY_NAME = NameOID.LOCALITY_NAME +OID_ORGANIZATIONAL_UNIT_NAME = NameOID.ORGANIZATIONAL_UNIT_NAME +OID_ORGANIZATION_NAME = NameOID.ORGANIZATION_NAME +OID_PSEUDONYM = NameOID.PSEUDONYM +OID_SERIAL_NUMBER = NameOID.SERIAL_NUMBER +OID_STATE_OR_PROVINCE_NAME = NameOID.STATE_OR_PROVINCE_NAME +OID_SURNAME = NameOID.SURNAME +OID_TITLE = NameOID.TITLE + +OID_CLIENT_AUTH = ExtendedKeyUsageOID.CLIENT_AUTH +OID_CODE_SIGNING = ExtendedKeyUsageOID.CODE_SIGNING +OID_EMAIL_PROTECTION = ExtendedKeyUsageOID.EMAIL_PROTECTION +OID_OCSP_SIGNING = ExtendedKeyUsageOID.OCSP_SIGNING +OID_SERVER_AUTH = ExtendedKeyUsageOID.SERVER_AUTH +OID_TIME_STAMPING = ExtendedKeyUsageOID.TIME_STAMPING + +OID_ANY_POLICY = CertificatePoliciesOID.ANY_POLICY +OID_CPS_QUALIFIER = CertificatePoliciesOID.CPS_QUALIFIER +OID_CPS_USER_NOTICE = CertificatePoliciesOID.CPS_USER_NOTICE + +OID_CERTIFICATE_ISSUER = CRLEntryExtensionOID.CERTIFICATE_ISSUER +OID_CRL_REASON = CRLEntryExtensionOID.CRL_REASON +OID_INVALIDITY_DATE = CRLEntryExtensionOID.INVALIDITY_DATE + +OID_CA_ISSUERS = AuthorityInformationAccessOID.CA_ISSUERS +OID_OCSP = AuthorityInformationAccessOID.OCSP + +__all__ = [ + "OID_CA_ISSUERS", + "OID_OCSP", + "AccessDescription", + "Admission", + "Admissions", + "Attribute", + "AttributeNotFound", + "Attributes", + "AuthorityInformationAccess", + "AuthorityKeyIdentifier", + "BasicConstraints", + "CRLDistributionPoints", + "CRLNumber", + "CRLReason", + "Certificate", + "CertificateBuilder", + "CertificateIssuer", + "CertificatePolicies", + "CertificateRevocationList", + "CertificateRevocationListBuilder", + "CertificateSigningRequest", + "CertificateSigningRequestBuilder", + "DNSName", + "DeltaCRLIndicator", + "DirectoryName", + "DistributionPoint", + "DuplicateExtension", + "ExtendedKeyUsage", + "Extension", + "ExtensionNotFound", + "ExtensionType", + "Extensions", + "FreshestCRL", + "GeneralName", + "GeneralNames", + "IPAddress", + "InhibitAnyPolicy", + "InvalidVersion", + "InvalidityDate", + "IssuerAlternativeName", + "IssuingDistributionPoint", + "KeyUsage", + "MSCertificateTemplate", + "Name", + "NameAttribute", + "NameConstraints", + "NameOID", + "NamingAuthority", + "NoticeReference", + "OCSPAcceptableResponses", + "OCSPNoCheck", + "OCSPNonce", + "ObjectIdentifier", + "OtherName", + "PolicyConstraints", + "PolicyInformation", + "PrecertPoison", + "PrecertificateSignedCertificateTimestamps", + "PrivateKeyUsagePeriod", + "ProfessionInfo", + "PublicKeyAlgorithmOID", + "RFC822Name", + "ReasonFlags", + "RegisteredID", + "RelativeDistinguishedName", + "RevokedCertificate", + "RevokedCertificateBuilder", + "SignatureAlgorithmOID", + "SignedCertificateTimestamps", + "SubjectAlternativeName", + "SubjectInformationAccess", + "SubjectKeyIdentifier", + "TLSFeature", + "TLSFeatureType", + "UniformResourceIdentifier", + "UnrecognizedExtension", + "UnsupportedGeneralNameType", + "UserNotice", + "Version", + "certificate_transparency", + "load_der_x509_certificate", + "load_der_x509_crl", + "load_der_x509_csr", + "load_pem_x509_certificate", + "load_pem_x509_certificates", + "load_pem_x509_crl", + "load_pem_x509_csr", + "oid", + "random_serial_number", + "verification", +] diff --git a/py_modules/cryptography/x509/base.py b/py_modules/cryptography/x509/base.py new file mode 100644 index 0000000..3ded5dc --- /dev/null +++ b/py_modules/cryptography/x509/base.py @@ -0,0 +1,834 @@ +# This file is dual licensed under the terms of the Apache License, Version +# 2.0, and the BSD License. See the LICENSE file in the root of this repository +# for complete details. + +from __future__ import annotations + +import datetime +import os +import typing +from collections.abc import Iterable + +from cryptography import utils +from cryptography.hazmat.bindings._rust import x509 as rust_x509 +from cryptography.hazmat.primitives import hashes +from cryptography.hazmat.primitives.asymmetric import ( + dsa, + ec, + ed448, + ed25519, + mldsa, + mlkem, + padding, + rsa, + x448, + x25519, +) +from cryptography.hazmat.primitives.asymmetric.types import ( + CertificateIssuerPrivateKeyTypes, + CertificatePublicKeyTypes, +) +from cryptography.x509.extensions import ( + Extension, + ExtensionType, + _make_sequence_methods, +) +from cryptography.x509.name import Name, _ASN1Type +from cryptography.x509.oid import ObjectIdentifier + +_EARLIEST_UTC_TIME = datetime.datetime(1950, 1, 1) + +# This must be kept in sync with sign.rs's list of allowable types in +# identify_hash_type +_AllowedHashTypes = typing.Union[ + hashes.SHA224, + hashes.SHA256, + hashes.SHA384, + hashes.SHA512, + hashes.SHA3_224, + hashes.SHA3_256, + hashes.SHA3_384, + hashes.SHA3_512, +] + + +class AttributeNotFound(Exception): + def __init__(self, msg: str, oid: ObjectIdentifier) -> None: + super().__init__(msg) + self.oid = oid + + +def _reject_duplicate_extension( + extension: Extension[ExtensionType], + extensions: list[Extension[ExtensionType]], +) -> None: + # This is quadratic in the number of extensions + for e in extensions: + if e.oid == extension.oid: + raise ValueError("This extension has already been set.") + + +def _reject_duplicate_attribute( + oid: ObjectIdentifier, + attributes: list[tuple[ObjectIdentifier, bytes, int | None]], +) -> None: + # This is quadratic in the number of attributes + for attr_oid, _, _ in attributes: + if attr_oid == oid: + raise ValueError("This attribute has already been set.") + + +def _convert_to_naive_utc_time(time: datetime.datetime) -> datetime.datetime: + """Normalizes a datetime to a naive datetime in UTC. + + time -- datetime to normalize. Assumed to be in UTC if not timezone + aware. + """ + if time.tzinfo is not None: + offset = time.utcoffset() + offset = offset if offset else datetime.timedelta() + return time.replace(tzinfo=None) - offset + else: + return time + + +class Attribute: + def __init__( + self, + oid: ObjectIdentifier, + value: bytes, + _type: int = _ASN1Type.UTF8String.value, + ) -> None: + self._oid = oid + self._value = value + self._type = _type + + @property + def oid(self) -> ObjectIdentifier: + return self._oid + + @property + def value(self) -> bytes: + return self._value + + def __repr__(self) -> str: + return f"" + + def __eq__(self, other: object) -> bool: + if not isinstance(other, Attribute): + return NotImplemented + + return ( + self.oid == other.oid + and self.value == other.value + and self._type == other._type + ) + + def __hash__(self) -> int: + return hash((self.oid, self.value, self._type)) + + +class Attributes: + def __init__( + self, + attributes: Iterable[Attribute], + ) -> None: + self._attributes = list(attributes) + + __len__, __iter__, __getitem__ = _make_sequence_methods("_attributes") + + def __repr__(self) -> str: + return f"" + + def get_attribute_for_oid(self, oid: ObjectIdentifier) -> Attribute: + for attr in self: + if attr.oid == oid: + return attr + + raise AttributeNotFound(f"No {oid} attribute was found", oid) + + +class Version(utils.Enum): + v1 = 0 + v3 = 2 + + +class InvalidVersion(Exception): + def __init__(self, msg: str, parsed_version: int) -> None: + super().__init__(msg) + self.parsed_version = parsed_version + + +Certificate = rust_x509.Certificate +RevokedCertificate = rust_x509.RevokedCertificate + + +CertificateRevocationList = rust_x509.CertificateRevocationList +CertificateSigningRequest = rust_x509.CertificateSigningRequest + + +load_pem_x509_certificate = rust_x509.load_pem_x509_certificate +load_der_x509_certificate = rust_x509.load_der_x509_certificate + +load_pem_x509_certificates = rust_x509.load_pem_x509_certificates + +load_pem_x509_csr = rust_x509.load_pem_x509_csr +load_der_x509_csr = rust_x509.load_der_x509_csr + +load_pem_x509_crl = rust_x509.load_pem_x509_crl +load_der_x509_crl = rust_x509.load_der_x509_crl + + +class CertificateSigningRequestBuilder: + def __init__( + self, + subject_name: Name | None = None, + extensions: list[Extension[ExtensionType]] = [], + attributes: list[tuple[ObjectIdentifier, bytes, int | None]] = [], + ): + """ + Creates an empty X.509 certificate request (v1). + """ + self._subject_name = subject_name + self._extensions = extensions + self._attributes = attributes + + def subject_name(self, name: Name) -> CertificateSigningRequestBuilder: + """ + Sets the certificate requestor's distinguished name. + """ + if not isinstance(name, Name): + raise TypeError("Expecting x509.Name object.") + if self._subject_name is not None: + raise ValueError("The subject name may only be set once.") + return CertificateSigningRequestBuilder( + name, self._extensions, self._attributes + ) + + def add_extension( + self, extval: ExtensionType, critical: bool + ) -> CertificateSigningRequestBuilder: + """ + Adds an X.509 extension to the certificate request. + """ + if not isinstance(extval, ExtensionType): + raise TypeError("extension must be an ExtensionType") + + extension = Extension(extval.oid, critical, extval) + _reject_duplicate_extension(extension, self._extensions) + + return CertificateSigningRequestBuilder( + self._subject_name, + [*self._extensions, extension], + self._attributes, + ) + + def add_attribute( + self, + oid: ObjectIdentifier, + value: bytes, + *, + _tag: _ASN1Type | None = None, + ) -> CertificateSigningRequestBuilder: + """ + Adds an X.509 attribute with an OID and associated value. + """ + if not isinstance(oid, ObjectIdentifier): + raise TypeError("oid must be an ObjectIdentifier") + + if not isinstance(value, bytes): + raise TypeError("value must be bytes") + + if _tag is not None and not isinstance(_tag, _ASN1Type): + raise TypeError("tag must be _ASN1Type") + + _reject_duplicate_attribute(oid, self._attributes) + + if _tag is not None: + tag = _tag.value + else: + tag = None + + return CertificateSigningRequestBuilder( + self._subject_name, + self._extensions, + [*self._attributes, (oid, value, tag)], + ) + + def sign( + self, + private_key: CertificateIssuerPrivateKeyTypes, + algorithm: _AllowedHashTypes | None, + backend: typing.Any = None, + *, + rsa_padding: padding.PSS | padding.PKCS1v15 | None = None, + ecdsa_deterministic: bool | None = None, + ) -> CertificateSigningRequest: + """ + Signs the request using the requestor's private key. + """ + if self._subject_name is None: + raise ValueError("A CertificateSigningRequest must have a subject") + + if rsa_padding is not None: + if not isinstance(rsa_padding, (padding.PSS, padding.PKCS1v15)): + raise TypeError("Padding must be PSS or PKCS1v15") + if not isinstance(private_key, rsa.RSAPrivateKey): + raise TypeError("Padding is only supported for RSA keys") + + if ecdsa_deterministic is not None: + if not isinstance(private_key, ec.EllipticCurvePrivateKey): + raise TypeError( + "Deterministic ECDSA is only supported for EC keys" + ) + + return rust_x509.create_x509_csr( + self, + private_key, + algorithm, + rsa_padding, + ecdsa_deterministic, + ) + + +class CertificateBuilder: + _extensions: list[Extension[ExtensionType]] + + def __init__( + self, + issuer_name: Name | None = None, + subject_name: Name | None = None, + public_key: CertificatePublicKeyTypes | None = None, + serial_number: int | None = None, + not_valid_before: datetime.datetime | None = None, + not_valid_after: datetime.datetime | None = None, + extensions: list[Extension[ExtensionType]] = [], + public_key_rsa_padding: type[padding.PSS] | None = None, + ) -> None: + self._version = Version.v3 + self._issuer_name = issuer_name + self._subject_name = subject_name + self._public_key = public_key + self._serial_number = serial_number + self._not_valid_before = not_valid_before + self._not_valid_after = not_valid_after + self._extensions = extensions + self._public_key_rsa_padding = public_key_rsa_padding + + def issuer_name(self, name: Name) -> CertificateBuilder: + """ + Sets the CA's distinguished name. + """ + if not isinstance(name, Name): + raise TypeError("Expecting x509.Name object.") + if self._issuer_name is not None: + raise ValueError("The issuer name may only be set once.") + return CertificateBuilder( + name, + self._subject_name, + self._public_key, + self._serial_number, + self._not_valid_before, + self._not_valid_after, + self._extensions, + self._public_key_rsa_padding, + ) + + def subject_name(self, name: Name) -> CertificateBuilder: + """ + Sets the requestor's distinguished name. + """ + if not isinstance(name, Name): + raise TypeError("Expecting x509.Name object.") + if self._subject_name is not None: + raise ValueError("The subject name may only be set once.") + return CertificateBuilder( + self._issuer_name, + name, + self._public_key, + self._serial_number, + self._not_valid_before, + self._not_valid_after, + self._extensions, + self._public_key_rsa_padding, + ) + + def public_key( + self, + key: CertificatePublicKeyTypes, + *, + rsa_padding: type[padding.PSS] | None = None, + ) -> CertificateBuilder: + """ + Sets the requestor's public key (as found in the signing request). + """ + if not isinstance( + key, + ( + dsa.DSAPublicKey, + rsa.RSAPublicKey, + ec.EllipticCurvePublicKey, + ed25519.Ed25519PublicKey, + ed448.Ed448PublicKey, + mldsa.MLDSA44PublicKey, + mldsa.MLDSA65PublicKey, + mldsa.MLDSA87PublicKey, + mlkem.MLKEM768PublicKey, + mlkem.MLKEM1024PublicKey, + x25519.X25519PublicKey, + x448.X448PublicKey, + ), + ): + raise TypeError( + "Expecting one of DSAPublicKey, RSAPublicKey," + " EllipticCurvePublicKey, Ed25519PublicKey," + " Ed448PublicKey, MLDSA44PublicKey, MLDSA65PublicKey," + " MLDSA87PublicKey, MLKEM768PublicKey, MLKEM1024PublicKey," + " X25519PublicKey or X448PublicKey." + ) + if rsa_padding is not None: + if rsa_padding is not padding.PSS: + raise TypeError( + "rsa_padding must be the PSS class, not an instance" + ) + if not isinstance(key, rsa.RSAPublicKey): + raise TypeError( + "rsa_padding is only supported with RSA public keys" + ) + if self._public_key is not None: + raise ValueError("The public key may only be set once.") + return CertificateBuilder( + self._issuer_name, + self._subject_name, + key, + self._serial_number, + self._not_valid_before, + self._not_valid_after, + self._extensions, + rsa_padding, + ) + + def serial_number(self, number: int) -> CertificateBuilder: + """ + Sets the certificate serial number. + """ + if not isinstance(number, int): + raise TypeError("Serial number must be of integral type.") + if self._serial_number is not None: + raise ValueError("The serial number may only be set once.") + if number <= 0: + raise ValueError("The serial number should be positive.") + + # ASN.1 integers are always signed, so most significant bit must be + # zero. + if number.bit_length() >= 160: # As defined in RFC 5280 + raise ValueError( + "The serial number should not be more than 159 bits." + ) + return CertificateBuilder( + self._issuer_name, + self._subject_name, + self._public_key, + number, + self._not_valid_before, + self._not_valid_after, + self._extensions, + self._public_key_rsa_padding, + ) + + def not_valid_before(self, time: datetime.datetime) -> CertificateBuilder: + """ + Sets the certificate activation time. + """ + if not isinstance(time, datetime.datetime): + raise TypeError("Expecting datetime object.") + if self._not_valid_before is not None: + raise ValueError("The not valid before may only be set once.") + time = _convert_to_naive_utc_time(time) + if time < _EARLIEST_UTC_TIME: + raise ValueError( + "The not valid before date must be on or after" + " 1950 January 1)." + ) + if self._not_valid_after is not None and time > self._not_valid_after: + raise ValueError( + "The not valid before date must be before the not valid after " + "date." + ) + return CertificateBuilder( + self._issuer_name, + self._subject_name, + self._public_key, + self._serial_number, + time, + self._not_valid_after, + self._extensions, + self._public_key_rsa_padding, + ) + + def not_valid_after(self, time: datetime.datetime) -> CertificateBuilder: + """ + Sets the certificate expiration time. + """ + if not isinstance(time, datetime.datetime): + raise TypeError("Expecting datetime object.") + if self._not_valid_after is not None: + raise ValueError("The not valid after may only be set once.") + time = _convert_to_naive_utc_time(time) + if time < _EARLIEST_UTC_TIME: + raise ValueError( + "The not valid after date must be on or after 1950 January 1." + ) + if ( + self._not_valid_before is not None + and time < self._not_valid_before + ): + raise ValueError( + "The not valid after date must be after the not valid before " + "date." + ) + return CertificateBuilder( + self._issuer_name, + self._subject_name, + self._public_key, + self._serial_number, + self._not_valid_before, + time, + self._extensions, + self._public_key_rsa_padding, + ) + + def add_extension( + self, extval: ExtensionType, critical: bool + ) -> CertificateBuilder: + """ + Adds an X.509 extension to the certificate. + """ + if not isinstance(extval, ExtensionType): + raise TypeError("extension must be an ExtensionType") + + extension = Extension(extval.oid, critical, extval) + _reject_duplicate_extension(extension, self._extensions) + + return CertificateBuilder( + self._issuer_name, + self._subject_name, + self._public_key, + self._serial_number, + self._not_valid_before, + self._not_valid_after, + [*self._extensions, extension], + self._public_key_rsa_padding, + ) + + def sign( + self, + private_key: CertificateIssuerPrivateKeyTypes, + algorithm: _AllowedHashTypes | None, + backend: typing.Any = None, + *, + rsa_padding: padding.PSS | padding.PKCS1v15 | None = None, + ecdsa_deterministic: bool | None = None, + ) -> Certificate: + """ + Signs the certificate using the CA's private key. + """ + if self._subject_name is None: + raise ValueError("A certificate must have a subject name") + + if self._issuer_name is None: + raise ValueError("A certificate must have an issuer name") + + if self._serial_number is None: + raise ValueError("A certificate must have a serial number") + + if self._not_valid_before is None: + raise ValueError("A certificate must have a not valid before time") + + if self._not_valid_after is None: + raise ValueError("A certificate must have a not valid after time") + + if self._public_key is None: + raise ValueError("A certificate must have a public key") + + if private_key is None: + raise TypeError("Need private key to sign certificate") + + if rsa_padding is not None: + if not isinstance(rsa_padding, (padding.PSS, padding.PKCS1v15)): + raise TypeError("Padding must be PSS or PKCS1v15") + if not isinstance(private_key, rsa.RSAPrivateKey): + raise TypeError("Padding is only supported for RSA keys") + + if ecdsa_deterministic is not None: + if not isinstance(private_key, ec.EllipticCurvePrivateKey): + raise TypeError( + "Deterministic ECDSA is only supported for EC keys" + ) + + return rust_x509.create_x509_certificate( + self, + private_key, + algorithm, + rsa_padding, + ecdsa_deterministic, + ) + + def create_unsigned(self) -> Certificate: + """ + Creates an unsigned certificate, per RFC 9925. + """ + if self._subject_name is None: + raise ValueError("A certificate must have a subject name") + + if self._issuer_name is None: + raise ValueError("A certificate must have an issuer name") + + if self._serial_number is None: + raise ValueError("A certificate must have a serial number") + + if self._not_valid_before is None: + raise ValueError("A certificate must have a not valid before time") + + if self._not_valid_after is None: + raise ValueError("A certificate must have a not valid after time") + + if self._public_key is None: + raise ValueError("A certificate must have a public key") + + return rust_x509.create_x509_certificate( + self, + private_key=None, + hash_algorithm=None, + rsa_padding=None, + ecdsa_deterministic=None, + ) + + +class CertificateRevocationListBuilder: + _extensions: list[Extension[ExtensionType]] + _revoked_certificates: list[RevokedCertificate] + + def __init__( + self, + issuer_name: Name | None = None, + last_update: datetime.datetime | None = None, + next_update: datetime.datetime | None = None, + extensions: list[Extension[ExtensionType]] = [], + revoked_certificates: list[RevokedCertificate] = [], + ): + self._issuer_name = issuer_name + self._last_update = last_update + self._next_update = next_update + self._extensions = extensions + self._revoked_certificates = revoked_certificates + + def issuer_name( + self, issuer_name: Name + ) -> CertificateRevocationListBuilder: + if not isinstance(issuer_name, Name): + raise TypeError("Expecting x509.Name object.") + if self._issuer_name is not None: + raise ValueError("The issuer name may only be set once.") + return CertificateRevocationListBuilder( + issuer_name, + self._last_update, + self._next_update, + self._extensions, + self._revoked_certificates, + ) + + def last_update( + self, last_update: datetime.datetime + ) -> CertificateRevocationListBuilder: + if not isinstance(last_update, datetime.datetime): + raise TypeError("Expecting datetime object.") + if self._last_update is not None: + raise ValueError("Last update may only be set once.") + last_update = _convert_to_naive_utc_time(last_update) + if last_update < _EARLIEST_UTC_TIME: + raise ValueError( + "The last update date must be on or after 1950 January 1." + ) + if self._next_update is not None and last_update > self._next_update: + raise ValueError( + "The last update date must be before the next update date." + ) + return CertificateRevocationListBuilder( + self._issuer_name, + last_update, + self._next_update, + self._extensions, + self._revoked_certificates, + ) + + def next_update( + self, next_update: datetime.datetime + ) -> CertificateRevocationListBuilder: + if not isinstance(next_update, datetime.datetime): + raise TypeError("Expecting datetime object.") + if self._next_update is not None: + raise ValueError("Last update may only be set once.") + next_update = _convert_to_naive_utc_time(next_update) + if next_update < _EARLIEST_UTC_TIME: + raise ValueError( + "The last update date must be on or after 1950 January 1." + ) + if self._last_update is not None and next_update < self._last_update: + raise ValueError( + "The next update date must be after the last update date." + ) + return CertificateRevocationListBuilder( + self._issuer_name, + self._last_update, + next_update, + self._extensions, + self._revoked_certificates, + ) + + def add_extension( + self, extval: ExtensionType, critical: bool + ) -> CertificateRevocationListBuilder: + """ + Adds an X.509 extension to the certificate revocation list. + """ + if not isinstance(extval, ExtensionType): + raise TypeError("extension must be an ExtensionType") + + extension = Extension(extval.oid, critical, extval) + _reject_duplicate_extension(extension, self._extensions) + return CertificateRevocationListBuilder( + self._issuer_name, + self._last_update, + self._next_update, + [*self._extensions, extension], + self._revoked_certificates, + ) + + def add_revoked_certificate( + self, revoked_certificate: RevokedCertificate + ) -> CertificateRevocationListBuilder: + """ + Adds a revoked certificate to the CRL. + """ + if not isinstance(revoked_certificate, RevokedCertificate): + raise TypeError("Must be an instance of RevokedCertificate") + + return CertificateRevocationListBuilder( + self._issuer_name, + self._last_update, + self._next_update, + self._extensions, + [*self._revoked_certificates, revoked_certificate], + ) + + def sign( + self, + private_key: CertificateIssuerPrivateKeyTypes, + algorithm: _AllowedHashTypes | None, + backend: typing.Any = None, + *, + rsa_padding: padding.PSS | padding.PKCS1v15 | None = None, + ecdsa_deterministic: bool | None = None, + ) -> CertificateRevocationList: + if self._issuer_name is None: + raise ValueError("A CRL must have an issuer name") + + if self._last_update is None: + raise ValueError("A CRL must have a last update time") + + if self._next_update is None: + raise ValueError("A CRL must have a next update time") + + if rsa_padding is not None: + if not isinstance(rsa_padding, (padding.PSS, padding.PKCS1v15)): + raise TypeError("Padding must be PSS or PKCS1v15") + if not isinstance(private_key, rsa.RSAPrivateKey): + raise TypeError("Padding is only supported for RSA keys") + + if ecdsa_deterministic is not None: + if not isinstance(private_key, ec.EllipticCurvePrivateKey): + raise TypeError( + "Deterministic ECDSA is only supported for EC keys" + ) + + return rust_x509.create_x509_crl( + self, + private_key, + algorithm, + rsa_padding, + ecdsa_deterministic, + ) + + +class RevokedCertificateBuilder: + def __init__( + self, + serial_number: int | None = None, + revocation_date: datetime.datetime | None = None, + extensions: list[Extension[ExtensionType]] = [], + ): + self._serial_number = serial_number + self._revocation_date = revocation_date + self._extensions = extensions + + def serial_number(self, number: int) -> RevokedCertificateBuilder: + if not isinstance(number, int): + raise TypeError("Serial number must be of integral type.") + if self._serial_number is not None: + raise ValueError("The serial number may only be set once.") + if number <= 0: + raise ValueError("The serial number should be positive") + + # ASN.1 integers are always signed, so most significant bit must be + # zero. + if number.bit_length() >= 160: # As defined in RFC 5280 + raise ValueError( + "The serial number should not be more than 159 bits." + ) + return RevokedCertificateBuilder( + number, self._revocation_date, self._extensions + ) + + def revocation_date( + self, time: datetime.datetime + ) -> RevokedCertificateBuilder: + if not isinstance(time, datetime.datetime): + raise TypeError("Expecting datetime object.") + if self._revocation_date is not None: + raise ValueError("The revocation date may only be set once.") + time = _convert_to_naive_utc_time(time) + if time < _EARLIEST_UTC_TIME: + raise ValueError( + "The revocation date must be on or after 1950 January 1." + ) + return RevokedCertificateBuilder( + self._serial_number, time, self._extensions + ) + + def add_extension( + self, extval: ExtensionType, critical: bool + ) -> RevokedCertificateBuilder: + if not isinstance(extval, ExtensionType): + raise TypeError("extension must be an ExtensionType") + + extension = Extension(extval.oid, critical, extval) + _reject_duplicate_extension(extension, self._extensions) + return RevokedCertificateBuilder( + self._serial_number, + self._revocation_date, + [*self._extensions, extension], + ) + + def build(self, backend: typing.Any = None) -> RevokedCertificate: + if self._serial_number is None: + raise ValueError("A revoked certificate must have a serial number") + if self._revocation_date is None: + raise ValueError( + "A revoked certificate must have a revocation date" + ) + return rust_x509.create_revoked_certificate(self) + + +def random_serial_number() -> int: + return int.from_bytes(os.urandom(20), "big") >> 1 diff --git a/py_modules/cryptography/x509/certificate_transparency.py b/py_modules/cryptography/x509/certificate_transparency.py new file mode 100644 index 0000000..fb66cc6 --- /dev/null +++ b/py_modules/cryptography/x509/certificate_transparency.py @@ -0,0 +1,35 @@ +# This file is dual licensed under the terms of the Apache License, Version +# 2.0, and the BSD License. See the LICENSE file in the root of this repository +# for complete details. + +from __future__ import annotations + +from cryptography import utils +from cryptography.hazmat.bindings._rust import x509 as rust_x509 + + +class LogEntryType(utils.Enum): + X509_CERTIFICATE = 0 + PRE_CERTIFICATE = 1 + + +class Version(utils.Enum): + v1 = 0 + + +class SignatureAlgorithm(utils.Enum): + """ + Signature algorithms that are valid for SCTs. + + These are exactly the same as SignatureAlgorithm in RFC 5246 (TLS 1.2). + + See: + """ + + ANONYMOUS = 0 + RSA = 1 + DSA = 2 + ECDSA = 3 + + +SignedCertificateTimestamp = rust_x509.Sct diff --git a/py_modules/cryptography/x509/extensions.py b/py_modules/cryptography/x509/extensions.py new file mode 100644 index 0000000..dc7728e --- /dev/null +++ b/py_modules/cryptography/x509/extensions.py @@ -0,0 +1,2533 @@ +# This file is dual licensed under the terms of the Apache License, Version +# 2.0, and the BSD License. See the LICENSE file in the root of this repository +# for complete details. + +from __future__ import annotations + +import abc +import datetime +import hashlib +import ipaddress +import typing +from collections.abc import Iterable, Iterator + +from cryptography import utils +from cryptography.hazmat.bindings._rust import asn1 +from cryptography.hazmat.bindings._rust import x509 as rust_x509 +from cryptography.hazmat.primitives import _serialization, constant_time +from cryptography.hazmat.primitives.asymmetric.ec import EllipticCurvePublicKey +from cryptography.hazmat.primitives.asymmetric.rsa import RSAPublicKey +from cryptography.hazmat.primitives.asymmetric.types import ( + CertificateIssuerPublicKeyTypes, + CertificatePublicKeyTypes, +) +from cryptography.x509.certificate_transparency import ( + SignedCertificateTimestamp, +) +from cryptography.x509.general_name import ( + DirectoryName, + DNSName, + GeneralName, + IPAddress, + OtherName, + RegisteredID, + RFC822Name, + UniformResourceIdentifier, + _IPAddressTypes, +) +from cryptography.x509.name import Name, RelativeDistinguishedName +from cryptography.x509.oid import ( + CRLEntryExtensionOID, + ExtensionOID, + ObjectIdentifier, + OCSPExtensionOID, +) + +ExtensionTypeVar = typing.TypeVar( + "ExtensionTypeVar", bound="ExtensionType", covariant=True +) + + +def _key_identifier_from_public_key( + public_key: CertificatePublicKeyTypes, +) -> bytes: + if isinstance(public_key, RSAPublicKey): + data = public_key.public_bytes( + _serialization.Encoding.DER, + _serialization.PublicFormat.PKCS1, + ) + elif isinstance(public_key, EllipticCurvePublicKey): + data = public_key.public_bytes( + _serialization.Encoding.X962, + _serialization.PublicFormat.UncompressedPoint, + ) + else: + # This is a very slow way to do this. + serialized = public_key.public_bytes( + _serialization.Encoding.DER, + _serialization.PublicFormat.SubjectPublicKeyInfo, + ) + data = asn1.parse_spki_for_data(serialized) + + return hashlib.sha1(data).digest() + + +def _make_sequence_methods(field_name: str): + def len_method(self) -> int: + return len(getattr(self, field_name)) + + def iter_method(self): + return iter(getattr(self, field_name)) + + def getitem_method(self, idx): + return getattr(self, field_name)[idx] + + return len_method, iter_method, getitem_method + + +class DuplicateExtension(Exception): + def __init__(self, msg: str, oid: ObjectIdentifier) -> None: + super().__init__(msg) + self.oid = oid + + +class ExtensionNotFound(Exception): + def __init__(self, msg: str, oid: ObjectIdentifier) -> None: + super().__init__(msg) + self.oid = oid + + +class ExtensionType(metaclass=abc.ABCMeta): + oid: typing.ClassVar[ObjectIdentifier] + + def public_bytes(self) -> bytes: + """ + Serializes the extension type to DER. + """ + raise NotImplementedError( + f"public_bytes is not implemented for extension type {self!r}" + ) + + +class Extensions: + def __init__(self, extensions: Iterable[Extension[ExtensionType]]) -> None: + self._extensions = list(extensions) + + def get_extension_for_oid( + self, oid: ObjectIdentifier + ) -> Extension[ExtensionType]: + for ext in self: + if ext.oid == oid: + return ext + + raise ExtensionNotFound(f"No {oid} extension was found", oid) + + def get_extension_for_class( + self, extclass: type[ExtensionTypeVar] + ) -> Extension[ExtensionTypeVar]: + if extclass is UnrecognizedExtension: + raise TypeError( + "UnrecognizedExtension can't be used with " + "get_extension_for_class because more than one instance of the" + " class may be present." + ) + + for ext in self: + if isinstance(ext.value, extclass): + return ext + + raise ExtensionNotFound( + f"No {extclass} extension was found", extclass.oid + ) + + __len__, __iter__, __getitem__ = _make_sequence_methods("_extensions") + + def __repr__(self) -> str: + return f"" + + +class CRLNumber(ExtensionType): + oid = ExtensionOID.CRL_NUMBER + + def __init__(self, crl_number: int) -> None: + if not isinstance(crl_number, int): + raise TypeError("crl_number must be an integer") + + self._crl_number = crl_number + + def __eq__(self, other: object) -> bool: + if not isinstance(other, CRLNumber): + return NotImplemented + + return self.crl_number == other.crl_number + + def __hash__(self) -> int: + return hash(self.crl_number) + + def __repr__(self) -> str: + return f"" + + @property + def crl_number(self) -> int: + return self._crl_number + + def public_bytes(self) -> bytes: + return rust_x509.encode_extension_value(self) + + +class AuthorityKeyIdentifier(ExtensionType): + oid = ExtensionOID.AUTHORITY_KEY_IDENTIFIER + + def __init__( + self, + key_identifier: bytes | None, + authority_cert_issuer: Iterable[GeneralName] | None, + authority_cert_serial_number: int | None, + ) -> None: + if (authority_cert_issuer is None) != ( + authority_cert_serial_number is None + ): + raise ValueError( + "authority_cert_issuer and authority_cert_serial_number " + "must both be present or both None" + ) + + if authority_cert_issuer is not None: + authority_cert_issuer = list(authority_cert_issuer) + if not all( + isinstance(x, GeneralName) for x in authority_cert_issuer + ): + raise TypeError( + "authority_cert_issuer must be a list of GeneralName " + "objects" + ) + + if authority_cert_serial_number is not None and not isinstance( + authority_cert_serial_number, int + ): + raise TypeError("authority_cert_serial_number must be an integer") + + self._key_identifier = key_identifier + self._authority_cert_issuer = authority_cert_issuer + self._authority_cert_serial_number = authority_cert_serial_number + + # This takes a subset of CertificatePublicKeyTypes because an issuer + # cannot have an X25519/X448 key. This introduces some unfortunate + # asymmetry that requires typing users to explicitly + # narrow their type, but we should make this accurate and not just + # convenient. + @classmethod + def from_issuer_public_key( + cls, public_key: CertificateIssuerPublicKeyTypes + ) -> AuthorityKeyIdentifier: + digest = _key_identifier_from_public_key(public_key) + return cls( + key_identifier=digest, + authority_cert_issuer=None, + authority_cert_serial_number=None, + ) + + @classmethod + def from_issuer_subject_key_identifier( + cls, ski: SubjectKeyIdentifier + ) -> AuthorityKeyIdentifier: + return cls( + key_identifier=ski.digest, + authority_cert_issuer=None, + authority_cert_serial_number=None, + ) + + def __repr__(self) -> str: + return ( + f"" + ) + + def __eq__(self, other: object) -> bool: + if not isinstance(other, AuthorityKeyIdentifier): + return NotImplemented + + return ( + self.key_identifier == other.key_identifier + and self.authority_cert_issuer == other.authority_cert_issuer + and self.authority_cert_serial_number + == other.authority_cert_serial_number + ) + + def __hash__(self) -> int: + if self.authority_cert_issuer is None: + aci = None + else: + aci = tuple(self.authority_cert_issuer) + return hash( + (self.key_identifier, aci, self.authority_cert_serial_number) + ) + + @property + def key_identifier(self) -> bytes | None: + return self._key_identifier + + @property + def authority_cert_issuer( + self, + ) -> list[GeneralName] | None: + return self._authority_cert_issuer + + @property + def authority_cert_serial_number(self) -> int | None: + return self._authority_cert_serial_number + + def public_bytes(self) -> bytes: + return rust_x509.encode_extension_value(self) + + +class SubjectKeyIdentifier(ExtensionType): + oid = ExtensionOID.SUBJECT_KEY_IDENTIFIER + + def __init__(self, digest: bytes) -> None: + self._digest = digest + + @classmethod + def from_public_key( + cls, public_key: CertificatePublicKeyTypes + ) -> SubjectKeyIdentifier: + return cls(_key_identifier_from_public_key(public_key)) + + @property + def digest(self) -> bytes: + return self._digest + + @property + def key_identifier(self) -> bytes: + return self._digest + + def __repr__(self) -> str: + return f"" + + def __eq__(self, other: object) -> bool: + if not isinstance(other, SubjectKeyIdentifier): + return NotImplemented + + return constant_time.bytes_eq(self.digest, other.digest) + + def __hash__(self) -> int: + return hash(self.digest) + + def public_bytes(self) -> bytes: + return rust_x509.encode_extension_value(self) + + +class AuthorityInformationAccess(ExtensionType): + oid = ExtensionOID.AUTHORITY_INFORMATION_ACCESS + + def __init__(self, descriptions: Iterable[AccessDescription]) -> None: + descriptions = list(descriptions) + if not all(isinstance(x, AccessDescription) for x in descriptions): + raise TypeError( + "Every item in the descriptions list must be an " + "AccessDescription" + ) + + self._descriptions = descriptions + + __len__, __iter__, __getitem__ = _make_sequence_methods("_descriptions") + + def __repr__(self) -> str: + return f"" + + def __eq__(self, other: object) -> bool: + if not isinstance(other, AuthorityInformationAccess): + return NotImplemented + + return self._descriptions == other._descriptions + + def __hash__(self) -> int: + return hash(tuple(self._descriptions)) + + def public_bytes(self) -> bytes: + return rust_x509.encode_extension_value(self) + + +class SubjectInformationAccess(ExtensionType): + oid = ExtensionOID.SUBJECT_INFORMATION_ACCESS + + def __init__(self, descriptions: Iterable[AccessDescription]) -> None: + descriptions = list(descriptions) + if not all(isinstance(x, AccessDescription) for x in descriptions): + raise TypeError( + "Every item in the descriptions list must be an " + "AccessDescription" + ) + + self._descriptions = descriptions + + __len__, __iter__, __getitem__ = _make_sequence_methods("_descriptions") + + def __repr__(self) -> str: + return f"" + + def __eq__(self, other: object) -> bool: + if not isinstance(other, SubjectInformationAccess): + return NotImplemented + + return self._descriptions == other._descriptions + + def __hash__(self) -> int: + return hash(tuple(self._descriptions)) + + def public_bytes(self) -> bytes: + return rust_x509.encode_extension_value(self) + + +class AccessDescription: + def __init__( + self, access_method: ObjectIdentifier, access_location: GeneralName + ) -> None: + if not isinstance(access_method, ObjectIdentifier): + raise TypeError("access_method must be an ObjectIdentifier") + + if not isinstance(access_location, GeneralName): + raise TypeError("access_location must be a GeneralName") + + self._access_method = access_method + self._access_location = access_location + + def __repr__(self) -> str: + return ( + f"" + ) + + def __eq__(self, other: object) -> bool: + if not isinstance(other, AccessDescription): + return NotImplemented + + return ( + self.access_method == other.access_method + and self.access_location == other.access_location + ) + + def __hash__(self) -> int: + return hash((self.access_method, self.access_location)) + + @property + def access_method(self) -> ObjectIdentifier: + return self._access_method + + @property + def access_location(self) -> GeneralName: + return self._access_location + + +class BasicConstraints(ExtensionType): + oid = ExtensionOID.BASIC_CONSTRAINTS + + def __init__(self, ca: bool, path_length: int | None) -> None: + if not isinstance(ca, bool): + raise TypeError("ca must be a boolean value") + + if path_length is not None and not ca: + raise ValueError("path_length must be None when ca is False") + + if path_length is not None and ( + not isinstance(path_length, int) or path_length < 0 + ): + raise TypeError( + "path_length must be a non-negative integer or None" + ) + + self._ca = ca + self._path_length = path_length + + @property + def ca(self) -> bool: + return self._ca + + @property + def path_length(self) -> int | None: + return self._path_length + + def __repr__(self) -> str: + return ( + f"" + ) + + def __eq__(self, other: object) -> bool: + if not isinstance(other, BasicConstraints): + return NotImplemented + + return self.ca == other.ca and self.path_length == other.path_length + + def __hash__(self) -> int: + return hash((self.ca, self.path_length)) + + def public_bytes(self) -> bytes: + return rust_x509.encode_extension_value(self) + + +class DeltaCRLIndicator(ExtensionType): + oid = ExtensionOID.DELTA_CRL_INDICATOR + + def __init__(self, crl_number: int) -> None: + if not isinstance(crl_number, int): + raise TypeError("crl_number must be an integer") + + self._crl_number = crl_number + + @property + def crl_number(self) -> int: + return self._crl_number + + def __eq__(self, other: object) -> bool: + if not isinstance(other, DeltaCRLIndicator): + return NotImplemented + + return self.crl_number == other.crl_number + + def __hash__(self) -> int: + return hash(self.crl_number) + + def __repr__(self) -> str: + return f"" + + def public_bytes(self) -> bytes: + return rust_x509.encode_extension_value(self) + + +class CRLDistributionPoints(ExtensionType): + oid = ExtensionOID.CRL_DISTRIBUTION_POINTS + + def __init__( + self, distribution_points: Iterable[DistributionPoint] + ) -> None: + distribution_points = list(distribution_points) + if not all( + isinstance(x, DistributionPoint) for x in distribution_points + ): + raise TypeError( + "distribution_points must be a list of DistributionPoint " + "objects" + ) + + self._distribution_points = distribution_points + + __len__, __iter__, __getitem__ = _make_sequence_methods( + "_distribution_points" + ) + + def __repr__(self) -> str: + return f"" + + def __eq__(self, other: object) -> bool: + if not isinstance(other, CRLDistributionPoints): + return NotImplemented + + return self._distribution_points == other._distribution_points + + def __hash__(self) -> int: + return hash(tuple(self._distribution_points)) + + def public_bytes(self) -> bytes: + return rust_x509.encode_extension_value(self) + + +class FreshestCRL(ExtensionType): + oid = ExtensionOID.FRESHEST_CRL + + def __init__( + self, distribution_points: Iterable[DistributionPoint] + ) -> None: + distribution_points = list(distribution_points) + if not all( + isinstance(x, DistributionPoint) for x in distribution_points + ): + raise TypeError( + "distribution_points must be a list of DistributionPoint " + "objects" + ) + + self._distribution_points = distribution_points + + __len__, __iter__, __getitem__ = _make_sequence_methods( + "_distribution_points" + ) + + def __repr__(self) -> str: + return f"" + + def __eq__(self, other: object) -> bool: + if not isinstance(other, FreshestCRL): + return NotImplemented + + return self._distribution_points == other._distribution_points + + def __hash__(self) -> int: + return hash(tuple(self._distribution_points)) + + def public_bytes(self) -> bytes: + return rust_x509.encode_extension_value(self) + + +class DistributionPoint: + def __init__( + self, + full_name: Iterable[GeneralName] | None, + relative_name: RelativeDistinguishedName | None, + reasons: frozenset[ReasonFlags] | None, + crl_issuer: Iterable[GeneralName] | None, + ) -> None: + if full_name and relative_name: + raise ValueError( + "You cannot provide both full_name and relative_name, at " + "least one must be None." + ) + if not full_name and not relative_name and not crl_issuer: + raise ValueError( + "Either full_name, relative_name or crl_issuer must be " + "provided." + ) + + if full_name is not None: + full_name = list(full_name) + if not all(isinstance(x, GeneralName) for x in full_name): + raise TypeError( + "full_name must be a list of GeneralName objects" + ) + + if relative_name: + if not isinstance(relative_name, RelativeDistinguishedName): + raise TypeError( + "relative_name must be a RelativeDistinguishedName" + ) + + if crl_issuer is not None: + crl_issuer = list(crl_issuer) + if not all(isinstance(x, GeneralName) for x in crl_issuer): + raise TypeError( + "crl_issuer must be None or a list of general names" + ) + + if reasons and ( + not isinstance(reasons, frozenset) + or not all(isinstance(x, ReasonFlags) for x in reasons) + ): + raise TypeError("reasons must be None or frozenset of ReasonFlags") + + if reasons and ( + ReasonFlags.unspecified in reasons + or ReasonFlags.remove_from_crl in reasons + ): + raise ValueError( + "unspecified and remove_from_crl are not valid reasons in a " + "DistributionPoint" + ) + + self._full_name = full_name + self._relative_name = relative_name + self._reasons = reasons + self._crl_issuer = crl_issuer + + def __repr__(self) -> str: + return ( + "".format(self) + ) + + def __eq__(self, other: object) -> bool: + if not isinstance(other, DistributionPoint): + return NotImplemented + + return ( + self.full_name == other.full_name + and self.relative_name == other.relative_name + and self.reasons == other.reasons + and self.crl_issuer == other.crl_issuer + ) + + def __hash__(self) -> int: + if self.full_name is not None: + fn: tuple[GeneralName, ...] | None = tuple(self.full_name) + else: + fn = None + + if self.crl_issuer is not None: + crl_issuer: tuple[GeneralName, ...] | None = tuple(self.crl_issuer) + else: + crl_issuer = None + + return hash((fn, self.relative_name, self.reasons, crl_issuer)) + + @property + def full_name(self) -> list[GeneralName] | None: + return self._full_name + + @property + def relative_name(self) -> RelativeDistinguishedName | None: + return self._relative_name + + @property + def reasons(self) -> frozenset[ReasonFlags] | None: + return self._reasons + + @property + def crl_issuer(self) -> list[GeneralName] | None: + return self._crl_issuer + + +class ReasonFlags(utils.Enum): + unspecified = "unspecified" + key_compromise = "keyCompromise" + ca_compromise = "cACompromise" + affiliation_changed = "affiliationChanged" + superseded = "superseded" + cessation_of_operation = "cessationOfOperation" + certificate_hold = "certificateHold" + privilege_withdrawn = "privilegeWithdrawn" + aa_compromise = "aACompromise" + remove_from_crl = "removeFromCRL" + + +# These are distribution point bit string mappings. Not to be confused with +# CRLReason reason flags bit string mappings. +# ReasonFlags ::= BIT STRING { +# unused (0), +# keyCompromise (1), +# cACompromise (2), +# affiliationChanged (3), +# superseded (4), +# cessationOfOperation (5), +# certificateHold (6), +# privilegeWithdrawn (7), +# aACompromise (8) } +_REASON_BIT_MAPPING = { + 1: ReasonFlags.key_compromise, + 2: ReasonFlags.ca_compromise, + 3: ReasonFlags.affiliation_changed, + 4: ReasonFlags.superseded, + 5: ReasonFlags.cessation_of_operation, + 6: ReasonFlags.certificate_hold, + 7: ReasonFlags.privilege_withdrawn, + 8: ReasonFlags.aa_compromise, +} + +_CRLREASONFLAGS = { + ReasonFlags.key_compromise: 1, + ReasonFlags.ca_compromise: 2, + ReasonFlags.affiliation_changed: 3, + ReasonFlags.superseded: 4, + ReasonFlags.cessation_of_operation: 5, + ReasonFlags.certificate_hold: 6, + ReasonFlags.privilege_withdrawn: 7, + ReasonFlags.aa_compromise: 8, +} + +# CRLReason ::= ENUMERATED { +# unspecified (0), +# keyCompromise (1), +# cACompromise (2), +# affiliationChanged (3), +# superseded (4), +# cessationOfOperation (5), +# certificateHold (6), +# -- value 7 is not used +# removeFromCRL (8), +# privilegeWithdrawn (9), +# aACompromise (10) } +_CRL_ENTRY_REASON_ENUM_TO_CODE = { + ReasonFlags.unspecified: 0, + ReasonFlags.key_compromise: 1, + ReasonFlags.ca_compromise: 2, + ReasonFlags.affiliation_changed: 3, + ReasonFlags.superseded: 4, + ReasonFlags.cessation_of_operation: 5, + ReasonFlags.certificate_hold: 6, + ReasonFlags.remove_from_crl: 8, + ReasonFlags.privilege_withdrawn: 9, + ReasonFlags.aa_compromise: 10, +} + + +class PolicyConstraints(ExtensionType): + oid = ExtensionOID.POLICY_CONSTRAINTS + + def __init__( + self, + require_explicit_policy: int | None, + inhibit_policy_mapping: int | None, + ) -> None: + if require_explicit_policy is not None and not isinstance( + require_explicit_policy, int + ): + raise TypeError( + "require_explicit_policy must be a non-negative integer or " + "None" + ) + + if inhibit_policy_mapping is not None and not isinstance( + inhibit_policy_mapping, int + ): + raise TypeError( + "inhibit_policy_mapping must be a non-negative integer or None" + ) + + if inhibit_policy_mapping is None and require_explicit_policy is None: + raise ValueError( + "At least one of require_explicit_policy and " + "inhibit_policy_mapping must not be None" + ) + + self._require_explicit_policy = require_explicit_policy + self._inhibit_policy_mapping = inhibit_policy_mapping + + def __repr__(self) -> str: + return ( + "".format(self) + ) + + def __eq__(self, other: object) -> bool: + if not isinstance(other, PolicyConstraints): + return NotImplemented + + return ( + self.require_explicit_policy == other.require_explicit_policy + and self.inhibit_policy_mapping == other.inhibit_policy_mapping + ) + + def __hash__(self) -> int: + return hash( + (self.require_explicit_policy, self.inhibit_policy_mapping) + ) + + @property + def require_explicit_policy(self) -> int | None: + return self._require_explicit_policy + + @property + def inhibit_policy_mapping(self) -> int | None: + return self._inhibit_policy_mapping + + def public_bytes(self) -> bytes: + return rust_x509.encode_extension_value(self) + + +class CertificatePolicies(ExtensionType): + oid = ExtensionOID.CERTIFICATE_POLICIES + + def __init__(self, policies: Iterable[PolicyInformation]) -> None: + policies = list(policies) + if not all(isinstance(x, PolicyInformation) for x in policies): + raise TypeError( + "Every item in the policies list must be a PolicyInformation" + ) + + self._policies = policies + + __len__, __iter__, __getitem__ = _make_sequence_methods("_policies") + + def __repr__(self) -> str: + return f"" + + def __eq__(self, other: object) -> bool: + if not isinstance(other, CertificatePolicies): + return NotImplemented + + return self._policies == other._policies + + def __hash__(self) -> int: + return hash(tuple(self._policies)) + + def public_bytes(self) -> bytes: + return rust_x509.encode_extension_value(self) + + +class PolicyInformation: + def __init__( + self, + policy_identifier: ObjectIdentifier, + policy_qualifiers: Iterable[str | UserNotice] | None, + ) -> None: + if not isinstance(policy_identifier, ObjectIdentifier): + raise TypeError("policy_identifier must be an ObjectIdentifier") + + self._policy_identifier = policy_identifier + + if policy_qualifiers is not None: + policy_qualifiers = list(policy_qualifiers) + if not all( + isinstance(x, (str, UserNotice)) for x in policy_qualifiers + ): + raise TypeError( + "policy_qualifiers must be a list of strings and/or " + "UserNotice objects or None" + ) + + self._policy_qualifiers = policy_qualifiers + + def __repr__(self) -> str: + return ( + f"" + ) + + def __eq__(self, other: object) -> bool: + if not isinstance(other, PolicyInformation): + return NotImplemented + + return ( + self.policy_identifier == other.policy_identifier + and self.policy_qualifiers == other.policy_qualifiers + ) + + def __hash__(self) -> int: + if self.policy_qualifiers is not None: + pq = tuple(self.policy_qualifiers) + else: + pq = None + + return hash((self.policy_identifier, pq)) + + @property + def policy_identifier(self) -> ObjectIdentifier: + return self._policy_identifier + + @property + def policy_qualifiers( + self, + ) -> list[str | UserNotice] | None: + return self._policy_qualifiers + + +class UserNotice: + def __init__( + self, + notice_reference: NoticeReference | None, + explicit_text: str | None, + ) -> None: + if notice_reference and not isinstance( + notice_reference, NoticeReference + ): + raise TypeError( + "notice_reference must be None or a NoticeReference" + ) + + self._notice_reference = notice_reference + self._explicit_text = explicit_text + + def __repr__(self) -> str: + return ( + f"" + ) + + def __eq__(self, other: object) -> bool: + if not isinstance(other, UserNotice): + return NotImplemented + + return ( + self.notice_reference == other.notice_reference + and self.explicit_text == other.explicit_text + ) + + def __hash__(self) -> int: + return hash((self.notice_reference, self.explicit_text)) + + @property + def notice_reference(self) -> NoticeReference | None: + return self._notice_reference + + @property + def explicit_text(self) -> str | None: + return self._explicit_text + + +class NoticeReference: + def __init__( + self, + organization: str | None, + notice_numbers: Iterable[int], + ) -> None: + self._organization = organization + notice_numbers = list(notice_numbers) + if not all(isinstance(x, int) for x in notice_numbers): + raise TypeError("notice_numbers must be a list of integers") + + self._notice_numbers = notice_numbers + + def __repr__(self) -> str: + return ( + f"" + ) + + def __eq__(self, other: object) -> bool: + if not isinstance(other, NoticeReference): + return NotImplemented + + return ( + self.organization == other.organization + and self.notice_numbers == other.notice_numbers + ) + + def __hash__(self) -> int: + return hash((self.organization, tuple(self.notice_numbers))) + + @property + def organization(self) -> str | None: + return self._organization + + @property + def notice_numbers(self) -> list[int]: + return self._notice_numbers + + +class ExtendedKeyUsage(ExtensionType): + oid = ExtensionOID.EXTENDED_KEY_USAGE + + def __init__(self, usages: Iterable[ObjectIdentifier]) -> None: + usages = list(usages) + if not all(isinstance(x, ObjectIdentifier) for x in usages): + raise TypeError( + "Every item in the usages list must be an ObjectIdentifier" + ) + + self._usages = usages + + __len__, __iter__, __getitem__ = _make_sequence_methods("_usages") + + def __repr__(self) -> str: + return f"" + + def __eq__(self, other: object) -> bool: + if not isinstance(other, ExtendedKeyUsage): + return NotImplemented + + return self._usages == other._usages + + def __hash__(self) -> int: + return hash(tuple(self._usages)) + + def public_bytes(self) -> bytes: + return rust_x509.encode_extension_value(self) + + +class OCSPNoCheck(ExtensionType): + oid = ExtensionOID.OCSP_NO_CHECK + + def __eq__(self, other: object) -> bool: + if not isinstance(other, OCSPNoCheck): + return NotImplemented + + return True + + def __hash__(self) -> int: + return hash(OCSPNoCheck) + + def __repr__(self) -> str: + return "" + + def public_bytes(self) -> bytes: + return rust_x509.encode_extension_value(self) + + +class PrecertPoison(ExtensionType): + oid = ExtensionOID.PRECERT_POISON + + def __eq__(self, other: object) -> bool: + if not isinstance(other, PrecertPoison): + return NotImplemented + + return True + + def __hash__(self) -> int: + return hash(PrecertPoison) + + def __repr__(self) -> str: + return "" + + def public_bytes(self) -> bytes: + return rust_x509.encode_extension_value(self) + + +class TLSFeature(ExtensionType): + oid = ExtensionOID.TLS_FEATURE + + def __init__(self, features: Iterable[TLSFeatureType]) -> None: + features = list(features) + if ( + not all(isinstance(x, TLSFeatureType) for x in features) + or len(features) == 0 + ): + raise TypeError( + "features must be a list of elements from the TLSFeatureType " + "enum" + ) + + self._features = features + + __len__, __iter__, __getitem__ = _make_sequence_methods("_features") + + def __repr__(self) -> str: + return f"" + + def __eq__(self, other: object) -> bool: + if not isinstance(other, TLSFeature): + return NotImplemented + + return self._features == other._features + + def __hash__(self) -> int: + return hash(tuple(self._features)) + + def public_bytes(self) -> bytes: + return rust_x509.encode_extension_value(self) + + +class TLSFeatureType(utils.Enum): + # status_request is defined in RFC 6066 and is used for what is commonly + # called OCSP Must-Staple when present in the TLS Feature extension in an + # X.509 certificate. + status_request = 5 + # status_request_v2 is defined in RFC 6961 and allows multiple OCSP + # responses to be provided. It is not currently in use by clients or + # servers. + status_request_v2 = 17 + + +_TLS_FEATURE_TYPE_TO_ENUM = {x.value: x for x in TLSFeatureType} + + +class InhibitAnyPolicy(ExtensionType): + oid = ExtensionOID.INHIBIT_ANY_POLICY + + def __init__(self, skip_certs: int) -> None: + if not isinstance(skip_certs, int): + raise TypeError("skip_certs must be an integer") + + if skip_certs < 0: + raise ValueError("skip_certs must be a non-negative integer") + + self._skip_certs = skip_certs + + def __repr__(self) -> str: + return f"" + + def __eq__(self, other: object) -> bool: + if not isinstance(other, InhibitAnyPolicy): + return NotImplemented + + return self.skip_certs == other.skip_certs + + def __hash__(self) -> int: + return hash(self.skip_certs) + + @property + def skip_certs(self) -> int: + return self._skip_certs + + def public_bytes(self) -> bytes: + return rust_x509.encode_extension_value(self) + + +class KeyUsage(ExtensionType): + oid = ExtensionOID.KEY_USAGE + + def __init__( + self, + digital_signature: bool, + content_commitment: bool, + key_encipherment: bool, + data_encipherment: bool, + key_agreement: bool, + key_cert_sign: bool, + crl_sign: bool, + encipher_only: bool, + decipher_only: bool, + ) -> None: + if not key_agreement and (encipher_only or decipher_only): + raise ValueError( + "encipher_only and decipher_only can only be true when " + "key_agreement is true" + ) + + self._digital_signature = digital_signature + self._content_commitment = content_commitment + self._key_encipherment = key_encipherment + self._data_encipherment = data_encipherment + self._key_agreement = key_agreement + self._key_cert_sign = key_cert_sign + self._crl_sign = crl_sign + self._encipher_only = encipher_only + self._decipher_only = decipher_only + + @property + def digital_signature(self) -> bool: + return self._digital_signature + + @property + def content_commitment(self) -> bool: + return self._content_commitment + + @property + def key_encipherment(self) -> bool: + return self._key_encipherment + + @property + def data_encipherment(self) -> bool: + return self._data_encipherment + + @property + def key_agreement(self) -> bool: + return self._key_agreement + + @property + def key_cert_sign(self) -> bool: + return self._key_cert_sign + + @property + def crl_sign(self) -> bool: + return self._crl_sign + + @property + def encipher_only(self) -> bool: + if not self.key_agreement: + raise ValueError( + "encipher_only is undefined unless key_agreement is true" + ) + else: + return self._encipher_only + + @property + def decipher_only(self) -> bool: + if not self.key_agreement: + raise ValueError( + "decipher_only is undefined unless key_agreement is true" + ) + else: + return self._decipher_only + + def __repr__(self) -> str: + try: + encipher_only = self.encipher_only + decipher_only = self.decipher_only + except ValueError: + # Users found None confusing because even though encipher/decipher + # have no meaning unless key_agreement is true, to construct an + # instance of the class you still need to pass False. + encipher_only = False + decipher_only = False + + return ( + f"" + ) + + def __eq__(self, other: object) -> bool: + if not isinstance(other, KeyUsage): + return NotImplemented + + return ( + self.digital_signature == other.digital_signature + and self.content_commitment == other.content_commitment + and self.key_encipherment == other.key_encipherment + and self.data_encipherment == other.data_encipherment + and self.key_agreement == other.key_agreement + and self.key_cert_sign == other.key_cert_sign + and self.crl_sign == other.crl_sign + and self._encipher_only == other._encipher_only + and self._decipher_only == other._decipher_only + ) + + def __hash__(self) -> int: + return hash( + ( + self.digital_signature, + self.content_commitment, + self.key_encipherment, + self.data_encipherment, + self.key_agreement, + self.key_cert_sign, + self.crl_sign, + self._encipher_only, + self._decipher_only, + ) + ) + + def public_bytes(self) -> bytes: + return rust_x509.encode_extension_value(self) + + +class PrivateKeyUsagePeriod(ExtensionType): + oid = ExtensionOID.PRIVATE_KEY_USAGE_PERIOD + + def __init__( + self, + not_before: datetime.datetime | None, + not_after: datetime.datetime | None, + ) -> None: + if ( + not isinstance(not_before, datetime.datetime) + and not_before is not None + ): + raise TypeError("not_before must be a datetime.datetime or None") + + if ( + not isinstance(not_after, datetime.datetime) + and not_after is not None + ): + raise TypeError("not_after must be a datetime.datetime or None") + + if not_before is None and not_after is None: + raise ValueError( + "At least one of not_before and not_after must not be None" + ) + + if ( + not_before is not None + and not_after is not None + and not_before > not_after + ): + raise ValueError("not_before must be before not_after") + + self._not_before = not_before + self._not_after = not_after + + @property + def not_before(self) -> datetime.datetime | None: + return self._not_before + + @property + def not_after(self) -> datetime.datetime | None: + return self._not_after + + def __repr__(self) -> str: + return ( + f"" + ) + + def __eq__(self, other: object) -> bool: + if not isinstance(other, PrivateKeyUsagePeriod): + return NotImplemented + + return ( + self.not_before == other.not_before + and self.not_after == other.not_after + ) + + def __hash__(self) -> int: + return hash((self.not_before, self.not_after)) + + def public_bytes(self) -> bytes: + return rust_x509.encode_extension_value(self) + + +class NameConstraints(ExtensionType): + oid = ExtensionOID.NAME_CONSTRAINTS + + def __init__( + self, + permitted_subtrees: Iterable[GeneralName] | None, + excluded_subtrees: Iterable[GeneralName] | None, + ) -> None: + if permitted_subtrees is not None: + permitted_subtrees = list(permitted_subtrees) + if not permitted_subtrees: + raise ValueError( + "permitted_subtrees must be a non-empty list or None" + ) + if not all(isinstance(x, GeneralName) for x in permitted_subtrees): + raise TypeError( + "permitted_subtrees must be a list of GeneralName objects " + "or None" + ) + + self._validate_tree(permitted_subtrees) + + if excluded_subtrees is not None: + excluded_subtrees = list(excluded_subtrees) + if not excluded_subtrees: + raise ValueError( + "excluded_subtrees must be a non-empty list or None" + ) + if not all(isinstance(x, GeneralName) for x in excluded_subtrees): + raise TypeError( + "excluded_subtrees must be a list of GeneralName objects " + "or None" + ) + + self._validate_tree(excluded_subtrees) + + if permitted_subtrees is None and excluded_subtrees is None: + raise ValueError( + "At least one of permitted_subtrees and excluded_subtrees " + "must not be None" + ) + + self._permitted_subtrees = permitted_subtrees + self._excluded_subtrees = excluded_subtrees + + def __eq__(self, other: object) -> bool: + if not isinstance(other, NameConstraints): + return NotImplemented + + return ( + self.excluded_subtrees == other.excluded_subtrees + and self.permitted_subtrees == other.permitted_subtrees + ) + + def _validate_tree(self, tree: Iterable[GeneralName]) -> None: + self._validate_ip_name(tree) + self._validate_dns_name(tree) + + def _validate_ip_name(self, tree: Iterable[GeneralName]) -> None: + if any( + isinstance(name, IPAddress) + and not isinstance( + name.value, (ipaddress.IPv4Network, ipaddress.IPv6Network) + ) + for name in tree + ): + raise TypeError( + "IPAddress name constraints must be an IPv4Network or" + " IPv6Network object" + ) + + def _validate_dns_name(self, tree: Iterable[GeneralName]) -> None: + if any( + isinstance(name, DNSName) and "*" in name.value for name in tree + ): + raise ValueError( + "DNSName name constraints must not contain the '*' wildcard" + " character" + ) + + def __repr__(self) -> str: + return ( + f"" + ) + + def __hash__(self) -> int: + if self.permitted_subtrees is not None: + ps: tuple[GeneralName, ...] | None = tuple(self.permitted_subtrees) + else: + ps = None + + if self.excluded_subtrees is not None: + es: tuple[GeneralName, ...] | None = tuple(self.excluded_subtrees) + else: + es = None + + return hash((ps, es)) + + @property + def permitted_subtrees( + self, + ) -> list[GeneralName] | None: + return self._permitted_subtrees + + @property + def excluded_subtrees( + self, + ) -> list[GeneralName] | None: + return self._excluded_subtrees + + def public_bytes(self) -> bytes: + return rust_x509.encode_extension_value(self) + + +class Extension(typing.Generic[ExtensionTypeVar]): + def __init__( + self, oid: ObjectIdentifier, critical: bool, value: ExtensionTypeVar + ) -> None: + if not isinstance(oid, ObjectIdentifier): + raise TypeError( + "oid argument must be an ObjectIdentifier instance." + ) + + if not isinstance(critical, bool): + raise TypeError("critical must be a boolean value") + + self._oid = oid + self._critical = critical + self._value = value + + @property + def oid(self) -> ObjectIdentifier: + return self._oid + + @property + def critical(self) -> bool: + return self._critical + + @property + def value(self) -> ExtensionTypeVar: + return self._value + + def __repr__(self) -> str: + return ( + f"" + ) + + def __eq__(self, other: object) -> bool: + if not isinstance(other, Extension): + return NotImplemented + + return ( + self.oid == other.oid + and self.critical == other.critical + and self.value == other.value + ) + + def __hash__(self) -> int: + return hash((self.oid, self.critical, self.value)) + + +class GeneralNames: + def __init__(self, general_names: Iterable[GeneralName]) -> None: + general_names = list(general_names) + if not all(isinstance(x, GeneralName) for x in general_names): + raise TypeError( + "Every item in the general_names list must be an " + "object conforming to the GeneralName interface" + ) + + self._general_names = general_names + + __len__, __iter__, __getitem__ = _make_sequence_methods("_general_names") + + @typing.overload + def get_values_for_type( + self, + type: type[DNSName] + | type[UniformResourceIdentifier] + | type[RFC822Name], + ) -> list[str]: ... + + @typing.overload + def get_values_for_type( + self, + type: type[DirectoryName], + ) -> list[Name]: ... + + @typing.overload + def get_values_for_type( + self, + type: type[RegisteredID], + ) -> list[ObjectIdentifier]: ... + + @typing.overload + def get_values_for_type( + self, type: type[IPAddress] + ) -> list[_IPAddressTypes]: ... + + @typing.overload + def get_values_for_type( + self, type: type[OtherName] + ) -> list[OtherName]: ... + + def get_values_for_type( + self, + type: type[DNSName] + | type[DirectoryName] + | type[IPAddress] + | type[OtherName] + | type[RFC822Name] + | type[RegisteredID] + | type[UniformResourceIdentifier], + ) -> ( + list[_IPAddressTypes] + | list[str] + | list[OtherName] + | list[Name] + | list[ObjectIdentifier] + ): + # Return the value of each GeneralName, except for OtherName instances + # which we return directly because it has two important properties not + # just one value. + objs = (i for i in self if isinstance(i, type)) + if type != OtherName: + return [i.value for i in objs] # type: ignore[return-value,unused-ignore] + return list(objs) # type: ignore[return-value,unused-ignore] + + def __repr__(self) -> str: + return f"" + + def __eq__(self, other: object) -> bool: + if not isinstance(other, GeneralNames): + return NotImplemented + + return self._general_names == other._general_names + + def __hash__(self) -> int: + return hash(tuple(self._general_names)) + + +class SubjectAlternativeName(ExtensionType): + oid = ExtensionOID.SUBJECT_ALTERNATIVE_NAME + + def __init__(self, general_names: Iterable[GeneralName]) -> None: + self._general_names = GeneralNames(general_names) + + __len__, __iter__, __getitem__ = _make_sequence_methods("_general_names") + + @typing.overload + def get_values_for_type( + self, + type: type[DNSName] + | type[UniformResourceIdentifier] + | type[RFC822Name], + ) -> list[str]: ... + + @typing.overload + def get_values_for_type( + self, + type: type[DirectoryName], + ) -> list[Name]: ... + + @typing.overload + def get_values_for_type( + self, + type: type[RegisteredID], + ) -> list[ObjectIdentifier]: ... + + @typing.overload + def get_values_for_type( + self, type: type[IPAddress] + ) -> list[_IPAddressTypes]: ... + + @typing.overload + def get_values_for_type( + self, type: type[OtherName] + ) -> list[OtherName]: ... + + def get_values_for_type( + self, + type: type[DNSName] + | type[DirectoryName] + | type[IPAddress] + | type[OtherName] + | type[RFC822Name] + | type[RegisteredID] + | type[UniformResourceIdentifier], + ) -> ( + list[_IPAddressTypes] + | list[str] + | list[OtherName] + | list[Name] + | list[ObjectIdentifier] + ): + return self._general_names.get_values_for_type(type) + + def __repr__(self) -> str: + return f"" + + def __eq__(self, other: object) -> bool: + if not isinstance(other, SubjectAlternativeName): + return NotImplemented + + return self._general_names == other._general_names + + def __hash__(self) -> int: + return hash(self._general_names) + + def public_bytes(self) -> bytes: + return rust_x509.encode_extension_value(self) + + +class IssuerAlternativeName(ExtensionType): + oid = ExtensionOID.ISSUER_ALTERNATIVE_NAME + + def __init__(self, general_names: Iterable[GeneralName]) -> None: + self._general_names = GeneralNames(general_names) + + __len__, __iter__, __getitem__ = _make_sequence_methods("_general_names") + + @typing.overload + def get_values_for_type( + self, + type: type[DNSName] + | type[UniformResourceIdentifier] + | type[RFC822Name], + ) -> list[str]: ... + + @typing.overload + def get_values_for_type( + self, + type: type[DirectoryName], + ) -> list[Name]: ... + + @typing.overload + def get_values_for_type( + self, + type: type[RegisteredID], + ) -> list[ObjectIdentifier]: ... + + @typing.overload + def get_values_for_type( + self, type: type[IPAddress] + ) -> list[_IPAddressTypes]: ... + + @typing.overload + def get_values_for_type( + self, type: type[OtherName] + ) -> list[OtherName]: ... + + def get_values_for_type( + self, + type: type[DNSName] + | type[DirectoryName] + | type[IPAddress] + | type[OtherName] + | type[RFC822Name] + | type[RegisteredID] + | type[UniformResourceIdentifier], + ) -> ( + list[_IPAddressTypes] + | list[str] + | list[OtherName] + | list[Name] + | list[ObjectIdentifier] + ): + return self._general_names.get_values_for_type(type) + + def __repr__(self) -> str: + return f"" + + def __eq__(self, other: object) -> bool: + if not isinstance(other, IssuerAlternativeName): + return NotImplemented + + return self._general_names == other._general_names + + def __hash__(self) -> int: + return hash(self._general_names) + + def public_bytes(self) -> bytes: + return rust_x509.encode_extension_value(self) + + +class CertificateIssuer(ExtensionType): + oid = CRLEntryExtensionOID.CERTIFICATE_ISSUER + + def __init__(self, general_names: Iterable[GeneralName]) -> None: + self._general_names = GeneralNames(general_names) + + __len__, __iter__, __getitem__ = _make_sequence_methods("_general_names") + + @typing.overload + def get_values_for_type( + self, + type: type[DNSName] + | type[UniformResourceIdentifier] + | type[RFC822Name], + ) -> list[str]: ... + + @typing.overload + def get_values_for_type( + self, + type: type[DirectoryName], + ) -> list[Name]: ... + + @typing.overload + def get_values_for_type( + self, + type: type[RegisteredID], + ) -> list[ObjectIdentifier]: ... + + @typing.overload + def get_values_for_type( + self, type: type[IPAddress] + ) -> list[_IPAddressTypes]: ... + + @typing.overload + def get_values_for_type( + self, type: type[OtherName] + ) -> list[OtherName]: ... + + def get_values_for_type( + self, + type: type[DNSName] + | type[DirectoryName] + | type[IPAddress] + | type[OtherName] + | type[RFC822Name] + | type[RegisteredID] + | type[UniformResourceIdentifier], + ) -> ( + list[_IPAddressTypes] + | list[str] + | list[OtherName] + | list[Name] + | list[ObjectIdentifier] + ): + return self._general_names.get_values_for_type(type) + + def __repr__(self) -> str: + return f"" + + def __eq__(self, other: object) -> bool: + if not isinstance(other, CertificateIssuer): + return NotImplemented + + return self._general_names == other._general_names + + def __hash__(self) -> int: + return hash(self._general_names) + + def public_bytes(self) -> bytes: + return rust_x509.encode_extension_value(self) + + +class CRLReason(ExtensionType): + oid = CRLEntryExtensionOID.CRL_REASON + + def __init__(self, reason: ReasonFlags) -> None: + if not isinstance(reason, ReasonFlags): + raise TypeError("reason must be an element from ReasonFlags") + + self._reason = reason + + def __repr__(self) -> str: + return f"" + + def __eq__(self, other: object) -> bool: + if not isinstance(other, CRLReason): + return NotImplemented + + return self.reason == other.reason + + def __hash__(self) -> int: + return hash(self.reason) + + @property + def reason(self) -> ReasonFlags: + return self._reason + + def public_bytes(self) -> bytes: + return rust_x509.encode_extension_value(self) + + +class InvalidityDate(ExtensionType): + oid = CRLEntryExtensionOID.INVALIDITY_DATE + + def __init__(self, invalidity_date: datetime.datetime) -> None: + if not isinstance(invalidity_date, datetime.datetime): + raise TypeError("invalidity_date must be a datetime.datetime") + + self._invalidity_date = invalidity_date + + def __repr__(self) -> str: + return f"" + + def __eq__(self, other: object) -> bool: + if not isinstance(other, InvalidityDate): + return NotImplemented + + return self.invalidity_date == other.invalidity_date + + def __hash__(self) -> int: + return hash(self.invalidity_date) + + @property + def invalidity_date(self) -> datetime.datetime: + return self._invalidity_date + + @property + def invalidity_date_utc(self) -> datetime.datetime: + if self._invalidity_date.tzinfo is None: + return self._invalidity_date.replace(tzinfo=datetime.timezone.utc) + else: + return self._invalidity_date.astimezone(tz=datetime.timezone.utc) + + def public_bytes(self) -> bytes: + return rust_x509.encode_extension_value(self) + + +class PrecertificateSignedCertificateTimestamps(ExtensionType): + oid = ExtensionOID.PRECERT_SIGNED_CERTIFICATE_TIMESTAMPS + + def __init__( + self, + signed_certificate_timestamps: Iterable[SignedCertificateTimestamp], + ) -> None: + signed_certificate_timestamps = list(signed_certificate_timestamps) + if not all( + isinstance(sct, SignedCertificateTimestamp) + for sct in signed_certificate_timestamps + ): + raise TypeError( + "Every item in the signed_certificate_timestamps list must be " + "a SignedCertificateTimestamp" + ) + self._signed_certificate_timestamps = signed_certificate_timestamps + + __len__, __iter__, __getitem__ = _make_sequence_methods( + "_signed_certificate_timestamps" + ) + + def __repr__(self) -> str: + return f"" + + def __hash__(self) -> int: + return hash(tuple(self._signed_certificate_timestamps)) + + def __eq__(self, other: object) -> bool: + if not isinstance(other, PrecertificateSignedCertificateTimestamps): + return NotImplemented + + return ( + self._signed_certificate_timestamps + == other._signed_certificate_timestamps + ) + + def public_bytes(self) -> bytes: + return rust_x509.encode_extension_value(self) + + +class SignedCertificateTimestamps(ExtensionType): + oid = ExtensionOID.SIGNED_CERTIFICATE_TIMESTAMPS + + def __init__( + self, + signed_certificate_timestamps: Iterable[SignedCertificateTimestamp], + ) -> None: + signed_certificate_timestamps = list(signed_certificate_timestamps) + if not all( + isinstance(sct, SignedCertificateTimestamp) + for sct in signed_certificate_timestamps + ): + raise TypeError( + "Every item in the signed_certificate_timestamps list must be " + "a SignedCertificateTimestamp" + ) + self._signed_certificate_timestamps = signed_certificate_timestamps + + __len__, __iter__, __getitem__ = _make_sequence_methods( + "_signed_certificate_timestamps" + ) + + def __repr__(self) -> str: + return f"" + + def __hash__(self) -> int: + return hash(tuple(self._signed_certificate_timestamps)) + + def __eq__(self, other: object) -> bool: + if not isinstance(other, SignedCertificateTimestamps): + return NotImplemented + + return ( + self._signed_certificate_timestamps + == other._signed_certificate_timestamps + ) + + def public_bytes(self) -> bytes: + return rust_x509.encode_extension_value(self) + + +class OCSPNonce(ExtensionType): + oid = OCSPExtensionOID.NONCE + + def __init__(self, nonce: bytes) -> None: + if not isinstance(nonce, bytes): + raise TypeError("nonce must be bytes") + + self._nonce = nonce + + def __eq__(self, other: object) -> bool: + if not isinstance(other, OCSPNonce): + return NotImplemented + + return self.nonce == other.nonce + + def __hash__(self) -> int: + return hash(self.nonce) + + def __repr__(self) -> str: + return f"" + + @property + def nonce(self) -> bytes: + return self._nonce + + def public_bytes(self) -> bytes: + return rust_x509.encode_extension_value(self) + + +class OCSPAcceptableResponses(ExtensionType): + oid = OCSPExtensionOID.ACCEPTABLE_RESPONSES + + def __init__(self, responses: Iterable[ObjectIdentifier]) -> None: + responses = list(responses) + if any(not isinstance(r, ObjectIdentifier) for r in responses): + raise TypeError("All responses must be ObjectIdentifiers") + + self._responses = responses + + def __eq__(self, other: object) -> bool: + if not isinstance(other, OCSPAcceptableResponses): + return NotImplemented + + return self._responses == other._responses + + def __hash__(self) -> int: + return hash(tuple(self._responses)) + + def __repr__(self) -> str: + return f"" + + def __iter__(self) -> Iterator[ObjectIdentifier]: + return iter(self._responses) + + def public_bytes(self) -> bytes: + return rust_x509.encode_extension_value(self) + + +class IssuingDistributionPoint(ExtensionType): + oid = ExtensionOID.ISSUING_DISTRIBUTION_POINT + + def __init__( + self, + full_name: Iterable[GeneralName] | None, + relative_name: RelativeDistinguishedName | None, + only_contains_user_certs: bool, + only_contains_ca_certs: bool, + only_some_reasons: frozenset[ReasonFlags] | None, + indirect_crl: bool, + only_contains_attribute_certs: bool, + ) -> None: + if full_name is not None: + full_name = list(full_name) + + if only_some_reasons and ( + not isinstance(only_some_reasons, frozenset) + or not all(isinstance(x, ReasonFlags) for x in only_some_reasons) + ): + raise TypeError( + "only_some_reasons must be None or frozenset of ReasonFlags" + ) + + if only_some_reasons and ( + ReasonFlags.unspecified in only_some_reasons + or ReasonFlags.remove_from_crl in only_some_reasons + ): + raise ValueError( + "unspecified and remove_from_crl are not valid reasons in an " + "IssuingDistributionPoint" + ) + + if not ( + isinstance(only_contains_user_certs, bool) + and isinstance(only_contains_ca_certs, bool) + and isinstance(indirect_crl, bool) + and isinstance(only_contains_attribute_certs, bool) + ): + raise TypeError( + "only_contains_user_certs, only_contains_ca_certs, " + "indirect_crl and only_contains_attribute_certs " + "must all be boolean." + ) + + # Per RFC5280 Section 5.2.5, the Issuing Distribution Point extension + # in a CRL can have only one of onlyContainsUserCerts, + # onlyContainsCACerts, onlyContainsAttributeCerts set to TRUE. + crl_constraints = [ + only_contains_user_certs, + only_contains_ca_certs, + only_contains_attribute_certs, + ] + + if len([x for x in crl_constraints if x]) > 1: + raise ValueError( + "Only one of the following can be set to True: " + "only_contains_user_certs, only_contains_ca_certs, " + "only_contains_attribute_certs" + ) + + if not any( + [ + only_contains_user_certs, + only_contains_ca_certs, + indirect_crl, + only_contains_attribute_certs, + full_name, + relative_name, + only_some_reasons, + ] + ): + raise ValueError( + "Cannot create empty extension: " + "if only_contains_user_certs, only_contains_ca_certs, " + "indirect_crl, and only_contains_attribute_certs are all False" + ", then either full_name, relative_name, or only_some_reasons " + "must have a value." + ) + + self._only_contains_user_certs = only_contains_user_certs + self._only_contains_ca_certs = only_contains_ca_certs + self._indirect_crl = indirect_crl + self._only_contains_attribute_certs = only_contains_attribute_certs + self._only_some_reasons = only_some_reasons + self._full_name = full_name + self._relative_name = relative_name + + def __repr__(self) -> str: + return ( + f"" + ) + + def __eq__(self, other: object) -> bool: + if not isinstance(other, IssuingDistributionPoint): + return NotImplemented + + return ( + self.full_name == other.full_name + and self.relative_name == other.relative_name + and self.only_contains_user_certs == other.only_contains_user_certs + and self.only_contains_ca_certs == other.only_contains_ca_certs + and self.only_some_reasons == other.only_some_reasons + and self.indirect_crl == other.indirect_crl + and self.only_contains_attribute_certs + == other.only_contains_attribute_certs + ) + + def __hash__(self) -> int: + if self.full_name is not None: + full_name: tuple[GeneralName, ...] | None = tuple(self.full_name) + else: + full_name = None + + return hash( + ( + full_name, + self.relative_name, + self.only_contains_user_certs, + self.only_contains_ca_certs, + self.only_some_reasons, + self.indirect_crl, + self.only_contains_attribute_certs, + ) + ) + + @property + def full_name(self) -> list[GeneralName] | None: + return self._full_name + + @property + def relative_name(self) -> RelativeDistinguishedName | None: + return self._relative_name + + @property + def only_contains_user_certs(self) -> bool: + return self._only_contains_user_certs + + @property + def only_contains_ca_certs(self) -> bool: + return self._only_contains_ca_certs + + @property + def only_some_reasons( + self, + ) -> frozenset[ReasonFlags] | None: + return self._only_some_reasons + + @property + def indirect_crl(self) -> bool: + return self._indirect_crl + + @property + def only_contains_attribute_certs(self) -> bool: + return self._only_contains_attribute_certs + + def public_bytes(self) -> bytes: + return rust_x509.encode_extension_value(self) + + +class MSCertificateTemplate(ExtensionType): + oid = ExtensionOID.MS_CERTIFICATE_TEMPLATE + + def __init__( + self, + template_id: ObjectIdentifier, + major_version: int | None, + minor_version: int | None, + ) -> None: + if not isinstance(template_id, ObjectIdentifier): + raise TypeError("oid must be an ObjectIdentifier") + self._template_id = template_id + if ( + major_version is not None and not isinstance(major_version, int) + ) or ( + minor_version is not None and not isinstance(minor_version, int) + ): + raise TypeError( + "major_version and minor_version must be integers or None" + ) + self._major_version = major_version + self._minor_version = minor_version + + @property + def template_id(self) -> ObjectIdentifier: + return self._template_id + + @property + def major_version(self) -> int | None: + return self._major_version + + @property + def minor_version(self) -> int | None: + return self._minor_version + + def __repr__(self) -> str: + return ( + f"" + ) + + def __eq__(self, other: object) -> bool: + if not isinstance(other, MSCertificateTemplate): + return NotImplemented + + return ( + self.template_id == other.template_id + and self.major_version == other.major_version + and self.minor_version == other.minor_version + ) + + def __hash__(self) -> int: + return hash((self.template_id, self.major_version, self.minor_version)) + + def public_bytes(self) -> bytes: + return rust_x509.encode_extension_value(self) + + +class NamingAuthority: + def __init__( + self, + id: ObjectIdentifier | None, + url: str | None, + text: str | None, + ) -> None: + if id is not None and not isinstance(id, ObjectIdentifier): + raise TypeError("id must be an ObjectIdentifier") + + if url is not None and not isinstance(url, str): + raise TypeError("url must be a str") + + if text is not None and not isinstance(text, str): + raise TypeError("text must be a str") + + self._id = id + self._url = url + self._text = text + + @property + def id(self) -> ObjectIdentifier | None: + return self._id + + @property + def url(self) -> str | None: + return self._url + + @property + def text(self) -> str | None: + return self._text + + def __repr__(self) -> str: + return ( + f"" + ) + + def __eq__(self, other: object) -> bool: + if not isinstance(other, NamingAuthority): + return NotImplemented + + return ( + self.id == other.id + and self.url == other.url + and self.text == other.text + ) + + def __hash__(self) -> int: + return hash( + ( + self.id, + self.url, + self.text, + ) + ) + + +class ProfessionInfo: + def __init__( + self, + naming_authority: NamingAuthority | None, + profession_items: Iterable[str], + profession_oids: Iterable[ObjectIdentifier] | None, + registration_number: str | None, + add_profession_info: bytes | None, + ) -> None: + if naming_authority is not None and not isinstance( + naming_authority, NamingAuthority + ): + raise TypeError("naming_authority must be a NamingAuthority") + + profession_items = list(profession_items) + if not all(isinstance(item, str) for item in profession_items): + raise TypeError( + "Every item in the profession_items list must be a str" + ) + + if profession_oids is not None: + profession_oids = list(profession_oids) + if not all( + isinstance(oid, ObjectIdentifier) for oid in profession_oids + ): + raise TypeError( + "Every item in the profession_oids list must be an " + "ObjectIdentifier" + ) + + if registration_number is not None and not isinstance( + registration_number, str + ): + raise TypeError("registration_number must be a str") + + if add_profession_info is not None and not isinstance( + add_profession_info, bytes + ): + raise TypeError("add_profession_info must be bytes") + + self._naming_authority = naming_authority + self._profession_items = profession_items + self._profession_oids = profession_oids + self._registration_number = registration_number + self._add_profession_info = add_profession_info + + @property + def naming_authority(self) -> NamingAuthority | None: + return self._naming_authority + + @property + def profession_items(self) -> list[str]: + return self._profession_items + + @property + def profession_oids(self) -> list[ObjectIdentifier] | None: + return self._profession_oids + + @property + def registration_number(self) -> str | None: + return self._registration_number + + @property + def add_profession_info(self) -> bytes | None: + return self._add_profession_info + + def __repr__(self) -> str: + return ( + f"" + ) + + def __eq__(self, other: object) -> bool: + if not isinstance(other, ProfessionInfo): + return NotImplemented + + return ( + self.naming_authority == other.naming_authority + and self.profession_items == other.profession_items + and self.profession_oids == other.profession_oids + and self.registration_number == other.registration_number + and self.add_profession_info == other.add_profession_info + ) + + def __hash__(self) -> int: + if self.profession_oids is not None: + profession_oids = tuple(self.profession_oids) + else: + profession_oids = None + return hash( + ( + self.naming_authority, + tuple(self.profession_items), + profession_oids, + self.registration_number, + self.add_profession_info, + ) + ) + + +class Admission: + def __init__( + self, + admission_authority: GeneralName | None, + naming_authority: NamingAuthority | None, + profession_infos: Iterable[ProfessionInfo], + ) -> None: + if admission_authority is not None and not isinstance( + admission_authority, GeneralName + ): + raise TypeError("admission_authority must be a GeneralName") + + if naming_authority is not None and not isinstance( + naming_authority, NamingAuthority + ): + raise TypeError("naming_authority must be a NamingAuthority") + + profession_infos = list(profession_infos) + if not all( + isinstance(info, ProfessionInfo) for info in profession_infos + ): + raise TypeError( + "Every item in the profession_infos list must be a " + "ProfessionInfo" + ) + + self._admission_authority = admission_authority + self._naming_authority = naming_authority + self._profession_infos = profession_infos + + @property + def admission_authority(self) -> GeneralName | None: + return self._admission_authority + + @property + def naming_authority(self) -> NamingAuthority | None: + return self._naming_authority + + @property + def profession_infos(self) -> list[ProfessionInfo]: + return self._profession_infos + + def __repr__(self) -> str: + return ( + f"" + ) + + def __eq__(self, other: object) -> bool: + if not isinstance(other, Admission): + return NotImplemented + + return ( + self.admission_authority == other.admission_authority + and self.naming_authority == other.naming_authority + and self.profession_infos == other.profession_infos + ) + + def __hash__(self) -> int: + return hash( + ( + self.admission_authority, + self.naming_authority, + tuple(self.profession_infos), + ) + ) + + +class Admissions(ExtensionType): + oid = ExtensionOID.ADMISSIONS + + def __init__( + self, + authority: GeneralName | None, + admissions: Iterable[Admission], + ) -> None: + if authority is not None and not isinstance(authority, GeneralName): + raise TypeError("authority must be a GeneralName") + + admissions = list(admissions) + if not all( + isinstance(admission, Admission) for admission in admissions + ): + raise TypeError( + "Every item in the contents_of_admissions list must be an " + "Admission" + ) + + self._authority = authority + self._admissions = admissions + + __len__, __iter__, __getitem__ = _make_sequence_methods("_admissions") + + @property + def authority(self) -> GeneralName | None: + return self._authority + + def __repr__(self) -> str: + return ( + f"" + ) + + def __eq__(self, other: object) -> bool: + if not isinstance(other, Admissions): + return NotImplemented + + return ( + self.authority == other.authority + and self._admissions == other._admissions + ) + + def __hash__(self) -> int: + return hash((self.authority, tuple(self._admissions))) + + def public_bytes(self) -> bytes: + return rust_x509.encode_extension_value(self) + + +class UnrecognizedExtension(ExtensionType): + def __init__(self, oid: ObjectIdentifier, value: bytes) -> None: + if not isinstance(oid, ObjectIdentifier): + raise TypeError("oid must be an ObjectIdentifier") + self._oid = oid + self._value = value + + @property + def oid(self) -> ObjectIdentifier: # type: ignore[override] + return self._oid + + @property + def value(self) -> bytes: + return self._value + + def __repr__(self) -> str: + return f"" + + def __eq__(self, other: object) -> bool: + if not isinstance(other, UnrecognizedExtension): + return NotImplemented + + return self.oid == other.oid and self.value == other.value + + def __hash__(self) -> int: + return hash((self.oid, self.value)) + + def public_bytes(self) -> bytes: + return self.value diff --git a/py_modules/cryptography/x509/general_name.py b/py_modules/cryptography/x509/general_name.py new file mode 100644 index 0000000..672f287 --- /dev/null +++ b/py_modules/cryptography/x509/general_name.py @@ -0,0 +1,281 @@ +# This file is dual licensed under the terms of the Apache License, Version +# 2.0, and the BSD License. See the LICENSE file in the root of this repository +# for complete details. + +from __future__ import annotations + +import abc +import ipaddress +import typing +from email.utils import parseaddr + +from cryptography.x509.name import Name +from cryptography.x509.oid import ObjectIdentifier + +_IPAddressTypes = typing.Union[ + ipaddress.IPv4Address, + ipaddress.IPv6Address, + ipaddress.IPv4Network, + ipaddress.IPv6Network, +] + + +class UnsupportedGeneralNameType(Exception): + pass + + +class GeneralName(metaclass=abc.ABCMeta): + @property + @abc.abstractmethod + def value(self) -> typing.Any: + """ + Return the value of the object + """ + + +class RFC822Name(GeneralName): + def __init__(self, value: str) -> None: + if isinstance(value, str): + try: + value.encode("ascii") + except UnicodeEncodeError: + raise ValueError( + "RFC822Name values should be passed as an A-label string. " + "This means unicode characters should be encoded via " + "a library like idna." + ) + else: + raise TypeError("value must be string") + + name, address = parseaddr(value) + if name or not address: + # parseaddr has found a name (e.g. Name ) or the entire + # value is an empty string. + raise ValueError("Invalid rfc822name value") + + self._value = value + + @property + def value(self) -> str: + return self._value + + @classmethod + def _init_without_validation(cls, value: str) -> RFC822Name: + instance = cls.__new__(cls) + instance._value = value + return instance + + def __repr__(self) -> str: + return f"" + + def __eq__(self, other: object) -> bool: + if not isinstance(other, RFC822Name): + return NotImplemented + + return self.value == other.value + + def __hash__(self) -> int: + return hash(self.value) + + +class DNSName(GeneralName): + def __init__(self, value: str) -> None: + if isinstance(value, str): + try: + value.encode("ascii") + except UnicodeEncodeError: + raise ValueError( + "DNSName values should be passed as an A-label string. " + "This means unicode characters should be encoded via " + "a library like idna." + ) + else: + raise TypeError("value must be string") + + self._value = value + + @property + def value(self) -> str: + return self._value + + @classmethod + def _init_without_validation(cls, value: str) -> DNSName: + instance = cls.__new__(cls) + instance._value = value + return instance + + def __repr__(self) -> str: + return f"" + + def __eq__(self, other: object) -> bool: + if not isinstance(other, DNSName): + return NotImplemented + + return self.value == other.value + + def __hash__(self) -> int: + return hash(self.value) + + +class UniformResourceIdentifier(GeneralName): + def __init__(self, value: str) -> None: + if isinstance(value, str): + try: + value.encode("ascii") + except UnicodeEncodeError: + raise ValueError( + "URI values should be passed as an A-label string. " + "This means unicode characters should be encoded via " + "a library like idna." + ) + else: + raise TypeError("value must be string") + + self._value = value + + @property + def value(self) -> str: + return self._value + + @classmethod + def _init_without_validation(cls, value: str) -> UniformResourceIdentifier: + instance = cls.__new__(cls) + instance._value = value + return instance + + def __repr__(self) -> str: + return f"" + + def __eq__(self, other: object) -> bool: + if not isinstance(other, UniformResourceIdentifier): + return NotImplemented + + return self.value == other.value + + def __hash__(self) -> int: + return hash(self.value) + + +class DirectoryName(GeneralName): + def __init__(self, value: Name) -> None: + if not isinstance(value, Name): + raise TypeError("value must be a Name") + + self._value = value + + @property + def value(self) -> Name: + return self._value + + def __repr__(self) -> str: + return f"" + + def __eq__(self, other: object) -> bool: + if not isinstance(other, DirectoryName): + return NotImplemented + + return self.value == other.value + + def __hash__(self) -> int: + return hash(self.value) + + +class RegisteredID(GeneralName): + def __init__(self, value: ObjectIdentifier) -> None: + if not isinstance(value, ObjectIdentifier): + raise TypeError("value must be an ObjectIdentifier") + + self._value = value + + @property + def value(self) -> ObjectIdentifier: + return self._value + + def __repr__(self) -> str: + return f"" + + def __eq__(self, other: object) -> bool: + if not isinstance(other, RegisteredID): + return NotImplemented + + return self.value == other.value + + def __hash__(self) -> int: + return hash(self.value) + + +class IPAddress(GeneralName): + def __init__(self, value: _IPAddressTypes) -> None: + if not isinstance( + value, + ( + ipaddress.IPv4Address, + ipaddress.IPv6Address, + ipaddress.IPv4Network, + ipaddress.IPv6Network, + ), + ): + raise TypeError( + "value must be an instance of ipaddress.IPv4Address, " + "ipaddress.IPv6Address, ipaddress.IPv4Network, or " + "ipaddress.IPv6Network" + ) + + self._value = value + + @property + def value(self) -> _IPAddressTypes: + return self._value + + def _packed(self) -> bytes: + if isinstance( + self.value, (ipaddress.IPv4Address, ipaddress.IPv6Address) + ): + return self.value.packed + else: + return ( + self.value.network_address.packed + self.value.netmask.packed + ) + + def __repr__(self) -> str: + return f"" + + def __eq__(self, other: object) -> bool: + if not isinstance(other, IPAddress): + return NotImplemented + + return self.value == other.value + + def __hash__(self) -> int: + return hash(self.value) + + +class OtherName(GeneralName): + def __init__(self, type_id: ObjectIdentifier, value: bytes) -> None: + if not isinstance(type_id, ObjectIdentifier): + raise TypeError("type_id must be an ObjectIdentifier") + if not isinstance(value, bytes): + raise TypeError("value must be a binary string") + + self._type_id = type_id + self._value = value + + @property + def type_id(self) -> ObjectIdentifier: + return self._type_id + + @property + def value(self) -> bytes: + return self._value + + def __repr__(self) -> str: + return f"" + + def __eq__(self, other: object) -> bool: + if not isinstance(other, OtherName): + return NotImplemented + + return self.type_id == other.type_id and self.value == other.value + + def __hash__(self) -> int: + return hash((self.type_id, self.value)) diff --git a/py_modules/cryptography/x509/name.py b/py_modules/cryptography/x509/name.py new file mode 100644 index 0000000..2e54edc --- /dev/null +++ b/py_modules/cryptography/x509/name.py @@ -0,0 +1,489 @@ +# This file is dual licensed under the terms of the Apache License, Version +# 2.0, and the BSD License. See the LICENSE file in the root of this repository +# for complete details. + +from __future__ import annotations + +import binascii +import re +import sys +import typing +import warnings +from collections.abc import Iterable, Iterator + +from cryptography import utils +from cryptography.hazmat.bindings._rust import x509 as rust_x509 +from cryptography.x509.oid import NameOID, ObjectIdentifier + + +class _ASN1Type(utils.Enum): + BitString = 3 + OctetString = 4 + UTF8String = 12 + NumericString = 18 + PrintableString = 19 + T61String = 20 + IA5String = 22 + UTCTime = 23 + GeneralizedTime = 24 + VisibleString = 26 + UniversalString = 28 + BMPString = 30 + + +_ASN1_TYPE_TO_ENUM = {i.value: i for i in _ASN1Type} +_NAMEOID_DEFAULT_TYPE: dict[ObjectIdentifier, _ASN1Type] = { + NameOID.COUNTRY_NAME: _ASN1Type.PrintableString, + NameOID.JURISDICTION_COUNTRY_NAME: _ASN1Type.PrintableString, + NameOID.SERIAL_NUMBER: _ASN1Type.PrintableString, + NameOID.DN_QUALIFIER: _ASN1Type.PrintableString, + NameOID.EMAIL_ADDRESS: _ASN1Type.IA5String, + NameOID.DOMAIN_COMPONENT: _ASN1Type.IA5String, +} + +# Type alias +_OidNameMap = typing.Mapping[ObjectIdentifier, str] +_NameOidMap = typing.Mapping[str, ObjectIdentifier] + +#: Short attribute names from RFC 4514: +#: https://tools.ietf.org/html/rfc4514#page-7 +_NAMEOID_TO_NAME: _OidNameMap = { + NameOID.COMMON_NAME: "CN", + NameOID.LOCALITY_NAME: "L", + NameOID.STATE_OR_PROVINCE_NAME: "ST", + NameOID.ORGANIZATION_NAME: "O", + NameOID.ORGANIZATIONAL_UNIT_NAME: "OU", + NameOID.COUNTRY_NAME: "C", + NameOID.STREET_ADDRESS: "STREET", + NameOID.DOMAIN_COMPONENT: "DC", + NameOID.USER_ID: "UID", +} +_NAME_TO_NAMEOID = {v: k for k, v in _NAMEOID_TO_NAME.items()} + +_NAMEOID_LENGTH_LIMIT = { + NameOID.COUNTRY_NAME: (2, 2), + NameOID.JURISDICTION_COUNTRY_NAME: (2, 2), + NameOID.COMMON_NAME: (1, 64), +} + + +def _escape_dn_value(val: str | bytes) -> str: + """Escape special characters in RFC4514 Distinguished Name value.""" + + if not val: + return "" + + # RFC 4514 Section 2.4 defines the value as being the # (U+0023) character + # followed by the hexadecimal encoding of the octets. + if isinstance(val, bytes): + return "#" + binascii.hexlify(val).decode("utf8") + + # See https://tools.ietf.org/html/rfc4514#section-2.4 + val = val.replace("\\", "\\\\") + val = val.replace('"', '\\"') + val = val.replace("+", "\\+") + val = val.replace(",", "\\,") + val = val.replace(";", "\\;") + val = val.replace("<", "\\<") + val = val.replace(">", "\\>") + val = val.replace("\0", "\\00") + + if val[0] == "#" or (val[0] == " " and len(val) > 1): + val = "\\" + val + if val[-1] == " ": + val = val[:-1] + "\\ " + + return val + + +def _unescape_dn_value(val: str) -> str: + if not val: + return "" + + # See https://tools.ietf.org/html/rfc4514#section-3 + + # special = escaped / SPACE / SHARP / EQUALS + # escaped = DQUOTE / PLUS / COMMA / SEMI / LANGLE / RANGLE + def sub(m): + val = m.group(0) + # Special character escape + if len(val) == 2: + return val[1:] + + # Unicode string of hex + return binascii.unhexlify(val.replace("\\", "")).decode() + + return _RFC4514NameParser._PAIR_MULTI_RE.sub(sub, val) + + +NameAttributeValueType = typing.TypeVar( + "NameAttributeValueType", + typing.Union[str, bytes], + str, + bytes, + covariant=True, +) + + +class NameAttribute(typing.Generic[NameAttributeValueType]): + def __init__( + self, + oid: ObjectIdentifier, + value: NameAttributeValueType, + _type: _ASN1Type | None = None, + *, + _validate: bool = True, + ) -> None: + if not isinstance(oid, ObjectIdentifier): + raise TypeError( + "oid argument must be an ObjectIdentifier instance." + ) + if _type == _ASN1Type.BitString: + if oid != NameOID.X500_UNIQUE_IDENTIFIER: + raise TypeError( + "oid must be X500_UNIQUE_IDENTIFIER for BitString type." + ) + if not isinstance(value, bytes): + raise TypeError("value must be bytes for BitString") + elif not isinstance(value, str): + raise TypeError("value argument must be a str") + + length_limits = _NAMEOID_LENGTH_LIMIT.get(oid) + if length_limits is not None: + min_length, max_length = length_limits + assert isinstance(value, str) + c_len = len(value.encode("utf8")) + if c_len < min_length or c_len > max_length: + msg = ( + f"Attribute's length must be >= {min_length} and " + f"<= {max_length}, but it was {c_len}" + ) + if _validate is True: + raise ValueError(msg) + else: + warnings.warn(msg, stacklevel=2) + + # The appropriate ASN1 string type varies by OID and is defined across + # multiple RFCs including 2459, 3280, and 5280. In general UTF8String + # is preferred (2459), but 3280 and 5280 specify several OIDs with + # alternate types. This means when we see the sentinel value we need + # to look up whether the OID has a non-UTF8 type. If it does, set it + # to that. Otherwise, UTF8! + if _type is None: + _type = _NAMEOID_DEFAULT_TYPE.get(oid, _ASN1Type.UTF8String) + + if not isinstance(_type, _ASN1Type): + raise TypeError("_type must be from the _ASN1Type enum") + + self._oid = oid + self._value: NameAttributeValueType = value + self._type: _ASN1Type = _type + + @property + def oid(self) -> ObjectIdentifier: + return self._oid + + @property + def value(self) -> NameAttributeValueType: + return self._value + + @property + def rfc4514_attribute_name(self) -> str: + """ + The short attribute name (for example "CN") if available, + otherwise the OID dotted string. + """ + return _NAMEOID_TO_NAME.get(self.oid, self.oid.dotted_string) + + def rfc4514_string( + self, attr_name_overrides: _OidNameMap | None = None + ) -> str: + """ + Format as RFC4514 Distinguished Name string. + + Use short attribute name if available, otherwise fall back to OID + dotted string. + """ + attr_name = ( + attr_name_overrides.get(self.oid) if attr_name_overrides else None + ) + if attr_name is None: + attr_name = self.rfc4514_attribute_name + + return f"{attr_name}={_escape_dn_value(self.value)}" + + def __eq__(self, other: object) -> bool: + if not isinstance(other, NameAttribute): + return NotImplemented + + return self.oid == other.oid and self.value == other.value + + def __hash__(self) -> int: + return hash((self.oid, self.value)) + + def __repr__(self) -> str: + return f"" + + +class RelativeDistinguishedName: + def __init__(self, attributes: Iterable[NameAttribute[str | bytes]]): + attributes = list(attributes) + if not attributes: + raise ValueError("a relative distinguished name cannot be empty") + if not all(isinstance(x, NameAttribute) for x in attributes): + raise TypeError("attributes must be an iterable of NameAttribute") + + # Keep list and frozenset to preserve attribute order where it matters + self._attributes = attributes + self._attribute_set = frozenset(attributes) + + if len(self._attribute_set) != len(attributes): + raise ValueError("duplicate attributes are not allowed") + + def get_attributes_for_oid( + self, + oid: ObjectIdentifier, + ) -> list[NameAttribute[str | bytes]]: + return [i for i in self if i.oid == oid] + + def rfc4514_string( + self, attr_name_overrides: _OidNameMap | None = None + ) -> str: + """ + Format as RFC4514 Distinguished Name string. + + Within each RDN, attributes are joined by '+', although that is rarely + used in certificates. + """ + return "+".join( + attr.rfc4514_string(attr_name_overrides) + for attr in self._attributes + ) + + def __eq__(self, other: object) -> bool: + if not isinstance(other, RelativeDistinguishedName): + return NotImplemented + + return self._attribute_set == other._attribute_set + + def __hash__(self) -> int: + return hash(self._attribute_set) + + def __iter__(self) -> Iterator[NameAttribute[str | bytes]]: + return iter(self._attributes) + + def __len__(self) -> int: + return len(self._attributes) + + def __repr__(self) -> str: + return f"" + + +class Name: + @typing.overload + def __init__( + self, attributes: Iterable[NameAttribute[str | bytes]] + ) -> None: ... + + @typing.overload + def __init__( + self, attributes: Iterable[RelativeDistinguishedName] + ) -> None: ... + + def __init__( + self, + attributes: Iterable[ + NameAttribute[str | bytes] | RelativeDistinguishedName + ], + ) -> None: + attributes = list(attributes) + if all(isinstance(x, NameAttribute) for x in attributes): + self._attributes = [ + RelativeDistinguishedName([typing.cast(NameAttribute, x)]) + for x in attributes + ] + elif all(isinstance(x, RelativeDistinguishedName) for x in attributes): + self._attributes = typing.cast( + list[RelativeDistinguishedName], attributes + ) + else: + raise TypeError( + "attributes must be a list of NameAttribute" + " or a list RelativeDistinguishedName" + ) + + @classmethod + def from_rfc4514_string( + cls, + data: str, + attr_name_overrides: _NameOidMap | None = None, + ) -> Name: + return _RFC4514NameParser(data, attr_name_overrides or {}).parse() + + @classmethod + def from_bytes(cls, data: bytes) -> Name: + """ + Parse a DER encoded X.509 name (the inverse of ``public_bytes``). + """ + return rust_x509.parse_name_bytes(data) + + def rfc4514_string( + self, attr_name_overrides: _OidNameMap | None = None + ) -> str: + """ + Format as RFC4514 Distinguished Name string. + For example 'CN=foobar.com,O=Foo Corp,C=US' + + An X.509 name is a two-level structure: a list of sets of attributes. + Each list element is separated by ',' and within each list element, set + elements are separated by '+'. The latter is almost never used in + real world certificates. According to RFC4514 section 2.1 the + RDNSequence must be reversed when converting to string representation. + """ + return ",".join( + attr.rfc4514_string(attr_name_overrides) + for attr in reversed(self._attributes) + ) + + def get_attributes_for_oid( + self, + oid: ObjectIdentifier, + ) -> list[NameAttribute[str | bytes]]: + return [i for i in self if i.oid == oid] + + @property + def rdns(self) -> list[RelativeDistinguishedName]: + return self._attributes + + def public_bytes(self, backend: typing.Any = None) -> bytes: + return rust_x509.encode_name_bytes(self) + + def __eq__(self, other: object) -> bool: + if not isinstance(other, Name): + return NotImplemented + + return self._attributes == other._attributes + + def __hash__(self) -> int: + # TODO: this is relatively expensive, if this looks like a bottleneck + # for you, consider optimizing! + return hash(tuple(self._attributes)) + + def __iter__(self) -> Iterator[NameAttribute[str | bytes]]: + for rdn in self._attributes: + yield from rdn + + def __len__(self) -> int: + return sum(len(rdn) for rdn in self._attributes) + + def __repr__(self) -> str: + return f"" + + +class _RFC4514NameParser: + _OID_RE = re.compile(r"(0|([1-9]\d*))(\.(0|([1-9]\d*)))+") + _DESCR_RE = re.compile(r"[a-zA-Z][a-zA-Z\d-]*") + + _ESCAPE_SPECIAL = r"[\\ #=\"\+,;<>]" + _ESCAPE_HEX = r"[\da-zA-Z]{2}" + _PAIR = rf"\\({_ESCAPE_SPECIAL}|{_ESCAPE_HEX})" + _PAIR_MULTI_RE = re.compile(rf"(\\{_ESCAPE_SPECIAL})|((\\{_ESCAPE_HEX})+)") + _LUTF1 = r"[\x01-\x1f\x21\x24-\x2A\x2D-\x3A\x3D\x3F-\x5B\x5D-\x7F]" + _SUTF1 = r"[\x01-\x21\x23-\x2A\x2D-\x3A\x3D\x3F-\x5B\x5D-\x7F]" + _TUTF1 = r"[\x01-\x1F\x21\x23-\x2A\x2D-\x3A\x3D\x3F-\x5B\x5D-\x7F]" + _UTFMB = rf"[\x80-{chr(sys.maxunicode)}]" + _LEADCHAR = rf"{_LUTF1}|{_UTFMB}" + _STRINGCHAR = rf"{_SUTF1}|{_UTFMB}" + _TRAILCHAR = rf"{_TUTF1}|{_UTFMB}" + _STRING_RE = re.compile( + rf""" + ( + ({_LEADCHAR}|{_PAIR}) + ( + ({_STRINGCHAR}|{_PAIR})* + ({_TRAILCHAR}|{_PAIR}) + )? + )? + """, + re.VERBOSE, + ) + _HEXSTRING_RE = re.compile(r"#([\da-zA-Z]{2})+") + + def __init__(self, data: str, attr_name_overrides: _NameOidMap) -> None: + self._data = data + self._idx = 0 + + self._attr_name_overrides = attr_name_overrides + + def _has_data(self) -> bool: + return self._idx < len(self._data) + + def _peek(self) -> str | None: + if self._has_data(): + return self._data[self._idx] + return None + + def _read_char(self, ch: str) -> None: + if self._peek() != ch: + raise ValueError + self._idx += 1 + + def _read_re(self, pat) -> str: + match = pat.match(self._data, pos=self._idx) + if match is None: + raise ValueError + val = match.group() + self._idx += len(val) + return val + + def parse(self) -> Name: + """ + Parses the `data` string and converts it to a Name. + + According to RFC4514 section 2.1 the RDNSequence must be + reversed when converting to string representation. So, when + we parse it, we need to reverse again to get the RDNs on the + correct order. + """ + + if not self._has_data(): + return Name([]) + + rdns = [self._parse_rdn()] + + while self._has_data(): + self._read_char(",") + rdns.append(self._parse_rdn()) + + return Name(reversed(rdns)) + + def _parse_rdn(self) -> RelativeDistinguishedName: + nas = [self._parse_na()] + while self._peek() == "+": + self._read_char("+") + nas.append(self._parse_na()) + + return RelativeDistinguishedName(nas) + + def _parse_na(self) -> NameAttribute[str]: + try: + oid_value = self._read_re(self._OID_RE) + except ValueError: + name = self._read_re(self._DESCR_RE) + oid = self._attr_name_overrides.get( + name, _NAME_TO_NAMEOID.get(name) + ) + if oid is None: + raise ValueError + else: + oid = ObjectIdentifier(oid_value) + + self._read_char("=") + if self._peek() == "#": + value = self._read_re(self._HEXSTRING_RE) + value = binascii.unhexlify(value[1:]).decode() + else: + raw_value = self._read_re(self._STRING_RE) + value = _unescape_dn_value(raw_value) + + return NameAttribute(oid, value) diff --git a/py_modules/cryptography/x509/ocsp.py b/py_modules/cryptography/x509/ocsp.py new file mode 100644 index 0000000..f61ed80 --- /dev/null +++ b/py_modules/cryptography/x509/ocsp.py @@ -0,0 +1,379 @@ +# This file is dual licensed under the terms of the Apache License, Version +# 2.0, and the BSD License. See the LICENSE file in the root of this repository +# for complete details. + +from __future__ import annotations + +import datetime +from collections.abc import Iterable + +from cryptography import utils, x509 +from cryptography.hazmat.bindings._rust import ocsp +from cryptography.hazmat.primitives import hashes +from cryptography.hazmat.primitives.asymmetric.types import ( + CertificateIssuerPrivateKeyTypes, +) +from cryptography.x509.base import _reject_duplicate_extension + + +class OCSPResponderEncoding(utils.Enum): + HASH = "By Hash" + NAME = "By Name" + + +class OCSPResponseStatus(utils.Enum): + SUCCESSFUL = 0 + MALFORMED_REQUEST = 1 + INTERNAL_ERROR = 2 + TRY_LATER = 3 + SIG_REQUIRED = 5 + UNAUTHORIZED = 6 + + +_ALLOWED_HASHES = ( + hashes.SHA1, + hashes.SHA224, + hashes.SHA256, + hashes.SHA384, + hashes.SHA512, +) + + +def _verify_algorithm(algorithm: hashes.HashAlgorithm) -> None: + if not isinstance(algorithm, _ALLOWED_HASHES): + raise ValueError( + "Algorithm must be SHA1, SHA224, SHA256, SHA384, or SHA512" + ) + + +class OCSPCertStatus(utils.Enum): + GOOD = 0 + REVOKED = 1 + UNKNOWN = 2 + + +class _SingleResponse: + def __init__( + self, + resp: tuple[x509.Certificate, x509.Certificate] | None, + resp_hash: tuple[bytes, bytes, int] | None, + algorithm: hashes.HashAlgorithm, + cert_status: OCSPCertStatus, + this_update: datetime.datetime, + next_update: datetime.datetime | None, + revocation_time: datetime.datetime | None, + revocation_reason: x509.ReasonFlags | None, + ): + _verify_algorithm(algorithm) + if not isinstance(this_update, datetime.datetime): + raise TypeError("this_update must be a datetime object") + if next_update is not None and not isinstance( + next_update, datetime.datetime + ): + raise TypeError("next_update must be a datetime object or None") + + self._resp = resp + self._resp_hash = resp_hash + self._algorithm = algorithm + self._this_update = this_update + self._next_update = next_update + + if not isinstance(cert_status, OCSPCertStatus): + raise TypeError( + "cert_status must be an item from the OCSPCertStatus enum" + ) + if cert_status is not OCSPCertStatus.REVOKED: + if revocation_time is not None: + raise ValueError( + "revocation_time can only be provided if the certificate " + "is revoked" + ) + if revocation_reason is not None: + raise ValueError( + "revocation_reason can only be provided if the certificate" + " is revoked" + ) + else: + if not isinstance(revocation_time, datetime.datetime): + raise TypeError("revocation_time must be a datetime object") + + if revocation_reason is not None and not isinstance( + revocation_reason, x509.ReasonFlags + ): + raise TypeError( + "revocation_reason must be an item from the ReasonFlags " + "enum or None" + ) + + self._cert_status = cert_status + self._revocation_time = revocation_time + self._revocation_reason = revocation_reason + + +OCSPRequest = ocsp.OCSPRequest +OCSPResponse = ocsp.OCSPResponse +OCSPSingleResponse = ocsp.OCSPSingleResponse + + +class OCSPRequestBuilder: + def __init__( + self, + request: tuple[ + x509.Certificate, x509.Certificate, hashes.HashAlgorithm + ] + | None = None, + request_hash: tuple[bytes, bytes, int, hashes.HashAlgorithm] + | None = None, + extensions: list[x509.Extension[x509.ExtensionType]] = [], + ) -> None: + self._request = request + self._request_hash = request_hash + self._extensions = extensions + + def add_certificate( + self, + cert: x509.Certificate, + issuer: x509.Certificate, + algorithm: hashes.HashAlgorithm, + ) -> OCSPRequestBuilder: + if self._request is not None or self._request_hash is not None: + raise ValueError("Only one certificate can be added to a request") + + _verify_algorithm(algorithm) + if not isinstance(cert, x509.Certificate) or not isinstance( + issuer, x509.Certificate + ): + raise TypeError("cert and issuer must be a Certificate") + + return OCSPRequestBuilder( + (cert, issuer, algorithm), self._request_hash, self._extensions + ) + + def add_certificate_by_hash( + self, + issuer_name_hash: bytes, + issuer_key_hash: bytes, + serial_number: int, + algorithm: hashes.HashAlgorithm, + ) -> OCSPRequestBuilder: + if self._request is not None or self._request_hash is not None: + raise ValueError("Only one certificate can be added to a request") + + if not isinstance(serial_number, int): + raise TypeError("serial_number must be an integer") + + _verify_algorithm(algorithm) + utils._check_bytes("issuer_name_hash", issuer_name_hash) + utils._check_bytes("issuer_key_hash", issuer_key_hash) + if algorithm.digest_size != len( + issuer_name_hash + ) or algorithm.digest_size != len(issuer_key_hash): + raise ValueError( + "issuer_name_hash and issuer_key_hash must be the same length " + "as the digest size of the algorithm" + ) + + return OCSPRequestBuilder( + self._request, + (issuer_name_hash, issuer_key_hash, serial_number, algorithm), + self._extensions, + ) + + def add_extension( + self, extval: x509.ExtensionType, critical: bool + ) -> OCSPRequestBuilder: + if not isinstance(extval, x509.ExtensionType): + raise TypeError("extension must be an ExtensionType") + + extension = x509.Extension(extval.oid, critical, extval) + _reject_duplicate_extension(extension, self._extensions) + + return OCSPRequestBuilder( + self._request, self._request_hash, [*self._extensions, extension] + ) + + def build(self) -> OCSPRequest: + if self._request is None and self._request_hash is None: + raise ValueError("You must add a certificate before building") + + return ocsp.create_ocsp_request(self) + + +class OCSPResponseBuilder: + def __init__( + self, + response: _SingleResponse | None = None, + responder_id: tuple[x509.Certificate, OCSPResponderEncoding] + | None = None, + certs: list[x509.Certificate] | None = None, + extensions: list[x509.Extension[x509.ExtensionType]] = [], + ): + self._response = response + self._responder_id = responder_id + self._certs = certs + self._extensions = extensions + + def add_response( + self, + cert: x509.Certificate, + issuer: x509.Certificate, + algorithm: hashes.HashAlgorithm, + cert_status: OCSPCertStatus, + this_update: datetime.datetime, + next_update: datetime.datetime | None, + revocation_time: datetime.datetime | None, + revocation_reason: x509.ReasonFlags | None, + ) -> OCSPResponseBuilder: + if self._response is not None: + raise ValueError("Only one response per OCSPResponse.") + + if not isinstance(cert, x509.Certificate) or not isinstance( + issuer, x509.Certificate + ): + raise TypeError("cert and issuer must be a Certificate") + + singleresp = _SingleResponse( + (cert, issuer), + None, + algorithm, + cert_status, + this_update, + next_update, + revocation_time, + revocation_reason, + ) + return OCSPResponseBuilder( + singleresp, + self._responder_id, + self._certs, + self._extensions, + ) + + def add_response_by_hash( + self, + issuer_name_hash: bytes, + issuer_key_hash: bytes, + serial_number: int, + algorithm: hashes.HashAlgorithm, + cert_status: OCSPCertStatus, + this_update: datetime.datetime, + next_update: datetime.datetime | None, + revocation_time: datetime.datetime | None, + revocation_reason: x509.ReasonFlags | None, + ) -> OCSPResponseBuilder: + if self._response is not None: + raise ValueError("Only one response per OCSPResponse.") + + if not isinstance(serial_number, int): + raise TypeError("serial_number must be an integer") + + utils._check_bytes("issuer_name_hash", issuer_name_hash) + utils._check_bytes("issuer_key_hash", issuer_key_hash) + _verify_algorithm(algorithm) + if algorithm.digest_size != len( + issuer_name_hash + ) or algorithm.digest_size != len(issuer_key_hash): + raise ValueError( + "issuer_name_hash and issuer_key_hash must be the same length " + "as the digest size of the algorithm" + ) + + singleresp = _SingleResponse( + None, + (issuer_name_hash, issuer_key_hash, serial_number), + algorithm, + cert_status, + this_update, + next_update, + revocation_time, + revocation_reason, + ) + return OCSPResponseBuilder( + singleresp, + self._responder_id, + self._certs, + self._extensions, + ) + + def responder_id( + self, encoding: OCSPResponderEncoding, responder_cert: x509.Certificate + ) -> OCSPResponseBuilder: + if self._responder_id is not None: + raise ValueError("responder_id can only be set once") + if not isinstance(responder_cert, x509.Certificate): + raise TypeError("responder_cert must be a Certificate") + if not isinstance(encoding, OCSPResponderEncoding): + raise TypeError( + "encoding must be an element from OCSPResponderEncoding" + ) + + return OCSPResponseBuilder( + self._response, + (responder_cert, encoding), + self._certs, + self._extensions, + ) + + def certificates( + self, certs: Iterable[x509.Certificate] + ) -> OCSPResponseBuilder: + if self._certs is not None: + raise ValueError("certificates may only be set once") + certs = list(certs) + if len(certs) == 0: + raise ValueError("certs must not be an empty list") + if not all(isinstance(x, x509.Certificate) for x in certs): + raise TypeError("certs must be a list of Certificates") + return OCSPResponseBuilder( + self._response, + self._responder_id, + certs, + self._extensions, + ) + + def add_extension( + self, extval: x509.ExtensionType, critical: bool + ) -> OCSPResponseBuilder: + if not isinstance(extval, x509.ExtensionType): + raise TypeError("extension must be an ExtensionType") + + extension = x509.Extension(extval.oid, critical, extval) + _reject_duplicate_extension(extension, self._extensions) + + return OCSPResponseBuilder( + self._response, + self._responder_id, + self._certs, + [*self._extensions, extension], + ) + + def sign( + self, + private_key: CertificateIssuerPrivateKeyTypes, + algorithm: hashes.HashAlgorithm | None, + ) -> OCSPResponse: + if self._response is None: + raise ValueError("You must add a response before signing") + if self._responder_id is None: + raise ValueError("You must add a responder_id before signing") + + return ocsp.create_ocsp_response( + OCSPResponseStatus.SUCCESSFUL, self, private_key, algorithm + ) + + @classmethod + def build_unsuccessful( + cls, response_status: OCSPResponseStatus + ) -> OCSPResponse: + if not isinstance(response_status, OCSPResponseStatus): + raise TypeError( + "response_status must be an item from OCSPResponseStatus" + ) + if response_status is OCSPResponseStatus.SUCCESSFUL: + raise ValueError("response_status cannot be SUCCESSFUL") + + return ocsp.create_ocsp_response(response_status, None, None, None) + + +load_der_ocsp_request = ocsp.load_der_ocsp_request +load_der_ocsp_response = ocsp.load_der_ocsp_response diff --git a/py_modules/cryptography/x509/oid.py b/py_modules/cryptography/x509/oid.py new file mode 100644 index 0000000..520fc7a --- /dev/null +++ b/py_modules/cryptography/x509/oid.py @@ -0,0 +1,37 @@ +# This file is dual licensed under the terms of the Apache License, Version +# 2.0, and the BSD License. See the LICENSE file in the root of this repository +# for complete details. + +from __future__ import annotations + +from cryptography.hazmat._oid import ( + AttributeOID, + AuthorityInformationAccessOID, + CertificatePoliciesOID, + CRLEntryExtensionOID, + ExtendedKeyUsageOID, + ExtensionOID, + NameOID, + ObjectIdentifier, + OCSPExtensionOID, + OtherNameFormOID, + PublicKeyAlgorithmOID, + SignatureAlgorithmOID, + SubjectInformationAccessOID, +) + +__all__ = [ + "AttributeOID", + "AuthorityInformationAccessOID", + "CRLEntryExtensionOID", + "CertificatePoliciesOID", + "ExtendedKeyUsageOID", + "ExtensionOID", + "NameOID", + "OCSPExtensionOID", + "ObjectIdentifier", + "OtherNameFormOID", + "PublicKeyAlgorithmOID", + "SignatureAlgorithmOID", + "SubjectInformationAccessOID", +] diff --git a/py_modules/cryptography/x509/verification.py b/py_modules/cryptography/x509/verification.py new file mode 100644 index 0000000..2db4324 --- /dev/null +++ b/py_modules/cryptography/x509/verification.py @@ -0,0 +1,34 @@ +# This file is dual licensed under the terms of the Apache License, Version +# 2.0, and the BSD License. See the LICENSE file in the root of this repository +# for complete details. + +from __future__ import annotations + +import typing + +from cryptography.hazmat.bindings._rust import x509 as rust_x509 +from cryptography.x509.general_name import DNSName, IPAddress + +__all__ = [ + "ClientVerifier", + "Criticality", + "ExtensionPolicy", + "Policy", + "PolicyBuilder", + "ServerVerifier", + "Store", + "Subject", + "VerificationError", + "VerifiedClient", +] + +Store = rust_x509.Store +Subject = typing.Union[DNSName, IPAddress] +VerifiedClient = rust_x509.VerifiedClient +ClientVerifier = rust_x509.ClientVerifier +ServerVerifier = rust_x509.ServerVerifier +PolicyBuilder = rust_x509.PolicyBuilder +Policy = rust_x509.Policy +ExtensionPolicy = rust_x509.ExtensionPolicy +Criticality = rust_x509.Criticality +VerificationError = rust_x509.VerificationError diff --git a/py_modules/frozenlist/__init__.py b/py_modules/frozenlist/__init__.py new file mode 100644 index 0000000..41c8595 --- /dev/null +++ b/py_modules/frozenlist/__init__.py @@ -0,0 +1,86 @@ +import os +import types +from collections.abc import MutableSequence +from functools import total_ordering + +__version__ = "1.8.0" + +__all__ = ("FrozenList", "PyFrozenList") # type: Tuple[str, ...] + + +NO_EXTENSIONS = bool(os.environ.get("FROZENLIST_NO_EXTENSIONS")) # type: bool + + +@total_ordering +class FrozenList(MutableSequence): + __slots__ = ("_frozen", "_items") + __class_getitem__ = classmethod(types.GenericAlias) + + def __init__(self, items=None): + self._frozen = False + if items is not None: + items = list(items) + else: + items = [] + self._items = items + + @property + def frozen(self): + return self._frozen + + def freeze(self): + self._frozen = True + + def __getitem__(self, index): + return self._items[index] + + def __setitem__(self, index, value): + if self._frozen: + raise RuntimeError("Cannot modify frozen list.") + self._items[index] = value + + def __delitem__(self, index): + if self._frozen: + raise RuntimeError("Cannot modify frozen list.") + del self._items[index] + + def __len__(self): + return self._items.__len__() + + def __iter__(self): + return self._items.__iter__() + + def __reversed__(self): + return self._items.__reversed__() + + def __eq__(self, other): + return list(self) == other + + def __le__(self, other): + return list(self) <= other + + def insert(self, pos, item): + if self._frozen: + raise RuntimeError("Cannot modify frozen list.") + self._items.insert(pos, item) + + def __repr__(self): + return f"" + + def __hash__(self): + if self._frozen: + return hash(tuple(self)) + else: + raise RuntimeError("Cannot hash unfrozen list.") + + +PyFrozenList = FrozenList + + +if not NO_EXTENSIONS: + try: + from ._frozenlist import FrozenList as CFrozenList # type: ignore + except ImportError: # pragma: no cover + pass + else: + FrozenList = CFrozenList # type: ignore diff --git a/py_modules/frozenlist/__init__.pyi b/py_modules/frozenlist/__init__.pyi new file mode 100644 index 0000000..ae803ef --- /dev/null +++ b/py_modules/frozenlist/__init__.pyi @@ -0,0 +1,47 @@ +from typing import ( + Generic, + Iterable, + Iterator, + List, + MutableSequence, + Optional, + TypeVar, + Union, + overload, +) + +_T = TypeVar("_T") +_Arg = Union[List[_T], Iterable[_T]] + +class FrozenList(MutableSequence[_T], Generic[_T]): + def __init__(self, items: Optional[_Arg[_T]] = None) -> None: ... + @property + def frozen(self) -> bool: ... + def freeze(self) -> None: ... + @overload + def __getitem__(self, i: int) -> _T: ... + @overload + def __getitem__(self, s: slice) -> FrozenList[_T]: ... + @overload + def __setitem__(self, i: int, o: _T) -> None: ... + @overload + def __setitem__(self, s: slice, o: Iterable[_T]) -> None: ... + @overload + def __delitem__(self, i: int) -> None: ... + @overload + def __delitem__(self, i: slice) -> None: ... + def __len__(self) -> int: ... + def __iter__(self) -> Iterator[_T]: ... + def __reversed__(self) -> Iterator[_T]: ... + def __eq__(self, other: object) -> bool: ... + def __le__(self, other: FrozenList[_T]) -> bool: ... + def __ne__(self, other: object) -> bool: ... + def __lt__(self, other: FrozenList[_T]) -> bool: ... + def __ge__(self, other: FrozenList[_T]) -> bool: ... + def __gt__(self, other: FrozenList[_T]) -> bool: ... + def insert(self, pos: int, item: _T) -> None: ... + def __repr__(self) -> str: ... + def __hash__(self) -> int: ... + +# types for C accelerators are the same +CFrozenList = PyFrozenList = FrozenList diff --git a/py_modules/frozenlist/_frozenlist.cpython-311-x86_64-linux-gnu.so b/py_modules/frozenlist/_frozenlist.cpython-311-x86_64-linux-gnu.so new file mode 100755 index 0000000..b049990 Binary files /dev/null and b/py_modules/frozenlist/_frozenlist.cpython-311-x86_64-linux-gnu.so differ diff --git a/py_modules/frozenlist/_frozenlist.pyx b/py_modules/frozenlist/_frozenlist.pyx new file mode 100644 index 0000000..a82d8c8 --- /dev/null +++ b/py_modules/frozenlist/_frozenlist.pyx @@ -0,0 +1,148 @@ +# cython: freethreading_compatible = True +# distutils: language = c++ + +from cpython.bool cimport PyBool_FromLong +from libcpp.atomic cimport atomic + +import copy +import types +from collections.abc import MutableSequence + + +cdef class FrozenList: + __class_getitem__ = classmethod(types.GenericAlias) + + cdef atomic[bint] _frozen + cdef list _items + + def __init__(self, items=None): + self._frozen.store(False) + if items is not None: + items = list(items) + else: + items = [] + self._items = items + + @property + def frozen(self): + return PyBool_FromLong(self._frozen.load()) + + cdef object _check_frozen(self): + if self._frozen.load(): + raise RuntimeError("Cannot modify frozen list.") + + cdef inline object _fast_len(self): + return len(self._items) + + def freeze(self): + self._frozen.store(True) + + def __getitem__(self, index): + return self._items[index] + + def __setitem__(self, index, value): + self._check_frozen() + self._items[index] = value + + def __delitem__(self, index): + self._check_frozen() + del self._items[index] + + def __len__(self): + return self._fast_len() + + def __iter__(self): + return self._items.__iter__() + + def __reversed__(self): + return self._items.__reversed__() + + def __richcmp__(self, other, op): + if op == 0: # < + return list(self) < other + if op == 1: # <= + return list(self) <= other + if op == 2: # == + return list(self) == other + if op == 3: # != + return list(self) != other + if op == 4: # > + return list(self) > other + if op == 5: # => + return list(self) >= other + + def insert(self, pos, item): + self._check_frozen() + self._items.insert(pos, item) + + def __contains__(self, item): + return item in self._items + + def __iadd__(self, items): + self._check_frozen() + self._items += list(items) + return self + + def index(self, item): + return self._items.index(item) + + def remove(self, item): + self._check_frozen() + self._items.remove(item) + + def clear(self): + self._check_frozen() + self._items.clear() + + def extend(self, items): + self._check_frozen() + self._items += list(items) + + def reverse(self): + self._check_frozen() + self._items.reverse() + + def pop(self, index=-1): + self._check_frozen() + return self._items.pop(index) + + def append(self, item): + self._check_frozen() + return self._items.append(item) + + def count(self, item): + return self._items.count(item) + + def __repr__(self): + return ''.format(self._frozen.load(), + self._items) + + def __hash__(self): + if self._frozen.load(): + return hash(tuple(self._items)) + else: + raise RuntimeError("Cannot hash unfrozen list.") + + def __deepcopy__(self, memo): + cdef FrozenList new_list + obj_id = id(self) + + # Return existing copy if already processed (circular reference) + if obj_id in memo: + return memo[obj_id] + + # Create new instance and register immediately + new_list = self.__class__([]) + memo[obj_id] = new_list + + # Deep copy items + new_list._items[:] = [copy.deepcopy(item, memo) for item in self._items] + + # Preserve frozen state + if self._frozen.load(): + new_list.freeze() + + return new_list + + +MutableSequence.register(FrozenList) diff --git a/py_modules/frozenlist/py.typed b/py_modules/frozenlist/py.typed new file mode 100644 index 0000000..f5642f7 --- /dev/null +++ b/py_modules/frozenlist/py.typed @@ -0,0 +1 @@ +Marker diff --git a/py_modules/idna/__init__.py b/py_modules/idna/__init__.py new file mode 100644 index 0000000..d6c890a --- /dev/null +++ b/py_modules/idna/__init__.py @@ -0,0 +1,47 @@ +from .core import ( + IDNABidiError, + IDNAError, + InvalidCodepoint, + InvalidCodepointContext, + alabel, + check_bidi, + check_hyphen_ok, + check_initial_combiner, + check_label, + check_nfc, + decode, + encode, + ulabel, + uts46_remap, + valid_contextj, + valid_contexto, + valid_label_length, + valid_string_length, +) +from .idnadata import __version__ as unicode_version +from .intranges import intranges_contain +from .package_data import __version__ + +__all__ = [ + "__version__", + "unicode_version", + "IDNABidiError", + "IDNAError", + "InvalidCodepoint", + "InvalidCodepointContext", + "alabel", + "check_bidi", + "check_hyphen_ok", + "check_initial_combiner", + "check_label", + "check_nfc", + "decode", + "encode", + "intranges_contain", + "ulabel", + "uts46_remap", + "valid_contextj", + "valid_contexto", + "valid_label_length", + "valid_string_length", +] diff --git a/py_modules/idna/__main__.py b/py_modules/idna/__main__.py new file mode 100644 index 0000000..dbdd066 --- /dev/null +++ b/py_modules/idna/__main__.py @@ -0,0 +1,6 @@ +import sys + +from .cli import main + +if __name__ == "__main__": + sys.exit(main()) diff --git a/py_modules/idna/cli.py b/py_modules/idna/cli.py new file mode 100644 index 0000000..894d8a0 --- /dev/null +++ b/py_modules/idna/cli.py @@ -0,0 +1,132 @@ +"""Command-line interface for the :mod:`idna` package. + +Invoked via ``python -m idna``. See :func:`main` for the entry point. +""" + +from __future__ import annotations + +import argparse +import sys +from itertools import chain +from typing import IO, TYPE_CHECKING + +from . import IDNAError, decode, encode, unicode_version +from .core import _alabel_prefix, _unicode_dots_re +from .package_data import __version__ + +if TYPE_CHECKING: + from collections.abc import Iterable + + +def _looks_like_alabel(s: str) -> bool: + """Return True if any label in ``s`` carries the ``xn--`` ACE prefix.""" + prefix = _alabel_prefix.decode("ascii") + return any(label.lower().startswith(prefix) for label in _unicode_dots_re.split(s)) + + +def _build_parser() -> argparse.ArgumentParser: + parser = argparse.ArgumentParser( + prog="python -m idna", + description=( + "Convert a domain name between its Unicode (U-label) and " + "ASCII-compatible (A-label) forms. With no mode flag, the " + "direction is chosen from the first input — if it contains " + "an xn-- label the stream is decoded, otherwise it is " + "encoded — and the same mode is applied to every remaining " + "input. UTS #46 mapping is applied by default; pass " + "--strict to disable it. When no domains are given on the " + "command line and stdin is piped, one domain per line is " + "read from stdin." + ), + ) + mode = parser.add_mutually_exclusive_group() + mode.add_argument( + "-e", + "--encode", + dest="mode", + action="store_const", + const="encode", + help="Encode the input to its ASCII A-label form.", + ) + mode.add_argument( + "-d", + "--decode", + dest="mode", + action="store_const", + const="decode", + help="Decode the input from its ASCII A-label form.", + ) + parser.add_argument( + "--strict", + action="store_true", + help="Disable the default UTS #46 mapping and apply IDNA 2008 rules verbatim.", + ) + parser.add_argument( + "--version", + action="version", + version=f"idna {__version__} (Unicode {unicode_version})", + ) + parser.add_argument( + "domain", + nargs="*", + help="One or more domain names to convert. Omit to read from stdin.", + ) + return parser + + +def _iter_stdin(stream: IO[str]) -> Iterable[str]: + """Yield non-empty stripped lines from ``stream``, ignoring blanks.""" + for line in stream: + stripped = line.strip() + if stripped: + yield stripped + + +def _convert_one(domain: str, mode: str, uts46: bool) -> bool: + """Convert ``domain`` and write the result; return ``False`` on failure.""" + try: + if mode == "decode": + print(decode(domain, uts46=uts46)) + else: + print(encode(domain, uts46=uts46).decode("ascii")) + except IDNAError as err: + print(f"idna: {mode} failed for {domain!r}: {err}", file=sys.stderr) + return False + return True + + +def main(argv: list[str] | None = None) -> int: + """Entry point for ``python -m idna``. + + When more than one domain is supplied (via positional arguments or + piped stdin) and no mode flag is given, the first input determines + the direction and that mode is applied uniformly to the rest. + + :param argv: Argument list excluding the program name. Defaults to + :data:`sys.argv` when ``None``. + :returns: ``0`` on success, ``1`` if any conversion fails. + """ + parser = _build_parser() + args = parser.parse_args(argv) + uts46 = not args.strict + + if args.domain: + domains: Iterable[str] = args.domain + elif not sys.stdin.isatty(): + domains = _iter_stdin(sys.stdin) + else: + parser.error("a domain argument is required when stdin is a terminal") + + iterator = iter(domains) + first = next(iterator, None) + if first is None: + return 0 + + mode = args.mode or ("decode" if _looks_like_alabel(first) else "encode") + + results = [_convert_one(domain, mode, uts46) for domain in chain([first], iterator)] + return 0 if all(results) else 1 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/py_modules/idna/codec.py b/py_modules/idna/codec.py new file mode 100644 index 0000000..17995c9 --- /dev/null +++ b/py_modules/idna/codec.py @@ -0,0 +1,163 @@ +from __future__ import annotations + +import codecs +from typing import Any + +from .core import IDNAError, _unicode_dots_re, alabel, decode, encode, ulabel + + +class Codec(codecs.Codec): + """Stateless IDNA 2008 codec. + + Implements the :class:`codecs.Codec` protocol so that the whole-domain + encoder (:func:`idna.encode`) and decoder (:func:`idna.decode`) are + accessible through the standard codec machinery as ``"idna2008"``. + + Only the ``"strict"`` error handler is supported; any other handler + raises :exc:`~idna.IDNAError`. + """ + + def encode(self, data: str, errors: str = "strict") -> tuple[bytes, int]: # ty: ignore[invalid-method-override] + if errors != "strict": + raise IDNAError(f'Unsupported error handling "{errors}"', code="unsupported_errors") + + if not data: + return b"", 0 + + return encode(data), len(data) + + def decode(self, data: bytes, errors: str = "strict") -> tuple[str, int]: # ty: ignore[invalid-method-override] + if errors != "strict": + raise IDNAError(f'Unsupported error handling "{errors}"', code="unsupported_errors") + + if not data: + return "", 0 + + return decode(data), len(data) + + +class IncrementalEncoder(codecs.BufferedIncrementalEncoder): + """Incremental IDNA 2008 encoder. + + Buffers a partial trailing label across calls until either the next + label separator is seen or ``final=True``, so that streamed input is + encoded one whole label at a time. Any of the four Unicode label + separators (``U+002E``, ``U+3002``, ``U+FF0E``, ``U+FF61``) ends a + label; the result always uses ``U+002E`` as the separator. + + Only the ``"strict"`` error handler is supported. + """ + + def _buffer_encode(self, data: str, errors: str, final: bool) -> tuple[bytes, int]: # ty: ignore[invalid-method-override] + if errors != "strict": + raise IDNAError(f'Unsupported error handling "{errors}"', code="unsupported_errors") + + if not data: + return b"", 0 + + labels = _unicode_dots_re.split(data) + trailing_dot = b"" + if labels: + if not labels[-1]: + trailing_dot = b"." + del labels[-1] + elif not final: + # Keep potentially unfinished label until the next call + del labels[-1] + if labels: + trailing_dot = b"." + + result = [] + size = 0 + for label in labels: + result.append(alabel(label)) + if size: + size += 1 + size += len(label) + + result_bytes = b".".join(result) + trailing_dot + size += len(trailing_dot) + return result_bytes, size + + +class IncrementalDecoder(codecs.BufferedIncrementalDecoder): + """Incremental IDNA 2008 decoder. + + Buffers a partial trailing label across calls until either the next + label separator is seen or ``final=True``, so that streamed input is + decoded one whole label at a time. + + Only the ``"strict"`` error handler is supported. + """ + + def _buffer_decode(self, data: Any, errors: str, final: bool) -> tuple[str, int]: # ty: ignore[invalid-method-override] + if errors != "strict": + raise IDNAError(f'Unsupported error handling "{errors}"', code="unsupported_errors") + + if not data: + return ("", 0) + + if not isinstance(data, str): + try: + data = str(data, "ascii") + except UnicodeDecodeError as err: + raise IDNAError("Invalid ASCII in A-label", code="invalid_ascii") from err + + labels = _unicode_dots_re.split(data) + trailing_dot = "" + if labels: + if not labels[-1]: + trailing_dot = "." + del labels[-1] + elif not final: + # Keep potentially unfinished label until the next call + del labels[-1] + if labels: + trailing_dot = "." + + result = [] + size = 0 + for label in labels: + result.append(ulabel(label)) + if size: + size += 1 + size += len(label) + + result_str = ".".join(result) + trailing_dot + size += len(trailing_dot) + return (result_str, size) + + +class StreamWriter(Codec, codecs.StreamWriter): + pass + + +class StreamReader(Codec, codecs.StreamReader): + pass + + +def search_function(name: str) -> codecs.CodecInfo | None: + """Codec search function registered with :mod:`codecs`. + + Returns a :class:`codecs.CodecInfo` for the ``"idna2008"`` codec name + so that ``str.encode("idna2008")`` and ``bytes.decode("idna2008")`` + invoke the IDNA 2008 codec defined in this module. + + :param name: The codec name being looked up. + :returns: A :class:`codecs.CodecInfo` instance if ``name`` is + ``"idna2008"``, otherwise ``None``. + """ + if name != "idna2008": + return None + return codecs.CodecInfo( + name=name, + encode=Codec().encode, + decode=Codec().decode, # type: ignore + incrementalencoder=IncrementalEncoder, + incrementaldecoder=IncrementalDecoder, + streamwriter=StreamWriter, + streamreader=StreamReader, + ) + + +codecs.register(search_function) diff --git a/py_modules/idna/compat.py b/py_modules/idna/compat.py new file mode 100644 index 0000000..5bedee9 --- /dev/null +++ b/py_modules/idna/compat.py @@ -0,0 +1,43 @@ +from __future__ import annotations + +from typing import Any + +from .core import decode, encode + + +def ToASCII(label: str) -> bytes: + """Compatibility shim for :rfc:`3490` ``ToASCII``. + + Delegates to :func:`idna.encode` (IDNA 2008). Provided to ease porting + of code written against the legacy :mod:`encodings.idna` API; new code + should call :func:`idna.encode` directly. + + :param label: The label or domain to encode. + :returns: The encoded form as ASCII :class:`bytes`. + """ + return encode(label) + + +def ToUnicode(label: bytes | bytearray) -> str: + """Compatibility shim for :rfc:`3490` ``ToUnicode``. + + Delegates to :func:`idna.decode` (IDNA 2008). Provided to ease porting + of code written against the legacy :mod:`encodings.idna` API; new code + should call :func:`idna.decode` directly. + + :param label: The label or domain to decode. + :returns: The decoded Unicode form. + """ + return decode(label) + + +def nameprep(s: Any) -> None: + """Stub for :rfc:`3491` Nameprep, which is not used by IDNA 2008. + + IDNA 2008 (:rfc:`5891`) replaces Nameprep with the per-codepoint + validity classes from :rfc:`5892`; this function exists only to + return a clear error if legacy code attempts to call it. + + :raises NotImplementedError: Always. + """ + raise NotImplementedError("IDNA 2008 does not utilise nameprep protocol") diff --git a/py_modules/idna/core.py b/py_modules/idna/core.py new file mode 100644 index 0000000..138ee10 --- /dev/null +++ b/py_modules/idna/core.py @@ -0,0 +1,862 @@ +from __future__ import annotations + +import bisect +import re +import unicodedata +import warnings +from typing import Literal + +from . import idnadata +from .intranges import intranges_contain + +_virama_combining_class = 9 +_alabel_prefix = b"xn--" +_max_input_length = 1024 +_STATUS_VALID, _STATUS_MAPPED, _STATUS_DEVIATION, _STATUS_IGNORED = b"VMDI" +_unicode_dots_re = re.compile("[\u002e\u3002\uff0e\uff61]") +_std3_disallowed_re = re.compile("[\x00-\x2c\x2f\x3a-\x40A-Z\x5b-\x60\x7b-\x7f]") +_bidi_rtl_first = frozenset({"R", "AL"}) +_bidi_rtl_categories = frozenset({"R", "AL", "AN"}) +_bidi_rtl_allowed = frozenset({"R", "AL", "AN", "EN", "ES", "CS", "ET", "ON", "BN", "NSM"}) +_bidi_rtl_valid_ending = frozenset({"R", "AL", "EN", "AN"}) +_bidi_rtl_numeric = frozenset({"AN", "EN"}) +_bidi_ltr_allowed = frozenset({"L", "EN", "ES", "CS", "ET", "ON", "BN", "NSM"}) +_bidi_ltr_valid_ending = frozenset({"L", "EN"}) +_bidi_joiner_l_or_d = frozenset({"L", "D"}) +_bidi_joiner_r_or_d = frozenset({"R", "D"}) + + +def _joining_type(cp: int) -> str | None: + for jt, ranges in idnadata.joining_types.items(): + if intranges_contain(cp, ranges): + return jt + return None + + +# Machine-readable identifiers for the rule an :class:`IDNAError` reports. +# These strings are stable and documented; exception message wording is not. +_ErrorCode = Literal[ + "input_too_long", + "label_too_long", + "domain_too_long", + "empty_label", + "empty_domain", + "not_nfc", + "hyphen_3_4", + "hyphen_start_end", + "leading_combiner", + "disallowed_codepoint", + "contextj", + "contexto", + "unknown_codepoint", + "bidi_rule_1", + "bidi_rule_2", + "bidi_rule_3", + "bidi_rule_4", + "bidi_rule_5", + "bidi_rule_6", + "bidi_unknown_direction", + "invalid_alabel", + "non_canonical_alabel", + "invalid_ascii", + "invalid_utf8", + "uts46_disallowed", + "uts46_std3", + "unsupported_errors", +] + + +class IDNAError(UnicodeError): + """Base exception for all IDNA-encoding related problems. + + ``str(err)`` is a human-readable description of the failure. The + exception also carries machine-readable attributes so callers do not + need to parse the message: + + * ``code`` -- a short, stable identifier for the rule that failed, such + as ``"disallowed_codepoint"`` or ``"bidi_rule_2"``; the full list is + documented in the README. Message wording, by contrast, may change + between releases. + * ``text`` -- the label (or, for UTS #46 processing, the domain) that + was being validated; + * ``codepoint`` -- the offending codepoint, as an ``int``; + * ``position`` -- the 1-based index of the offending character within + ``text``, matching the position quoted in the message. + + Each is ``None`` when it does not apply. + """ + + code: str | None + text: str | None + codepoint: int | None + position: int | None + + def __init__( + self, + *args: object, + code: _ErrorCode | None = None, + text: str | None = None, + codepoint: int | None = None, + position: int | None = None, + ) -> None: + super().__init__(*args) + self.code = code + self.text = text + self.codepoint = codepoint + self.position = position + + +class IDNABidiError(IDNAError): + """Exception when bidirectional requirements are not satisfied""" + + +class InvalidCodepoint(IDNAError): + """Exception when a disallowed or unallocated codepoint is used""" + + +class InvalidCodepointContext(IDNAError): + """Exception when the codepoint is not valid in the context it is used""" + + +def _combining_class(cp: int) -> int: + v = unicodedata.combining(chr(cp)) + if v == 0 and not unicodedata.name(chr(cp)): + raise ValueError("Unknown character in unicodedata") + return v + + +def _is_script(cp: str, script: str) -> bool: + return intranges_contain(ord(cp), idnadata.scripts[script]) + + +def _punycode(s: str) -> bytes: + return s.encode("punycode") + + +def _unot(s: int) -> str: + return f"U+{s:04X}" + + +def valid_label_length(label: bytes | str) -> bool: + """Check that a label does not exceed the maximum permitted length. + + Per :rfc:`1035` (and :rfc:`5891` §4.2.4) a DNS label must not exceed + 63 octets. The argument may be either a :class:`str` (a U-label, where + length is measured in characters) or :class:`bytes` (an A-label, where + length is measured in octets). + + :param label: The label to check. + :returns: ``True`` if the label is within the length limit, otherwise + ``False``. + """ + return len(label) <= 63 + + +def valid_string_length(domain: bytes | str, trailing_dot: bool) -> bool: + """Check that a full domain name does not exceed the maximum length. + + Per :rfc:`1035`, a domain name is limited to 253 octets when no trailing + dot is present, or 254 octets when one is included. + + :param domain: The full (possibly multi-label) domain name. + :param trailing_dot: ``True`` if ``domain`` includes a trailing ``.``. + :returns: ``True`` if the domain is within the length limit, otherwise + ``False``. + """ + return len(domain) <= (254 if trailing_dot else 253) + + +def check_bidi(label: str, check_ltr: bool = False) -> bool: + """Validate the Bidi Rule from :rfc:`5893` for a single label. + + The Bidi Rule constrains how bidirectional characters (Hebrew, Arabic, + etc.) may appear within a label. By default the check is only applied + when the label contains at least one right-to-left character (Unicode + bidirectional categories ``R``, ``AL``, or ``AN``); set ``check_ltr`` + to ``True`` to apply it to LTR-only labels as well. + + :param label: The label to validate, as a Unicode string. + :param check_ltr: If ``True``, apply the rules even when the label + contains no RTL characters. + :returns: ``True`` if the label satisfies the Bidi Rule. + :raises IDNABidiError: If any of Bidi Rule conditions 1-6 are violated, + or if the directional category of a codepoint cannot be determined. + """ + if len(label) > _max_input_length: + raise IDNAError("Label too long", code="input_too_long") + # Bidi rules should only be applied if string contains RTL characters + bidi_label = False + for idx, cp in enumerate(label, 1): + direction = unicodedata.bidirectional(cp) + if direction == "": + # String likely comes from a newer version of Unicode + raise IDNABidiError( + f"Unknown directionality in label {label!r} at position {idx}", + code="bidi_unknown_direction", + text=label, + codepoint=ord(cp), + position=idx, + ) + if direction in _bidi_rtl_categories: + bidi_label = True + if not bidi_label and not check_ltr: + return True + + # Bidi rule 1 + direction = unicodedata.bidirectional(label[0]) + if direction in _bidi_rtl_first: + rtl = True + elif direction == "L": + rtl = False + else: + raise IDNABidiError( + f"First codepoint in label {label!r} must be directionality L, R or AL", + code="bidi_rule_1", + text=label, + codepoint=ord(label[0]), + position=1, + ) + + valid_ending = False + ending_idx = 1 + number_type: str | None = None + for idx, cp in enumerate(label, 1): + direction = unicodedata.bidirectional(cp) + + if rtl: + # Bidi rule 2 + if direction not in _bidi_rtl_allowed: + raise IDNABidiError( + f"Invalid direction for codepoint at position {idx} in a right-to-left label", + code="bidi_rule_2", + text=label, + codepoint=ord(cp), + position=idx, + ) + # Bidi rule 3 + if direction in _bidi_rtl_valid_ending: + valid_ending = True + ending_idx = idx + elif direction != "NSM": + valid_ending = False + ending_idx = idx + # Bidi rule 4 + if direction in _bidi_rtl_numeric: + if not number_type: + number_type = direction + elif number_type != direction: + raise IDNABidiError( + "Can not mix numeral types in a right-to-left label", + code="bidi_rule_4", + text=label, + codepoint=ord(cp), + position=idx, + ) + else: + # Bidi rule 5 + if direction not in _bidi_ltr_allowed: + raise IDNABidiError( + f"Invalid direction for codepoint at position {idx} in a left-to-right label", + code="bidi_rule_5", + text=label, + codepoint=ord(cp), + position=idx, + ) + # Bidi rule 6 + if direction in _bidi_ltr_valid_ending: + valid_ending = True + ending_idx = idx + elif direction != "NSM": + valid_ending = False + ending_idx = idx + + if not valid_ending: + # Rules 3 and 6 concern the last character that is not a + # non-spacing mark, which is what ``ending_idx`` tracks. + raise IDNABidiError( + "Label ends with illegal codepoint directionality", + code="bidi_rule_3" if rtl else "bidi_rule_6", + text=label, + codepoint=ord(label[ending_idx - 1]), + position=ending_idx, + ) + + return True + + +def check_initial_combiner(label: str) -> bool: + """Reject labels that begin with a combining mark. + + Per :rfc:`5891` §4.2.3.2 a label must not start with a character of + Unicode general category ``M`` (Mark). + + :param label: The label to check. + :returns: ``True`` if the first character is not a combining mark. + :raises IDNAError: If the label begins with a combining character. + """ + if label and unicodedata.category(label[0])[0] == "M": + raise IDNAError( + "Label begins with an illegal combining character", + code="leading_combiner", + text=label, + codepoint=ord(label[0]), + position=1, + ) + return True + + +def check_hyphen_ok(label: str) -> bool: + """Validate the hyphen restrictions for a label. + + Per :rfc:`5891` §4.2.3.1 a label must not start or end with a hyphen + (``U+002D``), and must not have hyphens in both the third and fourth + positions (the prefix reserved for A-labels). + + :param label: The label to check. + :returns: ``True`` if the hyphen restrictions are satisfied. + :raises IDNAError: If any of the hyphen restrictions are violated. + """ + if label[2:4] == "--": + raise IDNAError("Label has disallowed hyphens in 3rd and 4th position", code="hyphen_3_4") + if label.startswith("-") or label.endswith("-"): + raise IDNAError("Label must not start or end with a hyphen", code="hyphen_start_end") + return True + + +def check_nfc(label: str) -> None: + """Require that a label is in Unicode Normalization Form C. + + :param label: The label to check. + :raises IDNAError: If ``label`` differs from its NFC normalisation. + """ + if len(label) > _max_input_length: + raise IDNAError("Label too long", code="input_too_long") + if unicodedata.normalize("NFC", label) != label: + raise IDNAError("Label must be in Normalization Form C", code="not_nfc") + + +def valid_contextj(label: str, pos: int) -> bool: + """Validate the CONTEXTJ rules from :rfc:`5892` Appendix A. + + These rules govern the contextual use of the joiner codepoints + ``U+200C`` (ZERO WIDTH NON-JOINER, Appendix A.1) and ``U+200D`` + (ZERO WIDTH JOINER, Appendix A.2) within a label. + + :param label: The label containing the codepoint. + :param pos: Index of the joiner codepoint within ``label``. + :returns: ``True`` if the codepoint at ``pos`` satisfies its CONTEXTJ + rule, ``False`` otherwise (including when the codepoint at + ``pos`` is not a recognised joiner). + :raises ValueError: If an adjacent codepoint has no Unicode name when + determining its combining class. + :raises IDNAError: If ``label`` exceeds the defensive input length limit. + """ + if len(label) > _max_input_length: + raise IDNAError("Label too long", code="input_too_long") + cp_value = ord(label[pos]) + + if cp_value == 0x200C: + if pos > 0 and _combining_class(ord(label[pos - 1])) == _virama_combining_class: + return True + + ok = False + for i in range(pos - 1, -1, -1): + joining_type = _joining_type(ord(label[i])) + if joining_type == "T": + continue + if joining_type in _bidi_joiner_l_or_d: + ok = True + break + break + + if not ok: + return False + + ok = False + for i in range(pos + 1, len(label)): + joining_type = _joining_type(ord(label[i])) + if joining_type == "T": + continue + if joining_type in _bidi_joiner_r_or_d: + ok = True + break + break + return ok + + if cp_value == 0x200D: + return pos > 0 and _combining_class(ord(label[pos - 1])) == _virama_combining_class + + return False + + +def valid_contexto(label: str, pos: int, exception: bool = False) -> bool: + """Validate the CONTEXTO rules from :rfc:`5892` Appendix A. + + Covers the contextual rules for codepoints such as MIDDLE DOT + (``U+00B7``), Greek lower numeral sign, Hebrew punctuation, Katakana + middle dot, and the Arabic-Indic / Extended Arabic-Indic digit ranges. + + :param label: The label containing the codepoint. + :param pos: Index of the codepoint within ``label``. + :param exception: Reserved for forward compatibility; currently unused. + :returns: ``True`` if the codepoint at ``pos`` satisfies its CONTEXTO + rule, ``False`` otherwise (including when the codepoint is not a + recognised CONTEXTO codepoint). + :raises IDNAError: If ``label`` exceeds the defensive input length limit. + """ + if len(label) > _max_input_length: + raise IDNAError("Label too long", code="input_too_long") + cp_value = ord(label[pos]) + + if cp_value == 0x00B7: + return 0 < pos < len(label) - 1 and ord(label[pos - 1]) == 0x006C and ord(label[pos + 1]) == 0x006C + + if cp_value == 0x0375: + if pos < len(label) - 1 and len(label) > 1: + return _is_script(label[pos + 1], "Greek") + return False + + if cp_value in {0x05F3, 0x05F4}: + if pos > 0: + return _is_script(label[pos - 1], "Hebrew") + return False + + if cp_value == 0x30FB: + for cp in label: + if cp == "\u30fb": + continue + if _is_script(cp, "Hiragana") or _is_script(cp, "Katakana") or _is_script(cp, "Han"): + return True + return False + + if 0x660 <= cp_value <= 0x669: + return not any(0x6F0 <= ord(cp) <= 0x06F9 for cp in label) + + if 0x6F0 <= cp_value <= 0x6F9: + return not any(0x660 <= ord(cp) <= 0x0669 for cp in label) + + return False + + +def check_label(label: str | bytes | bytearray) -> None: + """Run the full set of IDNA 2008 validity checks on a single label. + + Applies, in order: NFC normalisation (:func:`check_nfc`), hyphen + restrictions (:func:`check_hyphen_ok`), the no-leading-combiner rule + (:func:`check_initial_combiner`), per-codepoint validity (PVALID, + CONTEXTJ, CONTEXTO classes from :rfc:`5892`), and the Bidi Rule + (:func:`check_bidi`). + + :param label: The label to validate. ``bytes`` or ``bytearray`` input + is decoded as UTF-8 first. + :raises IDNAError: If the label is empty or fails a structural rule. + :raises InvalidCodepoint: If the label contains a DISALLOWED or + UNASSIGNED codepoint. + :raises InvalidCodepointContext: If a CONTEXTJ or CONTEXTO codepoint + is not valid in its context. + :raises IDNABidiError: If the Bidi Rule is violated. + """ + if len(label) > _max_input_length: + raise IDNAError("Label too long", code="input_too_long") + if isinstance(label, (bytes, bytearray)): + try: + label = label.decode("utf-8") + except UnicodeDecodeError as err: + raise IDNAError("Invalid UTF-8 in label", code="invalid_utf8") from err + if len(label) == 0: + raise IDNAError("Empty Label", code="empty_label") + + # Check against the domain length rather than the label length to + # support some UTS #46 use cases, while still bounding the work done + # by the label contextual rules below. + if not valid_string_length(label, trailing_dot=True): + raise IDNAError("Label too long", code="label_too_long") + + check_nfc(label) + check_hyphen_ok(label) + check_initial_combiner(label) + + for pos, cp in enumerate(label): + cp_value = ord(cp) + if intranges_contain(cp_value, idnadata.codepoint_classes["PVALID"]): + continue + if intranges_contain(cp_value, idnadata.codepoint_classes["CONTEXTJ"]): + try: + contextj_ok = valid_contextj(label, pos) + except ValueError as err: + raise IDNAError( + f"Unknown codepoint adjacent to joiner {_unot(cp_value)} at position {pos + 1} in {label!r}", + code="unknown_codepoint", + text=label, + codepoint=cp_value, + position=pos + 1, + ) from err + if not contextj_ok: + raise InvalidCodepointContext( + f"Joiner {_unot(cp_value)} not allowed at position {pos + 1} in {label!r}", + code="contextj", + text=label, + codepoint=cp_value, + position=pos + 1, + ) + elif intranges_contain(cp_value, idnadata.codepoint_classes["CONTEXTO"]): + if not valid_contexto(label, pos): + raise InvalidCodepointContext( + f"Codepoint {_unot(cp_value)} not allowed at position {pos + 1} in {label!r}", + code="contexto", + text=label, + codepoint=cp_value, + position=pos + 1, + ) + else: + raise InvalidCodepoint( + f"Codepoint {_unot(cp_value)} at position {pos + 1} of {label!r} not allowed", + code="disallowed_codepoint", + text=label, + codepoint=cp_value, + position=pos + 1, + ) + + check_bidi(label) + + +def alabel(label: str) -> bytes: + """Convert a single U-label into its A-label form. + + The result is the ASCII-Compatible Encoding (ACE) form per :rfc:`5891` + §4: the label is validated, Punycode-encoded, and prefixed with + ``xn--``. Pure ASCII labels that are already valid IDNA labels are + returned unchanged (as :class:`bytes`). + + :param label: The label to convert, as a Unicode string. + :returns: The A-label as ASCII-encoded :class:`bytes`. + :raises IDNAError: If the label is invalid or the resulting A-label + exceeds 63 octets. + """ + if len(label) > _max_input_length: + raise IDNAError("Label too long", code="input_too_long") + try: + label_bytes = label.encode("ascii") + except UnicodeEncodeError: + pass + else: + ulabel(label_bytes) + if not valid_label_length(label_bytes): + raise IDNAError("Label too long", code="label_too_long") + return label_bytes + + check_label(label) + label_bytes = _alabel_prefix + _punycode(label) + + if not valid_label_length(label_bytes): + raise IDNAError("Label too long", code="label_too_long") + + return label_bytes + + +def ulabel(label: str | bytes | bytearray) -> str: + """Convert a single A-label into its U-label form. + + Performs the inverse of :func:`alabel`: an ``xn--``-prefixed label is + Punycode-decoded and validated, and is rejected unless it is the + canonical A-label for the decoded U-label (:rfc:`5891` §5.3). Labels + that are already Unicode (or plain ASCII without the ACE prefix) are + validated and returned as a Unicode string. + + :param label: The label to convert. ``bytes`` or ``bytearray`` input + is treated as ASCII. + :returns: The U-label as a Unicode string. + :raises IDNAError: If the label is malformed or fails validation. + """ + if len(label) > _max_input_length: + raise IDNAError("Label too long", code="input_too_long") + if not isinstance(label, (bytes, bytearray)): + try: + label_bytes = label.encode("ascii") + except UnicodeEncodeError: + check_label(label) + return label + else: + label_bytes = bytes(label) + if not label_bytes.isascii(): + raise IDNAError("Invalid ASCII in A-label", code="invalid_ascii") + + label_bytes = label_bytes.lower() + if label_bytes.startswith(_alabel_prefix): + label_bytes = label_bytes[len(_alabel_prefix) :] + if not label_bytes: + raise IDNAError("Malformed A-label, no Punycode eligible content found", code="invalid_alabel") + if label_bytes.endswith(b"-"): + raise IDNAError("A-label must not end with a hyphen", code="invalid_alabel") + else: + check_label(label_bytes) + return label_bytes.decode("ascii") + + try: + label = label_bytes.decode("punycode") + except UnicodeError as err: + raise IDNAError("Invalid A-label", code="invalid_alabel") from err + # RFC 5891 §5.3: the label is rejected unless re-encoding the decoded + # form reproduces the (lowercased) input. This catches "fake A-labels" + # (RFC 5890 §2.3.2.1) such as ``xn---bbk``, a non-canonical Punycode + # spelling of ``xn--bbk`` that would otherwise decode to the same + # U-label and so display identically to a different wire-format name. + if _punycode(label) != label_bytes: + raise IDNAError("A-label is not the canonical Punycode encoding of its U-label", code="non_canonical_alabel") + check_label(label) + return label + + +def _check_std3(text: str, domain: str, offset: int) -> None: + """Raise if ``text``, a slice of ``domain`` starting at ``offset`` that + UTS #46 mapping left unchanged, contains an ASCII character disallowed + under ``UseSTD3ASCIIRules``.""" + match = _std3_disallowed_re.search(text) + if match: + codepoint = ord(match.group()) + position = offset + match.start() + 1 + raise InvalidCodepoint( + f"Codepoint {_unot(codepoint)} not allowed at position {position} in {domain!r}", + code="uts46_std3", + text=domain, + codepoint=codepoint, + position=position, + ) + + +def _warn_transitional() -> None: + warnings.warn( + "Transitional processing is deprecated in UTS #46 and has no effect. " + "The transitional argument will be removed in a future version.", + DeprecationWarning, + stacklevel=3, + ) + + +def uts46_remap(domain: str, std3_rules: bool = True, transitional: bool = False) -> str: + """Apply the UTS #46 character mapping to a domain string. + + Implements the mapping table from `UTS #46 §4 + `_: each character is kept, + replaced, or rejected based on its status (``V``, ``M``, ``D``, + ``I``, ``X``). The result is returned in Normalisation Form C. + + :param domain: The full domain name to remap. + :param std3_rules: If ``True``, apply UTS #46's ``UseSTD3ASCIIRules``: + after mapping, any ASCII character other than a lowercase letter, + digit, hyphen or the label separator ``.`` is rejected, whether it + appeared in the input or was produced by a mapping (e.g. U+FF01 + FULLWIDTH EXCLAMATION MARK maps to ``!``). If ``False``, such + characters are passed through. + :param transitional: Deprecated and ignored. UTS #46 deprecated + transitional processing in Unicode 15.1 and deviation (status + ``D``) codepoints are now always kept, so this has no effect + beyond emitting a :class:`DeprecationWarning`. It will be removed + in a future version. + :returns: The remapped domain, in Normalisation Form C. + :raises InvalidCodepoint: If the domain contains a disallowed + codepoint under the chosen rules. + :raises IDNAError: If ``domain`` exceeds the defensive input length limit. + """ + if transitional: + _warn_transitional() + if len(domain) > _max_input_length: + raise IDNAError("Domain too long", code="input_too_long") + if domain.isascii(): + # The only ASCII mapping in UTS #46 is upper- to lowercase, and + # ASCII is invariant under NFC, so lowercasing is the whole job. + result = domain.lower() + if std3_rules: + _check_std3(result, domain, 0) + return result + + from .uts46data import uts46_replacements, uts46_starts, uts46_statuses + + # ``start`` marks the run of unchanged input not yet copied; a run is + # only sliced out when a character must be replaced or dropped, so the + # common no-change case makes no copy. STD3 is checked per output piece + # to report a violation at its input position. + output: list[str] = [] + start = 0 + for pos, char in enumerate(domain): + code_point = ord(char) + i = code_point if code_point < 256 else bisect.bisect_right(uts46_starts, code_point) - 1 + status = uts46_statuses[i] + # UTS #46 §4: V valid, D deviation (kept), M mapped, I ignored, + # anything else disallowed. + if status == _STATUS_VALID: + continue + if status == _STATUS_MAPPED: + replacement = uts46_replacements[i] + elif status == _STATUS_DEVIATION: + continue + elif status == _STATUS_IGNORED: + replacement = None + else: + raise InvalidCodepoint( + f"Codepoint {_unot(code_point)} not allowed at position {pos + 1} in {domain!r}", + code="uts46_disallowed", + text=domain, + codepoint=code_point, + position=pos + 1, + ) + if start < pos: + run = domain[start:pos] + if std3_rules: + _check_std3(run, domain, start) + output.append(run) + if replacement: + if std3_rules and _std3_disallowed_re.search(replacement): + raise InvalidCodepoint( + f"Codepoint {_unot(code_point)} not allowed at position {pos + 1} in {domain!r}", + code="uts46_std3", + text=domain, + codepoint=code_point, + position=pos + 1, + ) + output.append(replacement) + start = pos + 1 + + if start == 0: + if std3_rules: + _check_std3(domain, domain, 0) + return unicodedata.normalize("NFC", domain) + tail = domain[start:] + if std3_rules: + _check_std3(tail, domain, start) + output.append(tail) + return unicodedata.normalize("NFC", "".join(output)) + + +def encode( + s: str | bytes | bytearray, + strict: bool = False, + uts46: bool = False, + std3_rules: bool = False, + transitional: bool = False, +) -> bytes: + """Encode a Unicode domain name into its ASCII (A-label) form. + + Splits the input on label separators (only ``U+002E`` if ``strict`` is + set; otherwise also IDEOGRAPHIC FULL STOP ``U+3002``, FULLWIDTH FULL + STOP ``U+FF0E``, and HALFWIDTH IDEOGRAPHIC FULL STOP ``U+FF61``), + encodes each label with :func:`alabel`, and rejoins them with ``.``. + Optionally pre-processes the input through :func:`uts46_remap`. + + :param s: The domain name to encode. + :param strict: If ``True``, only ``U+002E`` is recognised as a label + separator. + :param uts46: If ``True``, apply UTS #46 mapping before encoding. + :param std3_rules: Forwarded to :func:`uts46_remap` when ``uts46`` is + ``True``. + :param transitional: Deprecated and ignored (see :func:`uts46_remap`): + emits a :class:`DeprecationWarning` and will be removed in a + future version. + :returns: The encoded domain as ASCII :class:`bytes`. + :raises IDNAError: If the domain is empty, contains an invalid label, + or exceeds the maximum domain length. + """ + if transitional: + _warn_transitional() + if not isinstance(s, str): + try: + s = str(s, "ascii") + except (UnicodeDecodeError, TypeError) as err: + raise IDNAError( + "should pass a unicode string to the function rather than a byte string.", code="invalid_ascii" + ) from err + if len(s) > _max_input_length: + raise IDNAError("Domain too long", code="input_too_long") + if uts46: + s = uts46_remap(s, std3_rules) + + if not valid_string_length(s, trailing_dot=True): + raise IDNAError("Domain too long", code="domain_too_long") + + trailing_dot = False + result = [] + labels = s.split(".") if strict else _unicode_dots_re.split(s) + if not labels or labels == [""]: + raise IDNAError("Empty domain", code="empty_domain") + if labels[-1] == "": + del labels[-1] + trailing_dot = True + for label in labels: + s = alabel(label) + if s: + result.append(s) + else: + raise IDNAError("Empty label", code="empty_label") + if trailing_dot: + result.append(b"") + s = b".".join(result) + if not valid_string_length(s, trailing_dot): + raise IDNAError("Domain too long", code="domain_too_long") + return s + + +def decode( + s: str | bytes | bytearray, + strict: bool = False, + uts46: bool = False, + std3_rules: bool = False, + display: bool = False, +) -> str: + """Decode an A-label-encoded domain name back to Unicode. + + Splits the input on label separators (see :func:`encode` for the + rules), decodes each label with :func:`ulabel`, and rejoins them + with ``.``. Optionally pre-processes the input through + :func:`uts46_remap`. + + :param s: The domain name to decode. + :param strict: If ``True``, only ``U+002E`` is recognised as a label + separator. + :param uts46: If ``True``, apply UTS #46 mapping before decoding. + :param std3_rules: Forwarded to :func:`uts46_remap` when ``uts46`` is + ``True``. + :param display: If ``True``, any ``xn--`` label that fails IDNA + validation is passed through unchanged (lowercased) rather than + aborting the whole call. Intended for "decode for display" + consumers (e.g. URL libraries, HTTP clients) that want to show + the user the label as it appears on the wire when it cannot be + rendered as Unicode. Matches the per-label recovery prescribed + by UTS #46 §4 and the WHATWG URL "domain to Unicode" algorithm. + :returns: The decoded domain as a Unicode string. + :raises IDNAError: If the input is not valid ASCII, contains an + invalid label, or is empty. + """ + if not isinstance(s, str): + try: + s = str(s, "ascii") + except (UnicodeDecodeError, TypeError) as err: + raise IDNAError("Invalid ASCII in A-label", code="invalid_ascii") from err + if len(s) > _max_input_length: + raise IDNAError("Domain too long", code="input_too_long") + if uts46: + s = uts46_remap(s, std3_rules, False) + if not valid_string_length(s, trailing_dot=True): + raise IDNAError("Domain too long", code="domain_too_long") + trailing_dot = False + result = [] + labels = s.split(".") if strict else _unicode_dots_re.split(s) + if not labels or labels == [""]: + raise IDNAError("Empty domain", code="empty_domain") + if not labels[-1]: + del labels[-1] + trailing_dot = True + for label in labels: + try: + u = ulabel(label) + except IDNAError: + if display and label[:4].lower() == "xn--": + u = label.lower() + else: + raise + if u: + result.append(u) + else: + raise IDNAError("Empty label", code="empty_label") + if trailing_dot: + result.append("") + return ".".join(result) diff --git a/py_modules/idna/idnadata.py b/py_modules/idna/idnadata.py new file mode 100644 index 0000000..f2ab388 --- /dev/null +++ b/py_modules/idna/idnadata.py @@ -0,0 +1,1897 @@ +# This file is automatically generated by tools/idna-data + +__version__ = "17.0.0" + +scripts = { + "Greek": ( + 0x37000000374, + 0x37500000378, + 0x37A0000037E, + 0x37F00000380, + 0x38400000385, + 0x38600000387, + 0x3880000038B, + 0x38C0000038D, + 0x38E000003A2, + 0x3A3000003E2, + 0x3F000000400, + 0x1D2600001D2B, + 0x1D5D00001D62, + 0x1D6600001D6B, + 0x1DBF00001DC0, + 0x1F0000001F16, + 0x1F1800001F1E, + 0x1F2000001F46, + 0x1F4800001F4E, + 0x1F5000001F58, + 0x1F5900001F5A, + 0x1F5B00001F5C, + 0x1F5D00001F5E, + 0x1F5F00001F7E, + 0x1F8000001FB5, + 0x1FB600001FC5, + 0x1FC600001FD4, + 0x1FD600001FDC, + 0x1FDD00001FF0, + 0x1FF200001FF5, + 0x1FF600001FFF, + 0x212600002127, + 0xAB650000AB66, + 0x101400001018F, + 0x101A0000101A1, + 0x1D2000001D246, + ), + "Han": ( + 0x2E8000002E9A, + 0x2E9B00002EF4, + 0x2F0000002FD6, + 0x300500003006, + 0x300700003008, + 0x30210000302A, + 0x30380000303C, + 0x340000004DC0, + 0x4E000000A000, + 0xF9000000FA6E, + 0xFA700000FADA, + 0x16FE200016FE4, + 0x16FF000016FF7, + 0x200000002A6E0, + 0x2A7000002B81E, + 0x2B8200002CEAE, + 0x2CEB00002EBE1, + 0x2EBF00002EE5E, + 0x2F8000002FA1E, + 0x300000003134B, + 0x313500003347A, + ), + "Hebrew": ( + 0x591000005C8, + 0x5D0000005EB, + 0x5EF000005F5, + 0xFB1D0000FB37, + 0xFB380000FB3D, + 0xFB3E0000FB3F, + 0xFB400000FB42, + 0xFB430000FB45, + 0xFB460000FB50, + ), + "Hiragana": ( + 0x304100003097, + 0x309D000030A0, + 0x1B0010001B120, + 0x1B1320001B133, + 0x1B1500001B153, + 0x1F2000001F201, + ), + "Katakana": ( + 0x30A1000030FB, + 0x30FD00003100, + 0x31F000003200, + 0x32D0000032FF, + 0x330000003358, + 0xFF660000FF70, + 0xFF710000FF9E, + 0x1AFF00001AFF4, + 0x1AFF50001AFFC, + 0x1AFFD0001AFFF, + 0x1B0000001B001, + 0x1B1200001B123, + 0x1B1550001B156, + 0x1B1640001B168, + ), +} + + +joining_types = { + "C": ( + 0x64000000641, + 0x7FA000007FB, + 0x88300000886, + 0x180A0000180B, + 0x200D0000200E, + ), + "D": ( + 0x62000000621, + 0x62600000627, + 0x62800000629, + 0x62A0000062F, + 0x63300000640, + 0x64100000648, + 0x6490000064B, + 0x66E00000670, + 0x67800000688, + 0x69A000006C0, + 0x6C1000006C3, + 0x6CC000006CD, + 0x6CE000006CF, + 0x6D0000006D2, + 0x6FA000006FD, + 0x6FF00000700, + 0x71200000715, + 0x71A0000071E, + 0x71F00000728, + 0x7290000072A, + 0x72B0000072C, + 0x72D0000072F, + 0x74E00000759, + 0x75C0000076B, + 0x76D00000771, + 0x77200000773, + 0x77500000778, + 0x77A00000780, + 0x7CA000007EB, + 0x84100000846, + 0x84800000849, + 0x84A00000854, + 0x85500000856, + 0x86000000861, + 0x86200000866, + 0x86800000869, + 0x88600000887, + 0x8890000088E, + 0x88F00000890, + 0x8A0000008AA, + 0x8AF000008B1, + 0x8B3000008B9, + 0x8BA000008C9, + 0x180700001808, + 0x182000001879, + 0x1887000018A9, + 0x18AA000018AB, + 0xA8400000A872, + 0x10AC000010AC5, + 0x10AD300010AD7, + 0x10AD800010ADD, + 0x10ADE00010AE1, + 0x10AEB00010AEF, + 0x10B8000010B81, + 0x10B8200010B83, + 0x10B8600010B89, + 0x10B8A00010B8C, + 0x10B8D00010B8E, + 0x10B9000010B91, + 0x10BAD00010BAF, + 0x10D0100010D22, + 0x10D2300010D24, + 0x10EC300010EC5, + 0x10EC600010EC8, + 0x10F3000010F33, + 0x10F3400010F45, + 0x10F5100010F54, + 0x10F7000010F74, + 0x10F7600010F82, + 0x10FB000010FB1, + 0x10FB200010FB4, + 0x10FB800010FB9, + 0x10FBB00010FBD, + 0x10FBE00010FC0, + 0x10FC100010FC2, + 0x10FC400010FC5, + 0x10FCA00010FCB, + 0x1E9000001E944, + ), + "L": ( + 0xA8720000A873, + 0x10ACD00010ACE, + 0x10AD700010AD8, + 0x10D0000010D01, + 0x10FCB00010FCC, + ), + "R": ( + 0x62200000626, + 0x62700000628, + 0x6290000062A, + 0x62F00000633, + 0x64800000649, + 0x67100000674, + 0x67500000678, + 0x6880000069A, + 0x6C0000006C1, + 0x6C3000006CC, + 0x6CD000006CE, + 0x6CF000006D0, + 0x6D2000006D4, + 0x6D5000006D6, + 0x6EE000006F0, + 0x71000000711, + 0x7150000071A, + 0x71E0000071F, + 0x72800000729, + 0x72A0000072B, + 0x72C0000072D, + 0x72F00000730, + 0x74D0000074E, + 0x7590000075C, + 0x76B0000076D, + 0x77100000772, + 0x77300000775, + 0x7780000077A, + 0x84000000841, + 0x84600000848, + 0x8490000084A, + 0x85400000855, + 0x85600000859, + 0x86700000868, + 0x8690000086B, + 0x87000000883, + 0x88E0000088F, + 0x8AA000008AD, + 0x8AE000008AF, + 0x8B1000008B3, + 0x8B9000008BA, + 0x10AC500010AC6, + 0x10AC700010AC8, + 0x10AC900010ACB, + 0x10ACE00010AD3, + 0x10ADD00010ADE, + 0x10AE100010AE2, + 0x10AE400010AE5, + 0x10AEF00010AF0, + 0x10B8100010B82, + 0x10B8300010B86, + 0x10B8900010B8A, + 0x10B8C00010B8D, + 0x10B8E00010B90, + 0x10B9100010B92, + 0x10BA900010BAD, + 0x10D2200010D23, + 0x10EC200010EC3, + 0x10F3300010F34, + 0x10F5400010F55, + 0x10F7400010F76, + 0x10FB400010FB7, + 0x10FB900010FBB, + 0x10FBD00010FBE, + 0x10FC200010FC4, + 0x10FC900010FCA, + ), + "T": ( + 0xAD000000AE, + 0x30000000370, + 0x4830000048A, + 0x591000005BE, + 0x5BF000005C0, + 0x5C1000005C3, + 0x5C4000005C6, + 0x5C7000005C8, + 0x6100000061B, + 0x61C0000061D, + 0x64B00000660, + 0x67000000671, + 0x6D6000006DD, + 0x6DF000006E5, + 0x6E7000006E9, + 0x6EA000006EE, + 0x70F00000710, + 0x71100000712, + 0x7300000074B, + 0x7A6000007B1, + 0x7EB000007F4, + 0x7FD000007FE, + 0x8160000081A, + 0x81B00000824, + 0x82500000828, + 0x8290000082E, + 0x8590000085C, + 0x897000008A0, + 0x8CA000008E2, + 0x8E300000903, + 0x93A0000093B, + 0x93C0000093D, + 0x94100000949, + 0x94D0000094E, + 0x95100000958, + 0x96200000964, + 0x98100000982, + 0x9BC000009BD, + 0x9C1000009C5, + 0x9CD000009CE, + 0x9E2000009E4, + 0x9FE000009FF, + 0xA0100000A03, + 0xA3C00000A3D, + 0xA4100000A43, + 0xA4700000A49, + 0xA4B00000A4E, + 0xA5100000A52, + 0xA7000000A72, + 0xA7500000A76, + 0xA8100000A83, + 0xABC00000ABD, + 0xAC100000AC6, + 0xAC700000AC9, + 0xACD00000ACE, + 0xAE200000AE4, + 0xAFA00000B00, + 0xB0100000B02, + 0xB3C00000B3D, + 0xB3F00000B40, + 0xB4100000B45, + 0xB4D00000B4E, + 0xB5500000B57, + 0xB6200000B64, + 0xB8200000B83, + 0xBC000000BC1, + 0xBCD00000BCE, + 0xC0000000C01, + 0xC0400000C05, + 0xC3C00000C3D, + 0xC3E00000C41, + 0xC4600000C49, + 0xC4A00000C4E, + 0xC5500000C57, + 0xC6200000C64, + 0xC8100000C82, + 0xCBC00000CBD, + 0xCBF00000CC0, + 0xCC600000CC7, + 0xCCC00000CCE, + 0xCE200000CE4, + 0xD0000000D02, + 0xD3B00000D3D, + 0xD4100000D45, + 0xD4D00000D4E, + 0xD6200000D64, + 0xD8100000D82, + 0xDCA00000DCB, + 0xDD200000DD5, + 0xDD600000DD7, + 0xE3100000E32, + 0xE3400000E3B, + 0xE4700000E4F, + 0xEB100000EB2, + 0xEB400000EBD, + 0xEC800000ECF, + 0xF1800000F1A, + 0xF3500000F36, + 0xF3700000F38, + 0xF3900000F3A, + 0xF7100000F7F, + 0xF8000000F85, + 0xF8600000F88, + 0xF8D00000F98, + 0xF9900000FBD, + 0xFC600000FC7, + 0x102D00001031, + 0x103200001038, + 0x10390000103B, + 0x103D0000103F, + 0x10580000105A, + 0x105E00001061, + 0x107100001075, + 0x108200001083, + 0x108500001087, + 0x108D0000108E, + 0x109D0000109E, + 0x135D00001360, + 0x171200001715, + 0x173200001734, + 0x175200001754, + 0x177200001774, + 0x17B4000017B6, + 0x17B7000017BE, + 0x17C6000017C7, + 0x17C9000017D4, + 0x17DD000017DE, + 0x180B0000180E, + 0x180F00001810, + 0x188500001887, + 0x18A9000018AA, + 0x192000001923, + 0x192700001929, + 0x193200001933, + 0x19390000193C, + 0x1A1700001A19, + 0x1A1B00001A1C, + 0x1A5600001A57, + 0x1A5800001A5F, + 0x1A6000001A61, + 0x1A6200001A63, + 0x1A6500001A6D, + 0x1A7300001A7D, + 0x1A7F00001A80, + 0x1AB000001ADE, + 0x1AE000001AEC, + 0x1B0000001B04, + 0x1B3400001B35, + 0x1B3600001B3B, + 0x1B3C00001B3D, + 0x1B4200001B43, + 0x1B6B00001B74, + 0x1B8000001B82, + 0x1BA200001BA6, + 0x1BA800001BAA, + 0x1BAB00001BAE, + 0x1BE600001BE7, + 0x1BE800001BEA, + 0x1BED00001BEE, + 0x1BEF00001BF2, + 0x1C2C00001C34, + 0x1C3600001C38, + 0x1CD000001CD3, + 0x1CD400001CE1, + 0x1CE200001CE9, + 0x1CED00001CEE, + 0x1CF400001CF5, + 0x1CF800001CFA, + 0x1DC000001E00, + 0x200B0000200C, + 0x200E00002010, + 0x202A0000202F, + 0x206000002065, + 0x206A00002070, + 0x20D0000020F1, + 0x2CEF00002CF2, + 0x2D7F00002D80, + 0x2DE000002E00, + 0x302A0000302E, + 0x30990000309B, + 0xA66F0000A673, + 0xA6740000A67E, + 0xA69E0000A6A0, + 0xA6F00000A6F2, + 0xA8020000A803, + 0xA8060000A807, + 0xA80B0000A80C, + 0xA8250000A827, + 0xA82C0000A82D, + 0xA8C40000A8C6, + 0xA8E00000A8F2, + 0xA8FF0000A900, + 0xA9260000A92E, + 0xA9470000A952, + 0xA9800000A983, + 0xA9B30000A9B4, + 0xA9B60000A9BA, + 0xA9BC0000A9BE, + 0xA9E50000A9E6, + 0xAA290000AA2F, + 0xAA310000AA33, + 0xAA350000AA37, + 0xAA430000AA44, + 0xAA4C0000AA4D, + 0xAA7C0000AA7D, + 0xAAB00000AAB1, + 0xAAB20000AAB5, + 0xAAB70000AAB9, + 0xAABE0000AAC0, + 0xAAC10000AAC2, + 0xAAEC0000AAEE, + 0xAAF60000AAF7, + 0xABE50000ABE6, + 0xABE80000ABE9, + 0xABED0000ABEE, + 0xFB1E0000FB1F, + 0xFE000000FE10, + 0xFE200000FE30, + 0xFEFF0000FF00, + 0xFFF90000FFFC, + 0x101FD000101FE, + 0x102E0000102E1, + 0x103760001037B, + 0x10A0100010A04, + 0x10A0500010A07, + 0x10A0C00010A10, + 0x10A3800010A3B, + 0x10A3F00010A40, + 0x10AE500010AE7, + 0x10D2400010D28, + 0x10D6900010D6E, + 0x10EAB00010EAD, + 0x10EFA00010F00, + 0x10F4600010F51, + 0x10F8200010F86, + 0x1100100011002, + 0x1103800011047, + 0x1107000011071, + 0x1107300011075, + 0x1107F00011082, + 0x110B3000110B7, + 0x110B9000110BB, + 0x110C2000110C3, + 0x1110000011103, + 0x111270001112C, + 0x1112D00011135, + 0x1117300011174, + 0x1118000011182, + 0x111B6000111BF, + 0x111C9000111CD, + 0x111CF000111D0, + 0x1122F00011232, + 0x1123400011235, + 0x1123600011238, + 0x1123E0001123F, + 0x1124100011242, + 0x112DF000112E0, + 0x112E3000112EB, + 0x1130000011302, + 0x1133B0001133D, + 0x1134000011341, + 0x113660001136D, + 0x1137000011375, + 0x113BB000113C1, + 0x113CE000113CF, + 0x113D0000113D1, + 0x113D2000113D3, + 0x113E1000113E3, + 0x1143800011440, + 0x1144200011445, + 0x1144600011447, + 0x1145E0001145F, + 0x114B3000114B9, + 0x114BA000114BB, + 0x114BF000114C1, + 0x114C2000114C4, + 0x115B2000115B6, + 0x115BC000115BE, + 0x115BF000115C1, + 0x115DC000115DE, + 0x116330001163B, + 0x1163D0001163E, + 0x1163F00011641, + 0x116AB000116AC, + 0x116AD000116AE, + 0x116B0000116B6, + 0x116B7000116B8, + 0x1171D0001171E, + 0x1171F00011720, + 0x1172200011726, + 0x117270001172C, + 0x1182F00011838, + 0x118390001183B, + 0x1193B0001193D, + 0x1193E0001193F, + 0x1194300011944, + 0x119D4000119D8, + 0x119DA000119DC, + 0x119E0000119E1, + 0x11A0100011A0B, + 0x11A3300011A39, + 0x11A3B00011A3F, + 0x11A4700011A48, + 0x11A5100011A57, + 0x11A5900011A5C, + 0x11A8A00011A97, + 0x11A9800011A9A, + 0x11B6000011B61, + 0x11B6200011B65, + 0x11B6600011B67, + 0x11C3000011C37, + 0x11C3800011C3E, + 0x11C3F00011C40, + 0x11C9200011CA8, + 0x11CAA00011CB1, + 0x11CB200011CB4, + 0x11CB500011CB7, + 0x11D3100011D37, + 0x11D3A00011D3B, + 0x11D3C00011D3E, + 0x11D3F00011D46, + 0x11D4700011D48, + 0x11D9000011D92, + 0x11D9500011D96, + 0x11D9700011D98, + 0x11EF300011EF5, + 0x11F0000011F02, + 0x11F3600011F3B, + 0x11F4000011F41, + 0x11F4200011F43, + 0x11F5A00011F5B, + 0x1343000013441, + 0x1344700013456, + 0x1611E0001612A, + 0x1612D00016130, + 0x16AF000016AF5, + 0x16B3000016B37, + 0x16F4F00016F50, + 0x16F8F00016F93, + 0x16FE400016FE5, + 0x1BC9D0001BC9F, + 0x1BCA00001BCA4, + 0x1CF000001CF2E, + 0x1CF300001CF47, + 0x1D1670001D16A, + 0x1D1730001D183, + 0x1D1850001D18C, + 0x1D1AA0001D1AE, + 0x1D2420001D245, + 0x1DA000001DA37, + 0x1DA3B0001DA6D, + 0x1DA750001DA76, + 0x1DA840001DA85, + 0x1DA9B0001DAA0, + 0x1DAA10001DAB0, + 0x1E0000001E007, + 0x1E0080001E019, + 0x1E01B0001E022, + 0x1E0230001E025, + 0x1E0260001E02B, + 0x1E08F0001E090, + 0x1E1300001E137, + 0x1E2AE0001E2AF, + 0x1E2EC0001E2F0, + 0x1E4EC0001E4F0, + 0x1E5EE0001E5F0, + 0x1E6E30001E6E4, + 0x1E6E60001E6E7, + 0x1E6EE0001E6F0, + 0x1E6F50001E6F6, + 0x1E8D00001E8D7, + 0x1E9440001E94C, + 0xE0001000E0002, + 0xE0020000E0080, + 0xE0100000E01F0, + ), +} + + +codepoint_classes = { + "PVALID": ( + 0x2D0000002E, + 0x300000003A, + 0x610000007B, + 0xDF000000F7, + 0xF800000100, + 0x10100000102, + 0x10300000104, + 0x10500000106, + 0x10700000108, + 0x1090000010A, + 0x10B0000010C, + 0x10D0000010E, + 0x10F00000110, + 0x11100000112, + 0x11300000114, + 0x11500000116, + 0x11700000118, + 0x1190000011A, + 0x11B0000011C, + 0x11D0000011E, + 0x11F00000120, + 0x12100000122, + 0x12300000124, + 0x12500000126, + 0x12700000128, + 0x1290000012A, + 0x12B0000012C, + 0x12D0000012E, + 0x12F00000130, + 0x13100000132, + 0x13500000136, + 0x13700000139, + 0x13A0000013B, + 0x13C0000013D, + 0x13E0000013F, + 0x14200000143, + 0x14400000145, + 0x14600000147, + 0x14800000149, + 0x14B0000014C, + 0x14D0000014E, + 0x14F00000150, + 0x15100000152, + 0x15300000154, + 0x15500000156, + 0x15700000158, + 0x1590000015A, + 0x15B0000015C, + 0x15D0000015E, + 0x15F00000160, + 0x16100000162, + 0x16300000164, + 0x16500000166, + 0x16700000168, + 0x1690000016A, + 0x16B0000016C, + 0x16D0000016E, + 0x16F00000170, + 0x17100000172, + 0x17300000174, + 0x17500000176, + 0x17700000178, + 0x17A0000017B, + 0x17C0000017D, + 0x17E0000017F, + 0x18000000181, + 0x18300000184, + 0x18500000186, + 0x18800000189, + 0x18C0000018E, + 0x19200000193, + 0x19500000196, + 0x1990000019C, + 0x19E0000019F, + 0x1A1000001A2, + 0x1A3000001A4, + 0x1A5000001A6, + 0x1A8000001A9, + 0x1AA000001AC, + 0x1AD000001AE, + 0x1B0000001B1, + 0x1B4000001B5, + 0x1B6000001B7, + 0x1B9000001BC, + 0x1BD000001C4, + 0x1CE000001CF, + 0x1D0000001D1, + 0x1D2000001D3, + 0x1D4000001D5, + 0x1D6000001D7, + 0x1D8000001D9, + 0x1DA000001DB, + 0x1DC000001DE, + 0x1DF000001E0, + 0x1E1000001E2, + 0x1E3000001E4, + 0x1E5000001E6, + 0x1E7000001E8, + 0x1E9000001EA, + 0x1EB000001EC, + 0x1ED000001EE, + 0x1EF000001F1, + 0x1F5000001F6, + 0x1F9000001FA, + 0x1FB000001FC, + 0x1FD000001FE, + 0x1FF00000200, + 0x20100000202, + 0x20300000204, + 0x20500000206, + 0x20700000208, + 0x2090000020A, + 0x20B0000020C, + 0x20D0000020E, + 0x20F00000210, + 0x21100000212, + 0x21300000214, + 0x21500000216, + 0x21700000218, + 0x2190000021A, + 0x21B0000021C, + 0x21D0000021E, + 0x21F00000220, + 0x22100000222, + 0x22300000224, + 0x22500000226, + 0x22700000228, + 0x2290000022A, + 0x22B0000022C, + 0x22D0000022E, + 0x22F00000230, + 0x23100000232, + 0x2330000023A, + 0x23C0000023D, + 0x23F00000241, + 0x24200000243, + 0x24700000248, + 0x2490000024A, + 0x24B0000024C, + 0x24D0000024E, + 0x24F000002B0, + 0x2B9000002C2, + 0x2C6000002D2, + 0x2EC000002ED, + 0x2EE000002EF, + 0x30000000340, + 0x34200000343, + 0x3460000034F, + 0x35000000370, + 0x37100000372, + 0x37300000374, + 0x37700000378, + 0x37B0000037E, + 0x39000000391, + 0x3AC000003CF, + 0x3D7000003D8, + 0x3D9000003DA, + 0x3DB000003DC, + 0x3DD000003DE, + 0x3DF000003E0, + 0x3E1000003E2, + 0x3E3000003E4, + 0x3E5000003E6, + 0x3E7000003E8, + 0x3E9000003EA, + 0x3EB000003EC, + 0x3ED000003EE, + 0x3EF000003F0, + 0x3F3000003F4, + 0x3F8000003F9, + 0x3FB000003FD, + 0x43000000460, + 0x46100000462, + 0x46300000464, + 0x46500000466, + 0x46700000468, + 0x4690000046A, + 0x46B0000046C, + 0x46D0000046E, + 0x46F00000470, + 0x47100000472, + 0x47300000474, + 0x47500000476, + 0x47700000478, + 0x4790000047A, + 0x47B0000047C, + 0x47D0000047E, + 0x47F00000480, + 0x48100000482, + 0x48300000488, + 0x48B0000048C, + 0x48D0000048E, + 0x48F00000490, + 0x49100000492, + 0x49300000494, + 0x49500000496, + 0x49700000498, + 0x4990000049A, + 0x49B0000049C, + 0x49D0000049E, + 0x49F000004A0, + 0x4A1000004A2, + 0x4A3000004A4, + 0x4A5000004A6, + 0x4A7000004A8, + 0x4A9000004AA, + 0x4AB000004AC, + 0x4AD000004AE, + 0x4AF000004B0, + 0x4B1000004B2, + 0x4B3000004B4, + 0x4B5000004B6, + 0x4B7000004B8, + 0x4B9000004BA, + 0x4BB000004BC, + 0x4BD000004BE, + 0x4BF000004C0, + 0x4C2000004C3, + 0x4C4000004C5, + 0x4C6000004C7, + 0x4C8000004C9, + 0x4CA000004CB, + 0x4CC000004CD, + 0x4CE000004D0, + 0x4D1000004D2, + 0x4D3000004D4, + 0x4D5000004D6, + 0x4D7000004D8, + 0x4D9000004DA, + 0x4DB000004DC, + 0x4DD000004DE, + 0x4DF000004E0, + 0x4E1000004E2, + 0x4E3000004E4, + 0x4E5000004E6, + 0x4E7000004E8, + 0x4E9000004EA, + 0x4EB000004EC, + 0x4ED000004EE, + 0x4EF000004F0, + 0x4F1000004F2, + 0x4F3000004F4, + 0x4F5000004F6, + 0x4F7000004F8, + 0x4F9000004FA, + 0x4FB000004FC, + 0x4FD000004FE, + 0x4FF00000500, + 0x50100000502, + 0x50300000504, + 0x50500000506, + 0x50700000508, + 0x5090000050A, + 0x50B0000050C, + 0x50D0000050E, + 0x50F00000510, + 0x51100000512, + 0x51300000514, + 0x51500000516, + 0x51700000518, + 0x5190000051A, + 0x51B0000051C, + 0x51D0000051E, + 0x51F00000520, + 0x52100000522, + 0x52300000524, + 0x52500000526, + 0x52700000528, + 0x5290000052A, + 0x52B0000052C, + 0x52D0000052E, + 0x52F00000530, + 0x5590000055A, + 0x56000000587, + 0x58800000589, + 0x591000005BE, + 0x5BF000005C0, + 0x5C1000005C3, + 0x5C4000005C6, + 0x5C7000005C8, + 0x5D0000005EB, + 0x5EF000005F3, + 0x6100000061B, + 0x62000000640, + 0x64100000660, + 0x66E00000675, + 0x679000006D4, + 0x6D5000006DD, + 0x6DF000006E9, + 0x6EA000006F0, + 0x6FA00000700, + 0x7100000074B, + 0x74D000007B2, + 0x7C0000007F6, + 0x7FD000007FE, + 0x8000000082E, + 0x8400000085C, + 0x8600000086B, + 0x87000000888, + 0x88900000890, + 0x897000008E2, + 0x8E300000958, + 0x96000000964, + 0x96600000970, + 0x97100000984, + 0x9850000098D, + 0x98F00000991, + 0x993000009A9, + 0x9AA000009B1, + 0x9B2000009B3, + 0x9B6000009BA, + 0x9BC000009C5, + 0x9C7000009C9, + 0x9CB000009CF, + 0x9D7000009D8, + 0x9E0000009E4, + 0x9E6000009F2, + 0x9FC000009FD, + 0x9FE000009FF, + 0xA0100000A04, + 0xA0500000A0B, + 0xA0F00000A11, + 0xA1300000A29, + 0xA2A00000A31, + 0xA3200000A33, + 0xA3500000A36, + 0xA3800000A3A, + 0xA3C00000A3D, + 0xA3E00000A43, + 0xA4700000A49, + 0xA4B00000A4E, + 0xA5100000A52, + 0xA5C00000A5D, + 0xA6600000A76, + 0xA8100000A84, + 0xA8500000A8E, + 0xA8F00000A92, + 0xA9300000AA9, + 0xAAA00000AB1, + 0xAB200000AB4, + 0xAB500000ABA, + 0xABC00000AC6, + 0xAC700000ACA, + 0xACB00000ACE, + 0xAD000000AD1, + 0xAE000000AE4, + 0xAE600000AF0, + 0xAF900000B00, + 0xB0100000B04, + 0xB0500000B0D, + 0xB0F00000B11, + 0xB1300000B29, + 0xB2A00000B31, + 0xB3200000B34, + 0xB3500000B3A, + 0xB3C00000B45, + 0xB4700000B49, + 0xB4B00000B4E, + 0xB5500000B58, + 0xB5F00000B64, + 0xB6600000B70, + 0xB7100000B72, + 0xB8200000B84, + 0xB8500000B8B, + 0xB8E00000B91, + 0xB9200000B96, + 0xB9900000B9B, + 0xB9C00000B9D, + 0xB9E00000BA0, + 0xBA300000BA5, + 0xBA800000BAB, + 0xBAE00000BBA, + 0xBBE00000BC3, + 0xBC600000BC9, + 0xBCA00000BCE, + 0xBD000000BD1, + 0xBD700000BD8, + 0xBE600000BF0, + 0xC0000000C0D, + 0xC0E00000C11, + 0xC1200000C29, + 0xC2A00000C3A, + 0xC3C00000C45, + 0xC4600000C49, + 0xC4A00000C4E, + 0xC5500000C57, + 0xC5800000C5B, + 0xC5C00000C5E, + 0xC6000000C64, + 0xC6600000C70, + 0xC8000000C84, + 0xC8500000C8D, + 0xC8E00000C91, + 0xC9200000CA9, + 0xCAA00000CB4, + 0xCB500000CBA, + 0xCBC00000CC5, + 0xCC600000CC9, + 0xCCA00000CCE, + 0xCD500000CD7, + 0xCDC00000CDF, + 0xCE000000CE4, + 0xCE600000CF0, + 0xCF100000CF4, + 0xD0000000D0D, + 0xD0E00000D11, + 0xD1200000D45, + 0xD4600000D49, + 0xD4A00000D4F, + 0xD5400000D58, + 0xD5F00000D64, + 0xD6600000D70, + 0xD7A00000D80, + 0xD8100000D84, + 0xD8500000D97, + 0xD9A00000DB2, + 0xDB300000DBC, + 0xDBD00000DBE, + 0xDC000000DC7, + 0xDCA00000DCB, + 0xDCF00000DD5, + 0xDD600000DD7, + 0xDD800000DE0, + 0xDE600000DF0, + 0xDF200000DF4, + 0xE0100000E33, + 0xE3400000E3B, + 0xE4000000E4F, + 0xE5000000E5A, + 0xE8100000E83, + 0xE8400000E85, + 0xE8600000E8B, + 0xE8C00000EA4, + 0xEA500000EA6, + 0xEA700000EB3, + 0xEB400000EBE, + 0xEC000000EC5, + 0xEC600000EC7, + 0xEC800000ECF, + 0xED000000EDA, + 0xEDE00000EE0, + 0xF0000000F01, + 0xF0B00000F0C, + 0xF1800000F1A, + 0xF2000000F2A, + 0xF3500000F36, + 0xF3700000F38, + 0xF3900000F3A, + 0xF3E00000F43, + 0xF4400000F48, + 0xF4900000F4D, + 0xF4E00000F52, + 0xF5300000F57, + 0xF5800000F5C, + 0xF5D00000F69, + 0xF6A00000F6D, + 0xF7100000F73, + 0xF7400000F75, + 0xF7A00000F81, + 0xF8200000F85, + 0xF8600000F93, + 0xF9400000F98, + 0xF9900000F9D, + 0xF9E00000FA2, + 0xFA300000FA7, + 0xFA800000FAC, + 0xFAD00000FB9, + 0xFBA00000FBD, + 0xFC600000FC7, + 0x10000000104A, + 0x10500000109E, + 0x10D0000010FB, + 0x10FD00001100, + 0x120000001249, + 0x124A0000124E, + 0x125000001257, + 0x125800001259, + 0x125A0000125E, + 0x126000001289, + 0x128A0000128E, + 0x1290000012B1, + 0x12B2000012B6, + 0x12B8000012BF, + 0x12C0000012C1, + 0x12C2000012C6, + 0x12C8000012D7, + 0x12D800001311, + 0x131200001316, + 0x13180000135B, + 0x135D00001360, + 0x138000001390, + 0x13A0000013F6, + 0x14010000166D, + 0x166F00001680, + 0x16810000169B, + 0x16A0000016EB, + 0x16F1000016F9, + 0x170000001716, + 0x171F00001735, + 0x174000001754, + 0x17600000176D, + 0x176E00001771, + 0x177200001774, + 0x1780000017B4, + 0x17B6000017D4, + 0x17D7000017D8, + 0x17DC000017DE, + 0x17E0000017EA, + 0x18100000181A, + 0x182000001879, + 0x1880000018AB, + 0x18B0000018F6, + 0x19000000191F, + 0x19200000192C, + 0x19300000193C, + 0x19460000196E, + 0x197000001975, + 0x1980000019AC, + 0x19B0000019CA, + 0x19D0000019DA, + 0x1A0000001A1C, + 0x1A2000001A5F, + 0x1A6000001A7D, + 0x1A7F00001A8A, + 0x1A9000001A9A, + 0x1AA700001AA8, + 0x1AB000001ABE, + 0x1ABF00001ADE, + 0x1AE000001AEC, + 0x1B0000001B4D, + 0x1B5000001B5A, + 0x1B6B00001B74, + 0x1B8000001BF4, + 0x1C0000001C38, + 0x1C4000001C4A, + 0x1C4D00001C7E, + 0x1C8A00001C8B, + 0x1CD000001CD3, + 0x1CD400001CFB, + 0x1D0000001D2C, + 0x1D2F00001D30, + 0x1D3B00001D3C, + 0x1D4E00001D4F, + 0x1D6B00001D78, + 0x1D7900001D9B, + 0x1DC000001E00, + 0x1E0100001E02, + 0x1E0300001E04, + 0x1E0500001E06, + 0x1E0700001E08, + 0x1E0900001E0A, + 0x1E0B00001E0C, + 0x1E0D00001E0E, + 0x1E0F00001E10, + 0x1E1100001E12, + 0x1E1300001E14, + 0x1E1500001E16, + 0x1E1700001E18, + 0x1E1900001E1A, + 0x1E1B00001E1C, + 0x1E1D00001E1E, + 0x1E1F00001E20, + 0x1E2100001E22, + 0x1E2300001E24, + 0x1E2500001E26, + 0x1E2700001E28, + 0x1E2900001E2A, + 0x1E2B00001E2C, + 0x1E2D00001E2E, + 0x1E2F00001E30, + 0x1E3100001E32, + 0x1E3300001E34, + 0x1E3500001E36, + 0x1E3700001E38, + 0x1E3900001E3A, + 0x1E3B00001E3C, + 0x1E3D00001E3E, + 0x1E3F00001E40, + 0x1E4100001E42, + 0x1E4300001E44, + 0x1E4500001E46, + 0x1E4700001E48, + 0x1E4900001E4A, + 0x1E4B00001E4C, + 0x1E4D00001E4E, + 0x1E4F00001E50, + 0x1E5100001E52, + 0x1E5300001E54, + 0x1E5500001E56, + 0x1E5700001E58, + 0x1E5900001E5A, + 0x1E5B00001E5C, + 0x1E5D00001E5E, + 0x1E5F00001E60, + 0x1E6100001E62, + 0x1E6300001E64, + 0x1E6500001E66, + 0x1E6700001E68, + 0x1E6900001E6A, + 0x1E6B00001E6C, + 0x1E6D00001E6E, + 0x1E6F00001E70, + 0x1E7100001E72, + 0x1E7300001E74, + 0x1E7500001E76, + 0x1E7700001E78, + 0x1E7900001E7A, + 0x1E7B00001E7C, + 0x1E7D00001E7E, + 0x1E7F00001E80, + 0x1E8100001E82, + 0x1E8300001E84, + 0x1E8500001E86, + 0x1E8700001E88, + 0x1E8900001E8A, + 0x1E8B00001E8C, + 0x1E8D00001E8E, + 0x1E8F00001E90, + 0x1E9100001E92, + 0x1E9300001E94, + 0x1E9500001E9A, + 0x1E9C00001E9E, + 0x1E9F00001EA0, + 0x1EA100001EA2, + 0x1EA300001EA4, + 0x1EA500001EA6, + 0x1EA700001EA8, + 0x1EA900001EAA, + 0x1EAB00001EAC, + 0x1EAD00001EAE, + 0x1EAF00001EB0, + 0x1EB100001EB2, + 0x1EB300001EB4, + 0x1EB500001EB6, + 0x1EB700001EB8, + 0x1EB900001EBA, + 0x1EBB00001EBC, + 0x1EBD00001EBE, + 0x1EBF00001EC0, + 0x1EC100001EC2, + 0x1EC300001EC4, + 0x1EC500001EC6, + 0x1EC700001EC8, + 0x1EC900001ECA, + 0x1ECB00001ECC, + 0x1ECD00001ECE, + 0x1ECF00001ED0, + 0x1ED100001ED2, + 0x1ED300001ED4, + 0x1ED500001ED6, + 0x1ED700001ED8, + 0x1ED900001EDA, + 0x1EDB00001EDC, + 0x1EDD00001EDE, + 0x1EDF00001EE0, + 0x1EE100001EE2, + 0x1EE300001EE4, + 0x1EE500001EE6, + 0x1EE700001EE8, + 0x1EE900001EEA, + 0x1EEB00001EEC, + 0x1EED00001EEE, + 0x1EEF00001EF0, + 0x1EF100001EF2, + 0x1EF300001EF4, + 0x1EF500001EF6, + 0x1EF700001EF8, + 0x1EF900001EFA, + 0x1EFB00001EFC, + 0x1EFD00001EFE, + 0x1EFF00001F08, + 0x1F1000001F16, + 0x1F2000001F28, + 0x1F3000001F38, + 0x1F4000001F46, + 0x1F5000001F58, + 0x1F6000001F68, + 0x1F7000001F71, + 0x1F7200001F73, + 0x1F7400001F75, + 0x1F7600001F77, + 0x1F7800001F79, + 0x1F7A00001F7B, + 0x1F7C00001F7D, + 0x1FB000001FB2, + 0x1FB600001FB7, + 0x1FC600001FC7, + 0x1FD000001FD3, + 0x1FD600001FD8, + 0x1FE000001FE3, + 0x1FE400001FE8, + 0x1FF600001FF7, + 0x214E0000214F, + 0x218400002185, + 0x2C3000002C60, + 0x2C6100002C62, + 0x2C6500002C67, + 0x2C6800002C69, + 0x2C6A00002C6B, + 0x2C6C00002C6D, + 0x2C7100002C72, + 0x2C7300002C75, + 0x2C7600002C7C, + 0x2C8100002C82, + 0x2C8300002C84, + 0x2C8500002C86, + 0x2C8700002C88, + 0x2C8900002C8A, + 0x2C8B00002C8C, + 0x2C8D00002C8E, + 0x2C8F00002C90, + 0x2C9100002C92, + 0x2C9300002C94, + 0x2C9500002C96, + 0x2C9700002C98, + 0x2C9900002C9A, + 0x2C9B00002C9C, + 0x2C9D00002C9E, + 0x2C9F00002CA0, + 0x2CA100002CA2, + 0x2CA300002CA4, + 0x2CA500002CA6, + 0x2CA700002CA8, + 0x2CA900002CAA, + 0x2CAB00002CAC, + 0x2CAD00002CAE, + 0x2CAF00002CB0, + 0x2CB100002CB2, + 0x2CB300002CB4, + 0x2CB500002CB6, + 0x2CB700002CB8, + 0x2CB900002CBA, + 0x2CBB00002CBC, + 0x2CBD00002CBE, + 0x2CBF00002CC0, + 0x2CC100002CC2, + 0x2CC300002CC4, + 0x2CC500002CC6, + 0x2CC700002CC8, + 0x2CC900002CCA, + 0x2CCB00002CCC, + 0x2CCD00002CCE, + 0x2CCF00002CD0, + 0x2CD100002CD2, + 0x2CD300002CD4, + 0x2CD500002CD6, + 0x2CD700002CD8, + 0x2CD900002CDA, + 0x2CDB00002CDC, + 0x2CDD00002CDE, + 0x2CDF00002CE0, + 0x2CE100002CE2, + 0x2CE300002CE5, + 0x2CEC00002CED, + 0x2CEE00002CF2, + 0x2CF300002CF4, + 0x2D0000002D26, + 0x2D2700002D28, + 0x2D2D00002D2E, + 0x2D3000002D68, + 0x2D7F00002D97, + 0x2DA000002DA7, + 0x2DA800002DAF, + 0x2DB000002DB7, + 0x2DB800002DBF, + 0x2DC000002DC7, + 0x2DC800002DCF, + 0x2DD000002DD7, + 0x2DD800002DDF, + 0x2DE000002E00, + 0x2E2F00002E30, + 0x300500003008, + 0x302A0000302E, + 0x303C0000303D, + 0x304100003097, + 0x30990000309B, + 0x309D0000309F, + 0x30A1000030FB, + 0x30FC000030FF, + 0x310500003130, + 0x31A0000031C0, + 0x31F000003200, + 0x340000004DC0, + 0x4E000000A48D, + 0xA4D00000A4FE, + 0xA5000000A60D, + 0xA6100000A62C, + 0xA6410000A642, + 0xA6430000A644, + 0xA6450000A646, + 0xA6470000A648, + 0xA6490000A64A, + 0xA64B0000A64C, + 0xA64D0000A64E, + 0xA64F0000A650, + 0xA6510000A652, + 0xA6530000A654, + 0xA6550000A656, + 0xA6570000A658, + 0xA6590000A65A, + 0xA65B0000A65C, + 0xA65D0000A65E, + 0xA65F0000A660, + 0xA6610000A662, + 0xA6630000A664, + 0xA6650000A666, + 0xA6670000A668, + 0xA6690000A66A, + 0xA66B0000A66C, + 0xA66D0000A670, + 0xA6740000A67E, + 0xA67F0000A680, + 0xA6810000A682, + 0xA6830000A684, + 0xA6850000A686, + 0xA6870000A688, + 0xA6890000A68A, + 0xA68B0000A68C, + 0xA68D0000A68E, + 0xA68F0000A690, + 0xA6910000A692, + 0xA6930000A694, + 0xA6950000A696, + 0xA6970000A698, + 0xA6990000A69A, + 0xA69B0000A69C, + 0xA69E0000A6E6, + 0xA6F00000A6F2, + 0xA7170000A720, + 0xA7230000A724, + 0xA7250000A726, + 0xA7270000A728, + 0xA7290000A72A, + 0xA72B0000A72C, + 0xA72D0000A72E, + 0xA72F0000A732, + 0xA7330000A734, + 0xA7350000A736, + 0xA7370000A738, + 0xA7390000A73A, + 0xA73B0000A73C, + 0xA73D0000A73E, + 0xA73F0000A740, + 0xA7410000A742, + 0xA7430000A744, + 0xA7450000A746, + 0xA7470000A748, + 0xA7490000A74A, + 0xA74B0000A74C, + 0xA74D0000A74E, + 0xA74F0000A750, + 0xA7510000A752, + 0xA7530000A754, + 0xA7550000A756, + 0xA7570000A758, + 0xA7590000A75A, + 0xA75B0000A75C, + 0xA75D0000A75E, + 0xA75F0000A760, + 0xA7610000A762, + 0xA7630000A764, + 0xA7650000A766, + 0xA7670000A768, + 0xA7690000A76A, + 0xA76B0000A76C, + 0xA76D0000A76E, + 0xA76F0000A770, + 0xA7710000A779, + 0xA77A0000A77B, + 0xA77C0000A77D, + 0xA77F0000A780, + 0xA7810000A782, + 0xA7830000A784, + 0xA7850000A786, + 0xA7870000A789, + 0xA78C0000A78D, + 0xA78E0000A790, + 0xA7910000A792, + 0xA7930000A796, + 0xA7970000A798, + 0xA7990000A79A, + 0xA79B0000A79C, + 0xA79D0000A79E, + 0xA79F0000A7A0, + 0xA7A10000A7A2, + 0xA7A30000A7A4, + 0xA7A50000A7A6, + 0xA7A70000A7A8, + 0xA7A90000A7AA, + 0xA7AF0000A7B0, + 0xA7B50000A7B6, + 0xA7B70000A7B8, + 0xA7B90000A7BA, + 0xA7BB0000A7BC, + 0xA7BD0000A7BE, + 0xA7BF0000A7C0, + 0xA7C10000A7C2, + 0xA7C30000A7C4, + 0xA7C80000A7C9, + 0xA7CA0000A7CB, + 0xA7CD0000A7CE, + 0xA7CF0000A7D0, + 0xA7D10000A7D2, + 0xA7D30000A7D4, + 0xA7D50000A7D6, + 0xA7D70000A7D8, + 0xA7D90000A7DA, + 0xA7DB0000A7DC, + 0xA7F60000A7F8, + 0xA7FA0000A828, + 0xA82C0000A82D, + 0xA8400000A874, + 0xA8800000A8C6, + 0xA8D00000A8DA, + 0xA8E00000A8F8, + 0xA8FB0000A8FC, + 0xA8FD0000A92E, + 0xA9300000A954, + 0xA9800000A9C1, + 0xA9CF0000A9DA, + 0xA9E00000A9FF, + 0xAA000000AA37, + 0xAA400000AA4E, + 0xAA500000AA5A, + 0xAA600000AA77, + 0xAA7A0000AAC3, + 0xAADB0000AADE, + 0xAAE00000AAF0, + 0xAAF20000AAF7, + 0xAB010000AB07, + 0xAB090000AB0F, + 0xAB110000AB17, + 0xAB200000AB27, + 0xAB280000AB2F, + 0xAB300000AB5B, + 0xAB600000AB69, + 0xABC00000ABEB, + 0xABEC0000ABEE, + 0xABF00000ABFA, + 0xAC000000D7A4, + 0xFA0E0000FA10, + 0xFA110000FA12, + 0xFA130000FA15, + 0xFA1F0000FA20, + 0xFA210000FA22, + 0xFA230000FA25, + 0xFA270000FA2A, + 0xFB1E0000FB1F, + 0xFE200000FE30, + 0xFE730000FE74, + 0x100000001000C, + 0x1000D00010027, + 0x100280001003B, + 0x1003C0001003E, + 0x1003F0001004E, + 0x100500001005E, + 0x10080000100FB, + 0x101FD000101FE, + 0x102800001029D, + 0x102A0000102D1, + 0x102E0000102E1, + 0x1030000010320, + 0x1032D00010341, + 0x103420001034A, + 0x103500001037B, + 0x103800001039E, + 0x103A0000103C4, + 0x103C8000103D0, + 0x104280001049E, + 0x104A0000104AA, + 0x104D8000104FC, + 0x1050000010528, + 0x1053000010564, + 0x10597000105A2, + 0x105A3000105B2, + 0x105B3000105BA, + 0x105BB000105BD, + 0x105C0000105F4, + 0x1060000010737, + 0x1074000010756, + 0x1076000010768, + 0x1078000010781, + 0x1080000010806, + 0x1080800010809, + 0x1080A00010836, + 0x1083700010839, + 0x1083C0001083D, + 0x1083F00010856, + 0x1086000010877, + 0x108800001089F, + 0x108E0000108F3, + 0x108F4000108F6, + 0x1090000010916, + 0x109200001093A, + 0x109400001095A, + 0x10980000109B8, + 0x109BE000109C0, + 0x10A0000010A04, + 0x10A0500010A07, + 0x10A0C00010A14, + 0x10A1500010A18, + 0x10A1900010A36, + 0x10A3800010A3B, + 0x10A3F00010A40, + 0x10A6000010A7D, + 0x10A8000010A9D, + 0x10AC000010AC8, + 0x10AC900010AE7, + 0x10B0000010B36, + 0x10B4000010B56, + 0x10B6000010B73, + 0x10B8000010B92, + 0x10C0000010C49, + 0x10CC000010CF3, + 0x10D0000010D28, + 0x10D3000010D3A, + 0x10D4000010D50, + 0x10D6900010D6E, + 0x10D6F00010D86, + 0x10E8000010EAA, + 0x10EAB00010EAD, + 0x10EB000010EB2, + 0x10EC200010EC8, + 0x10EFA00010F1D, + 0x10F2700010F28, + 0x10F3000010F51, + 0x10F7000010F86, + 0x10FB000010FC5, + 0x10FE000010FF7, + 0x1100000011047, + 0x1106600011076, + 0x1107F000110BB, + 0x110C2000110C3, + 0x110D0000110E9, + 0x110F0000110FA, + 0x1110000011135, + 0x1113600011140, + 0x1114400011148, + 0x1115000011174, + 0x1117600011177, + 0x11180000111C5, + 0x111C9000111CD, + 0x111CE000111DB, + 0x111DC000111DD, + 0x1120000011212, + 0x1121300011238, + 0x1123E00011242, + 0x1128000011287, + 0x1128800011289, + 0x1128A0001128E, + 0x1128F0001129E, + 0x1129F000112A9, + 0x112B0000112EB, + 0x112F0000112FA, + 0x1130000011304, + 0x113050001130D, + 0x1130F00011311, + 0x1131300011329, + 0x1132A00011331, + 0x1133200011334, + 0x113350001133A, + 0x1133B00011345, + 0x1134700011349, + 0x1134B0001134E, + 0x1135000011351, + 0x1135700011358, + 0x1135D00011364, + 0x113660001136D, + 0x1137000011375, + 0x113800001138A, + 0x1138B0001138C, + 0x1138E0001138F, + 0x11390000113B6, + 0x113B7000113C1, + 0x113C2000113C3, + 0x113C5000113C6, + 0x113C7000113CB, + 0x113CC000113D4, + 0x113E1000113E3, + 0x114000001144B, + 0x114500001145A, + 0x1145E00011462, + 0x11480000114C6, + 0x114C7000114C8, + 0x114D0000114DA, + 0x11580000115B6, + 0x115B8000115C1, + 0x115D8000115DE, + 0x1160000011641, + 0x1164400011645, + 0x116500001165A, + 0x11680000116B9, + 0x116C0000116CA, + 0x116D0000116E4, + 0x117000001171B, + 0x1171D0001172C, + 0x117300001173A, + 0x1174000011747, + 0x118000001183B, + 0x118C0000118EA, + 0x118FF00011907, + 0x119090001190A, + 0x1190C00011914, + 0x1191500011917, + 0x1191800011936, + 0x1193700011939, + 0x1193B00011944, + 0x119500001195A, + 0x119A0000119A8, + 0x119AA000119D8, + 0x119DA000119E2, + 0x119E3000119E5, + 0x11A0000011A3F, + 0x11A4700011A48, + 0x11A5000011A9A, + 0x11A9D00011A9E, + 0x11AB000011AF9, + 0x11B6000011B68, + 0x11BC000011BE1, + 0x11BF000011BFA, + 0x11C0000011C09, + 0x11C0A00011C37, + 0x11C3800011C41, + 0x11C5000011C5A, + 0x11C7200011C90, + 0x11C9200011CA8, + 0x11CA900011CB7, + 0x11D0000011D07, + 0x11D0800011D0A, + 0x11D0B00011D37, + 0x11D3A00011D3B, + 0x11D3C00011D3E, + 0x11D3F00011D48, + 0x11D5000011D5A, + 0x11D6000011D66, + 0x11D6700011D69, + 0x11D6A00011D8F, + 0x11D9000011D92, + 0x11D9300011D99, + 0x11DA000011DAA, + 0x11DB000011DDC, + 0x11DE000011DEA, + 0x11EE000011EF7, + 0x11F0000011F11, + 0x11F1200011F3B, + 0x11F3E00011F43, + 0x11F5000011F5B, + 0x11FB000011FB1, + 0x120000001239A, + 0x1248000012544, + 0x12F9000012FF1, + 0x1300000013430, + 0x1344000013456, + 0x13460000143FB, + 0x1440000014647, + 0x161000001613A, + 0x1680000016A39, + 0x16A4000016A5F, + 0x16A6000016A6A, + 0x16A7000016ABF, + 0x16AC000016ACA, + 0x16AD000016AEE, + 0x16AF000016AF5, + 0x16B0000016B37, + 0x16B4000016B44, + 0x16B5000016B5A, + 0x16B6300016B78, + 0x16B7D00016B90, + 0x16D4000016D6D, + 0x16D7000016D7A, + 0x16E6000016E80, + 0x16EBB00016ED4, + 0x16F0000016F4B, + 0x16F4F00016F88, + 0x16F8F00016FA0, + 0x16FE000016FE2, + 0x16FE300016FE5, + 0x16FF000016FF4, + 0x1700000018CD6, + 0x18CFF00018D1F, + 0x18D8000018DF3, + 0x1AFF00001AFF4, + 0x1AFF50001AFFC, + 0x1AFFD0001AFFF, + 0x1B0000001B123, + 0x1B1320001B133, + 0x1B1500001B153, + 0x1B1550001B156, + 0x1B1640001B168, + 0x1B1700001B2FC, + 0x1BC000001BC6B, + 0x1BC700001BC7D, + 0x1BC800001BC89, + 0x1BC900001BC9A, + 0x1BC9D0001BC9F, + 0x1CF000001CF2E, + 0x1CF300001CF47, + 0x1DA000001DA37, + 0x1DA3B0001DA6D, + 0x1DA750001DA76, + 0x1DA840001DA85, + 0x1DA9B0001DAA0, + 0x1DAA10001DAB0, + 0x1DF000001DF1F, + 0x1DF250001DF2B, + 0x1E0000001E007, + 0x1E0080001E019, + 0x1E01B0001E022, + 0x1E0230001E025, + 0x1E0260001E02B, + 0x1E08F0001E090, + 0x1E1000001E12D, + 0x1E1300001E13E, + 0x1E1400001E14A, + 0x1E14E0001E14F, + 0x1E2900001E2AF, + 0x1E2C00001E2FA, + 0x1E4D00001E4FA, + 0x1E5D00001E5FB, + 0x1E6C00001E6DF, + 0x1E6E00001E6F6, + 0x1E6FE0001E700, + 0x1E7E00001E7E7, + 0x1E7E80001E7EC, + 0x1E7ED0001E7EF, + 0x1E7F00001E7FF, + 0x1E8000001E8C5, + 0x1E8D00001E8D7, + 0x1E9220001E94C, + 0x1E9500001E95A, + 0x200000002A6E0, + 0x2A7000002B81E, + 0x2B8200002CEAE, + 0x2CEB00002EBE1, + 0x2EBF00002EE5E, + 0x300000003134B, + 0x313500003347A, + ), + "CONTEXTJ": (0x200C0000200E,), + "CONTEXTO": ( + 0xB7000000B8, + 0x37500000376, + 0x5F3000005F5, + 0x6600000066A, + 0x6F0000006FA, + 0x30FB000030FC, + ), +} diff --git a/py_modules/idna/intranges.py b/py_modules/idna/intranges.py new file mode 100644 index 0000000..19d7781 --- /dev/null +++ b/py_modules/idna/intranges.py @@ -0,0 +1,55 @@ +""" +Given a list of integers, made up of (hopefully) a small number of long runs +of consecutive integers, compute a representation of the form +((start1, end1), (start2, end2) ...). Then answer the question "was x present +in the original list?" in time O(log(# runs)). +""" + +import bisect + + +def intranges_from_list(list_: list[int]) -> tuple[int, ...]: + """Represent a list of integers as a sequence of ranges: + ((start_0, end_0), (start_1, end_1), ...), such that the original + integers are exactly those x such that start_i <= x < end_i for some i. + + Ranges are encoded as single integers (start << 32 | end), not as tuples. + """ + + sorted_list = sorted(list_) + ranges = [] + last_write = -1 + for i in range(len(sorted_list)): + if i + 1 < len(sorted_list) and sorted_list[i] == sorted_list[i + 1] - 1: + continue + current_range = sorted_list[last_write + 1 : i + 1] + ranges.append(_encode_range(current_range[0], current_range[-1] + 1)) + last_write = i + + return tuple(ranges) + + +def _encode_range(start: int, end: int) -> int: + return (start << 32) | end + + +def _decode_range(r: int) -> tuple[int, int]: + return (r >> 32), (r & ((1 << 32) - 1)) + + +def intranges_contain(int_: int, ranges: tuple[int, ...]) -> bool: + """Determine if `int_` falls into one of the ranges in `ranges`.""" + tuple_ = _encode_range(int_, 0) + pos = bisect.bisect_left(ranges, tuple_) + # we could be immediately ahead of a tuple (start, end) + # with start < int_ <= end + if pos > 0: + left, right = _decode_range(ranges[pos - 1]) + if left <= int_ < right: + return True + # or we could be immediately behind a tuple (int_, end) + if pos < len(ranges): + left, _ = _decode_range(ranges[pos]) + if left == int_: + return True + return False diff --git a/py_modules/idna/package_data.py b/py_modules/idna/package_data.py new file mode 100644 index 0000000..684f1ce --- /dev/null +++ b/py_modules/idna/package_data.py @@ -0,0 +1 @@ +__version__ = "3.19" diff --git a/py_modules/idna/py.typed b/py_modules/idna/py.typed new file mode 100644 index 0000000..e69de29 diff --git a/py_modules/idna/uts46data.py b/py_modules/idna/uts46data.py new file mode 100644 index 0000000..d6712f1 --- /dev/null +++ b/py_modules/idna/uts46data.py @@ -0,0 +1,16897 @@ +# This file is automatically generated by tools/idna-data + +from __future__ import annotations + +from array import array + +"""IDNA Mapping Table from UTS46.""" + + +__version__ = "17.0.0" + +uts46_starts: array[int] = array( + "I", + ( + 0x0, + 0x1, + 0x2, + 0x3, + 0x4, + 0x5, + 0x6, + 0x7, + 0x8, + 0x9, + 0xA, + 0xB, + 0xC, + 0xD, + 0xE, + 0xF, + 0x10, + 0x11, + 0x12, + 0x13, + 0x14, + 0x15, + 0x16, + 0x17, + 0x18, + 0x19, + 0x1A, + 0x1B, + 0x1C, + 0x1D, + 0x1E, + 0x1F, + 0x20, + 0x21, + 0x22, + 0x23, + 0x24, + 0x25, + 0x26, + 0x27, + 0x28, + 0x29, + 0x2A, + 0x2B, + 0x2C, + 0x2D, + 0x2E, + 0x2F, + 0x30, + 0x31, + 0x32, + 0x33, + 0x34, + 0x35, + 0x36, + 0x37, + 0x38, + 0x39, + 0x3A, + 0x3B, + 0x3C, + 0x3D, + 0x3E, + 0x3F, + 0x40, + 0x41, + 0x42, + 0x43, + 0x44, + 0x45, + 0x46, + 0x47, + 0x48, + 0x49, + 0x4A, + 0x4B, + 0x4C, + 0x4D, + 0x4E, + 0x4F, + 0x50, + 0x51, + 0x52, + 0x53, + 0x54, + 0x55, + 0x56, + 0x57, + 0x58, + 0x59, + 0x5A, + 0x5B, + 0x5C, + 0x5D, + 0x5E, + 0x5F, + 0x60, + 0x61, + 0x62, + 0x63, + 0x64, + 0x65, + 0x66, + 0x67, + 0x68, + 0x69, + 0x6A, + 0x6B, + 0x6C, + 0x6D, + 0x6E, + 0x6F, + 0x70, + 0x71, + 0x72, + 0x73, + 0x74, + 0x75, + 0x76, + 0x77, + 0x78, + 0x79, + 0x7A, + 0x7B, + 0x7C, + 0x7D, + 0x7E, + 0x7F, + 0x80, + 0x81, + 0x82, + 0x83, + 0x84, + 0x85, + 0x86, + 0x87, + 0x88, + 0x89, + 0x8A, + 0x8B, + 0x8C, + 0x8D, + 0x8E, + 0x8F, + 0x90, + 0x91, + 0x92, + 0x93, + 0x94, + 0x95, + 0x96, + 0x97, + 0x98, + 0x99, + 0x9A, + 0x9B, + 0x9C, + 0x9D, + 0x9E, + 0x9F, + 0xA0, + 0xA1, + 0xA2, + 0xA3, + 0xA4, + 0xA5, + 0xA6, + 0xA7, + 0xA8, + 0xA9, + 0xAA, + 0xAB, + 0xAC, + 0xAD, + 0xAE, + 0xAF, + 0xB0, + 0xB1, + 0xB2, + 0xB3, + 0xB4, + 0xB5, + 0xB6, + 0xB7, + 0xB8, + 0xB9, + 0xBA, + 0xBB, + 0xBC, + 0xBD, + 0xBE, + 0xBF, + 0xC0, + 0xC1, + 0xC2, + 0xC3, + 0xC4, + 0xC5, + 0xC6, + 0xC7, + 0xC8, + 0xC9, + 0xCA, + 0xCB, + 0xCC, + 0xCD, + 0xCE, + 0xCF, + 0xD0, + 0xD1, + 0xD2, + 0xD3, + 0xD4, + 0xD5, + 0xD6, + 0xD7, + 0xD8, + 0xD9, + 0xDA, + 0xDB, + 0xDC, + 0xDD, + 0xDE, + 0xDF, + 0xE0, + 0xE1, + 0xE2, + 0xE3, + 0xE4, + 0xE5, + 0xE6, + 0xE7, + 0xE8, + 0xE9, + 0xEA, + 0xEB, + 0xEC, + 0xED, + 0xEE, + 0xEF, + 0xF0, + 0xF1, + 0xF2, + 0xF3, + 0xF4, + 0xF5, + 0xF6, + 0xF7, + 0xF8, + 0xF9, + 0xFA, + 0xFB, + 0xFC, + 0xFD, + 0xFE, + 0xFF, + 0x100, + 0x101, + 0x102, + 0x103, + 0x104, + 0x105, + 0x106, + 0x107, + 0x108, + 0x109, + 0x10A, + 0x10B, + 0x10C, + 0x10D, + 0x10E, + 0x10F, + 0x110, + 0x111, + 0x112, + 0x113, + 0x114, + 0x115, + 0x116, + 0x117, + 0x118, + 0x119, + 0x11A, + 0x11B, + 0x11C, + 0x11D, + 0x11E, + 0x11F, + 0x120, + 0x121, + 0x122, + 0x123, + 0x124, + 0x125, + 0x126, + 0x127, + 0x128, + 0x129, + 0x12A, + 0x12B, + 0x12C, + 0x12D, + 0x12E, + 0x12F, + 0x130, + 0x131, + 0x132, + 0x134, + 0x135, + 0x136, + 0x137, + 0x139, + 0x13A, + 0x13B, + 0x13C, + 0x13D, + 0x13E, + 0x13F, + 0x141, + 0x142, + 0x143, + 0x144, + 0x145, + 0x146, + 0x147, + 0x148, + 0x149, + 0x14A, + 0x14B, + 0x14C, + 0x14D, + 0x14E, + 0x14F, + 0x150, + 0x151, + 0x152, + 0x153, + 0x154, + 0x155, + 0x156, + 0x157, + 0x158, + 0x159, + 0x15A, + 0x15B, + 0x15C, + 0x15D, + 0x15E, + 0x15F, + 0x160, + 0x161, + 0x162, + 0x163, + 0x164, + 0x165, + 0x166, + 0x167, + 0x168, + 0x169, + 0x16A, + 0x16B, + 0x16C, + 0x16D, + 0x16E, + 0x16F, + 0x170, + 0x171, + 0x172, + 0x173, + 0x174, + 0x175, + 0x176, + 0x177, + 0x178, + 0x179, + 0x17A, + 0x17B, + 0x17C, + 0x17D, + 0x17E, + 0x17F, + 0x180, + 0x181, + 0x182, + 0x183, + 0x184, + 0x185, + 0x186, + 0x187, + 0x188, + 0x189, + 0x18A, + 0x18B, + 0x18C, + 0x18E, + 0x18F, + 0x190, + 0x191, + 0x192, + 0x193, + 0x194, + 0x195, + 0x196, + 0x197, + 0x198, + 0x199, + 0x19C, + 0x19D, + 0x19E, + 0x19F, + 0x1A0, + 0x1A1, + 0x1A2, + 0x1A3, + 0x1A4, + 0x1A5, + 0x1A6, + 0x1A7, + 0x1A8, + 0x1A9, + 0x1AA, + 0x1AC, + 0x1AD, + 0x1AE, + 0x1AF, + 0x1B0, + 0x1B1, + 0x1B2, + 0x1B3, + 0x1B4, + 0x1B5, + 0x1B6, + 0x1B7, + 0x1B8, + 0x1B9, + 0x1BC, + 0x1BD, + 0x1C4, + 0x1C7, + 0x1CA, + 0x1CD, + 0x1CE, + 0x1CF, + 0x1D0, + 0x1D1, + 0x1D2, + 0x1D3, + 0x1D4, + 0x1D5, + 0x1D6, + 0x1D7, + 0x1D8, + 0x1D9, + 0x1DA, + 0x1DB, + 0x1DC, + 0x1DE, + 0x1DF, + 0x1E0, + 0x1E1, + 0x1E2, + 0x1E3, + 0x1E4, + 0x1E5, + 0x1E6, + 0x1E7, + 0x1E8, + 0x1E9, + 0x1EA, + 0x1EB, + 0x1EC, + 0x1ED, + 0x1EE, + 0x1EF, + 0x1F1, + 0x1F4, + 0x1F5, + 0x1F6, + 0x1F7, + 0x1F8, + 0x1F9, + 0x1FA, + 0x1FB, + 0x1FC, + 0x1FD, + 0x1FE, + 0x1FF, + 0x200, + 0x201, + 0x202, + 0x203, + 0x204, + 0x205, + 0x206, + 0x207, + 0x208, + 0x209, + 0x20A, + 0x20B, + 0x20C, + 0x20D, + 0x20E, + 0x20F, + 0x210, + 0x211, + 0x212, + 0x213, + 0x214, + 0x215, + 0x216, + 0x217, + 0x218, + 0x219, + 0x21A, + 0x21B, + 0x21C, + 0x21D, + 0x21E, + 0x21F, + 0x220, + 0x221, + 0x222, + 0x223, + 0x224, + 0x225, + 0x226, + 0x227, + 0x228, + 0x229, + 0x22A, + 0x22B, + 0x22C, + 0x22D, + 0x22E, + 0x22F, + 0x230, + 0x231, + 0x232, + 0x233, + 0x23A, + 0x23B, + 0x23C, + 0x23D, + 0x23E, + 0x23F, + 0x241, + 0x242, + 0x243, + 0x244, + 0x245, + 0x246, + 0x247, + 0x248, + 0x249, + 0x24A, + 0x24B, + 0x24C, + 0x24D, + 0x24E, + 0x24F, + 0x2B0, + 0x2B1, + 0x2B2, + 0x2B3, + 0x2B4, + 0x2B5, + 0x2B6, + 0x2B7, + 0x2B8, + 0x2B9, + 0x2D8, + 0x2D9, + 0x2DA, + 0x2DB, + 0x2DC, + 0x2DD, + 0x2DE, + 0x2E0, + 0x2E1, + 0x2E2, + 0x2E3, + 0x2E4, + 0x2E5, + 0x340, + 0x341, + 0x342, + 0x343, + 0x344, + 0x345, + 0x346, + 0x34F, + 0x350, + 0x370, + 0x371, + 0x372, + 0x373, + 0x374, + 0x375, + 0x376, + 0x377, + 0x378, + 0x37A, + 0x37B, + 0x37E, + 0x37F, + 0x380, + 0x384, + 0x385, + 0x386, + 0x387, + 0x388, + 0x389, + 0x38A, + 0x38B, + 0x38C, + 0x38D, + 0x38E, + 0x38F, + 0x390, + 0x391, + 0x392, + 0x393, + 0x394, + 0x395, + 0x396, + 0x397, + 0x398, + 0x399, + 0x39A, + 0x39B, + 0x39C, + 0x39D, + 0x39E, + 0x39F, + 0x3A0, + 0x3A1, + 0x3A2, + 0x3A3, + 0x3A4, + 0x3A5, + 0x3A6, + 0x3A7, + 0x3A8, + 0x3A9, + 0x3AA, + 0x3AB, + 0x3AC, + 0x3C2, + 0x3C3, + 0x3CF, + 0x3D0, + 0x3D1, + 0x3D2, + 0x3D3, + 0x3D4, + 0x3D5, + 0x3D6, + 0x3D7, + 0x3D8, + 0x3D9, + 0x3DA, + 0x3DB, + 0x3DC, + 0x3DD, + 0x3DE, + 0x3DF, + 0x3E0, + 0x3E1, + 0x3E2, + 0x3E3, + 0x3E4, + 0x3E5, + 0x3E6, + 0x3E7, + 0x3E8, + 0x3E9, + 0x3EA, + 0x3EB, + 0x3EC, + 0x3ED, + 0x3EE, + 0x3EF, + 0x3F0, + 0x3F1, + 0x3F2, + 0x3F3, + 0x3F4, + 0x3F5, + 0x3F6, + 0x3F7, + 0x3F8, + 0x3F9, + 0x3FA, + 0x3FB, + 0x3FD, + 0x3FE, + 0x3FF, + 0x400, + 0x401, + 0x402, + 0x403, + 0x404, + 0x405, + 0x406, + 0x407, + 0x408, + 0x409, + 0x40A, + 0x40B, + 0x40C, + 0x40D, + 0x40E, + 0x40F, + 0x410, + 0x411, + 0x412, + 0x413, + 0x414, + 0x415, + 0x416, + 0x417, + 0x418, + 0x419, + 0x41A, + 0x41B, + 0x41C, + 0x41D, + 0x41E, + 0x41F, + 0x420, + 0x421, + 0x422, + 0x423, + 0x424, + 0x425, + 0x426, + 0x427, + 0x428, + 0x429, + 0x42A, + 0x42B, + 0x42C, + 0x42D, + 0x42E, + 0x42F, + 0x430, + 0x460, + 0x461, + 0x462, + 0x463, + 0x464, + 0x465, + 0x466, + 0x467, + 0x468, + 0x469, + 0x46A, + 0x46B, + 0x46C, + 0x46D, + 0x46E, + 0x46F, + 0x470, + 0x471, + 0x472, + 0x473, + 0x474, + 0x475, + 0x476, + 0x477, + 0x478, + 0x479, + 0x47A, + 0x47B, + 0x47C, + 0x47D, + 0x47E, + 0x47F, + 0x480, + 0x481, + 0x48A, + 0x48B, + 0x48C, + 0x48D, + 0x48E, + 0x48F, + 0x490, + 0x491, + 0x492, + 0x493, + 0x494, + 0x495, + 0x496, + 0x497, + 0x498, + 0x499, + 0x49A, + 0x49B, + 0x49C, + 0x49D, + 0x49E, + 0x49F, + 0x4A0, + 0x4A1, + 0x4A2, + 0x4A3, + 0x4A4, + 0x4A5, + 0x4A6, + 0x4A7, + 0x4A8, + 0x4A9, + 0x4AA, + 0x4AB, + 0x4AC, + 0x4AD, + 0x4AE, + 0x4AF, + 0x4B0, + 0x4B1, + 0x4B2, + 0x4B3, + 0x4B4, + 0x4B5, + 0x4B6, + 0x4B7, + 0x4B8, + 0x4B9, + 0x4BA, + 0x4BB, + 0x4BC, + 0x4BD, + 0x4BE, + 0x4BF, + 0x4C0, + 0x4C1, + 0x4C2, + 0x4C3, + 0x4C4, + 0x4C5, + 0x4C6, + 0x4C7, + 0x4C8, + 0x4C9, + 0x4CA, + 0x4CB, + 0x4CC, + 0x4CD, + 0x4CE, + 0x4D0, + 0x4D1, + 0x4D2, + 0x4D3, + 0x4D4, + 0x4D5, + 0x4D6, + 0x4D7, + 0x4D8, + 0x4D9, + 0x4DA, + 0x4DB, + 0x4DC, + 0x4DD, + 0x4DE, + 0x4DF, + 0x4E0, + 0x4E1, + 0x4E2, + 0x4E3, + 0x4E4, + 0x4E5, + 0x4E6, + 0x4E7, + 0x4E8, + 0x4E9, + 0x4EA, + 0x4EB, + 0x4EC, + 0x4ED, + 0x4EE, + 0x4EF, + 0x4F0, + 0x4F1, + 0x4F2, + 0x4F3, + 0x4F4, + 0x4F5, + 0x4F6, + 0x4F7, + 0x4F8, + 0x4F9, + 0x4FA, + 0x4FB, + 0x4FC, + 0x4FD, + 0x4FE, + 0x4FF, + 0x500, + 0x501, + 0x502, + 0x503, + 0x504, + 0x505, + 0x506, + 0x507, + 0x508, + 0x509, + 0x50A, + 0x50B, + 0x50C, + 0x50D, + 0x50E, + 0x50F, + 0x510, + 0x511, + 0x512, + 0x513, + 0x514, + 0x515, + 0x516, + 0x517, + 0x518, + 0x519, + 0x51A, + 0x51B, + 0x51C, + 0x51D, + 0x51E, + 0x51F, + 0x520, + 0x521, + 0x522, + 0x523, + 0x524, + 0x525, + 0x526, + 0x527, + 0x528, + 0x529, + 0x52A, + 0x52B, + 0x52C, + 0x52D, + 0x52E, + 0x52F, + 0x530, + 0x531, + 0x532, + 0x533, + 0x534, + 0x535, + 0x536, + 0x537, + 0x538, + 0x539, + 0x53A, + 0x53B, + 0x53C, + 0x53D, + 0x53E, + 0x53F, + 0x540, + 0x541, + 0x542, + 0x543, + 0x544, + 0x545, + 0x546, + 0x547, + 0x548, + 0x549, + 0x54A, + 0x54B, + 0x54C, + 0x54D, + 0x54E, + 0x54F, + 0x550, + 0x551, + 0x552, + 0x553, + 0x554, + 0x555, + 0x556, + 0x557, + 0x559, + 0x587, + 0x588, + 0x58B, + 0x58D, + 0x590, + 0x591, + 0x5C8, + 0x5D0, + 0x5EB, + 0x5EF, + 0x5F5, + 0x606, + 0x61C, + 0x61D, + 0x675, + 0x676, + 0x677, + 0x678, + 0x679, + 0x6DD, + 0x6DE, + 0x70E, + 0x710, + 0x74B, + 0x74D, + 0x7B2, + 0x7C0, + 0x7FB, + 0x7FD, + 0x82E, + 0x830, + 0x83F, + 0x840, + 0x85C, + 0x85E, + 0x85F, + 0x860, + 0x86B, + 0x870, + 0x890, + 0x897, + 0x8E2, + 0x8E3, + 0x958, + 0x959, + 0x95A, + 0x95B, + 0x95C, + 0x95D, + 0x95E, + 0x95F, + 0x960, + 0x984, + 0x985, + 0x98D, + 0x98F, + 0x991, + 0x993, + 0x9A9, + 0x9AA, + 0x9B1, + 0x9B2, + 0x9B3, + 0x9B6, + 0x9BA, + 0x9BC, + 0x9C5, + 0x9C7, + 0x9C9, + 0x9CB, + 0x9CF, + 0x9D7, + 0x9D8, + 0x9DC, + 0x9DD, + 0x9DE, + 0x9DF, + 0x9E0, + 0x9E4, + 0x9E6, + 0x9FF, + 0xA01, + 0xA04, + 0xA05, + 0xA0B, + 0xA0F, + 0xA11, + 0xA13, + 0xA29, + 0xA2A, + 0xA31, + 0xA32, + 0xA33, + 0xA34, + 0xA35, + 0xA36, + 0xA37, + 0xA38, + 0xA3A, + 0xA3C, + 0xA3D, + 0xA3E, + 0xA43, + 0xA47, + 0xA49, + 0xA4B, + 0xA4E, + 0xA51, + 0xA52, + 0xA59, + 0xA5A, + 0xA5B, + 0xA5C, + 0xA5D, + 0xA5E, + 0xA5F, + 0xA66, + 0xA77, + 0xA81, + 0xA84, + 0xA85, + 0xA8E, + 0xA8F, + 0xA92, + 0xA93, + 0xAA9, + 0xAAA, + 0xAB1, + 0xAB2, + 0xAB4, + 0xAB5, + 0xABA, + 0xABC, + 0xAC6, + 0xAC7, + 0xACA, + 0xACB, + 0xACE, + 0xAD0, + 0xAD1, + 0xAE0, + 0xAE4, + 0xAE6, + 0xAF2, + 0xAF9, + 0xB00, + 0xB01, + 0xB04, + 0xB05, + 0xB0D, + 0xB0F, + 0xB11, + 0xB13, + 0xB29, + 0xB2A, + 0xB31, + 0xB32, + 0xB34, + 0xB35, + 0xB3A, + 0xB3C, + 0xB45, + 0xB47, + 0xB49, + 0xB4B, + 0xB4E, + 0xB55, + 0xB58, + 0xB5C, + 0xB5D, + 0xB5E, + 0xB5F, + 0xB64, + 0xB66, + 0xB78, + 0xB82, + 0xB84, + 0xB85, + 0xB8B, + 0xB8E, + 0xB91, + 0xB92, + 0xB96, + 0xB99, + 0xB9B, + 0xB9C, + 0xB9D, + 0xB9E, + 0xBA0, + 0xBA3, + 0xBA5, + 0xBA8, + 0xBAB, + 0xBAE, + 0xBBA, + 0xBBE, + 0xBC3, + 0xBC6, + 0xBC9, + 0xBCA, + 0xBCE, + 0xBD0, + 0xBD1, + 0xBD7, + 0xBD8, + 0xBE6, + 0xBFB, + 0xC00, + 0xC0D, + 0xC0E, + 0xC11, + 0xC12, + 0xC29, + 0xC2A, + 0xC3A, + 0xC3C, + 0xC45, + 0xC46, + 0xC49, + 0xC4A, + 0xC4E, + 0xC55, + 0xC57, + 0xC58, + 0xC5B, + 0xC5C, + 0xC5E, + 0xC60, + 0xC64, + 0xC66, + 0xC70, + 0xC77, + 0xC8D, + 0xC8E, + 0xC91, + 0xC92, + 0xCA9, + 0xCAA, + 0xCB4, + 0xCB5, + 0xCBA, + 0xCBC, + 0xCC5, + 0xCC6, + 0xCC9, + 0xCCA, + 0xCCE, + 0xCD5, + 0xCD7, + 0xCDC, + 0xCDF, + 0xCE0, + 0xCE4, + 0xCE6, + 0xCF0, + 0xCF1, + 0xCF4, + 0xD00, + 0xD0D, + 0xD0E, + 0xD11, + 0xD12, + 0xD45, + 0xD46, + 0xD49, + 0xD4A, + 0xD50, + 0xD54, + 0xD64, + 0xD66, + 0xD80, + 0xD81, + 0xD84, + 0xD85, + 0xD97, + 0xD9A, + 0xDB2, + 0xDB3, + 0xDBC, + 0xDBD, + 0xDBE, + 0xDC0, + 0xDC7, + 0xDCA, + 0xDCB, + 0xDCF, + 0xDD5, + 0xDD6, + 0xDD7, + 0xDD8, + 0xDE0, + 0xDE6, + 0xDF0, + 0xDF2, + 0xDF5, + 0xE01, + 0xE33, + 0xE34, + 0xE3B, + 0xE3F, + 0xE5C, + 0xE81, + 0xE83, + 0xE84, + 0xE85, + 0xE86, + 0xE8B, + 0xE8C, + 0xEA4, + 0xEA5, + 0xEA6, + 0xEA7, + 0xEB3, + 0xEB4, + 0xEBE, + 0xEC0, + 0xEC5, + 0xEC6, + 0xEC7, + 0xEC8, + 0xECF, + 0xED0, + 0xEDA, + 0xEDC, + 0xEDD, + 0xEDE, + 0xEE0, + 0xF00, + 0xF0C, + 0xF0D, + 0xF43, + 0xF44, + 0xF48, + 0xF49, + 0xF4D, + 0xF4E, + 0xF52, + 0xF53, + 0xF57, + 0xF58, + 0xF5C, + 0xF5D, + 0xF69, + 0xF6A, + 0xF6D, + 0xF71, + 0xF73, + 0xF74, + 0xF75, + 0xF76, + 0xF77, + 0xF78, + 0xF79, + 0xF7A, + 0xF81, + 0xF82, + 0xF93, + 0xF94, + 0xF98, + 0xF99, + 0xF9D, + 0xF9E, + 0xFA2, + 0xFA3, + 0xFA7, + 0xFA8, + 0xFAC, + 0xFAD, + 0xFB9, + 0xFBA, + 0xFBD, + 0xFBE, + 0xFCD, + 0xFCE, + 0xFDB, + 0x1000, + 0x10A0, + 0x10A1, + 0x10A2, + 0x10A3, + 0x10A4, + 0x10A5, + 0x10A6, + 0x10A7, + 0x10A8, + 0x10A9, + 0x10AA, + 0x10AB, + 0x10AC, + 0x10AD, + 0x10AE, + 0x10AF, + 0x10B0, + 0x10B1, + 0x10B2, + 0x10B3, + 0x10B4, + 0x10B5, + 0x10B6, + 0x10B7, + 0x10B8, + 0x10B9, + 0x10BA, + 0x10BB, + 0x10BC, + 0x10BD, + 0x10BE, + 0x10BF, + 0x10C0, + 0x10C1, + 0x10C2, + 0x10C3, + 0x10C4, + 0x10C5, + 0x10C6, + 0x10C7, + 0x10C8, + 0x10CD, + 0x10CE, + 0x10D0, + 0x10FC, + 0x10FD, + 0x115F, + 0x1161, + 0x1249, + 0x124A, + 0x124E, + 0x1250, + 0x1257, + 0x1258, + 0x1259, + 0x125A, + 0x125E, + 0x1260, + 0x1289, + 0x128A, + 0x128E, + 0x1290, + 0x12B1, + 0x12B2, + 0x12B6, + 0x12B8, + 0x12BF, + 0x12C0, + 0x12C1, + 0x12C2, + 0x12C6, + 0x12C8, + 0x12D7, + 0x12D8, + 0x1311, + 0x1312, + 0x1316, + 0x1318, + 0x135B, + 0x135D, + 0x137D, + 0x1380, + 0x139A, + 0x13A0, + 0x13F6, + 0x13F8, + 0x13F9, + 0x13FA, + 0x13FB, + 0x13FC, + 0x13FD, + 0x13FE, + 0x1400, + 0x1680, + 0x1681, + 0x169D, + 0x16A0, + 0x16F9, + 0x1700, + 0x1716, + 0x171F, + 0x1737, + 0x1740, + 0x1754, + 0x1760, + 0x176D, + 0x176E, + 0x1771, + 0x1772, + 0x1774, + 0x1780, + 0x17B4, + 0x17B6, + 0x17DE, + 0x17E0, + 0x17EA, + 0x17F0, + 0x17FA, + 0x1800, + 0x180B, + 0x1810, + 0x181A, + 0x1820, + 0x1879, + 0x1880, + 0x18AB, + 0x18B0, + 0x18F6, + 0x1900, + 0x191F, + 0x1920, + 0x192C, + 0x1930, + 0x193C, + 0x1940, + 0x1941, + 0x1944, + 0x196E, + 0x1970, + 0x1975, + 0x1980, + 0x19AC, + 0x19B0, + 0x19CA, + 0x19D0, + 0x19DB, + 0x19DE, + 0x1A1C, + 0x1A1E, + 0x1A5F, + 0x1A60, + 0x1A7D, + 0x1A7F, + 0x1A8A, + 0x1A90, + 0x1A9A, + 0x1AA0, + 0x1AAE, + 0x1AB0, + 0x1ADE, + 0x1AE0, + 0x1AEC, + 0x1B00, + 0x1B4D, + 0x1B4E, + 0x1BF4, + 0x1BFC, + 0x1C38, + 0x1C3B, + 0x1C4A, + 0x1C4D, + 0x1C80, + 0x1C81, + 0x1C82, + 0x1C83, + 0x1C84, + 0x1C86, + 0x1C87, + 0x1C88, + 0x1C89, + 0x1C8A, + 0x1C8B, + 0x1C90, + 0x1C91, + 0x1C92, + 0x1C93, + 0x1C94, + 0x1C95, + 0x1C96, + 0x1C97, + 0x1C98, + 0x1C99, + 0x1C9A, + 0x1C9B, + 0x1C9C, + 0x1C9D, + 0x1C9E, + 0x1C9F, + 0x1CA0, + 0x1CA1, + 0x1CA2, + 0x1CA3, + 0x1CA4, + 0x1CA5, + 0x1CA6, + 0x1CA7, + 0x1CA8, + 0x1CA9, + 0x1CAA, + 0x1CAB, + 0x1CAC, + 0x1CAD, + 0x1CAE, + 0x1CAF, + 0x1CB0, + 0x1CB1, + 0x1CB2, + 0x1CB3, + 0x1CB4, + 0x1CB5, + 0x1CB6, + 0x1CB7, + 0x1CB8, + 0x1CB9, + 0x1CBA, + 0x1CBB, + 0x1CBD, + 0x1CBE, + 0x1CBF, + 0x1CC0, + 0x1CC8, + 0x1CD0, + 0x1CFB, + 0x1D00, + 0x1D2C, + 0x1D2D, + 0x1D2E, + 0x1D2F, + 0x1D30, + 0x1D31, + 0x1D32, + 0x1D33, + 0x1D34, + 0x1D35, + 0x1D36, + 0x1D37, + 0x1D38, + 0x1D39, + 0x1D3A, + 0x1D3B, + 0x1D3C, + 0x1D3D, + 0x1D3E, + 0x1D3F, + 0x1D40, + 0x1D41, + 0x1D42, + 0x1D43, + 0x1D44, + 0x1D45, + 0x1D46, + 0x1D47, + 0x1D48, + 0x1D49, + 0x1D4A, + 0x1D4B, + 0x1D4C, + 0x1D4D, + 0x1D4E, + 0x1D4F, + 0x1D50, + 0x1D51, + 0x1D52, + 0x1D53, + 0x1D54, + 0x1D55, + 0x1D56, + 0x1D57, + 0x1D58, + 0x1D59, + 0x1D5A, + 0x1D5B, + 0x1D5C, + 0x1D5D, + 0x1D5E, + 0x1D5F, + 0x1D60, + 0x1D61, + 0x1D62, + 0x1D63, + 0x1D64, + 0x1D65, + 0x1D66, + 0x1D67, + 0x1D68, + 0x1D69, + 0x1D6A, + 0x1D6B, + 0x1D78, + 0x1D79, + 0x1D9B, + 0x1D9C, + 0x1D9D, + 0x1D9E, + 0x1D9F, + 0x1DA0, + 0x1DA1, + 0x1DA2, + 0x1DA3, + 0x1DA4, + 0x1DA5, + 0x1DA6, + 0x1DA7, + 0x1DA8, + 0x1DA9, + 0x1DAA, + 0x1DAB, + 0x1DAC, + 0x1DAD, + 0x1DAE, + 0x1DAF, + 0x1DB0, + 0x1DB1, + 0x1DB2, + 0x1DB3, + 0x1DB4, + 0x1DB5, + 0x1DB6, + 0x1DB7, + 0x1DB8, + 0x1DB9, + 0x1DBA, + 0x1DBB, + 0x1DBC, + 0x1DBD, + 0x1DBE, + 0x1DBF, + 0x1DC0, + 0x1E00, + 0x1E01, + 0x1E02, + 0x1E03, + 0x1E04, + 0x1E05, + 0x1E06, + 0x1E07, + 0x1E08, + 0x1E09, + 0x1E0A, + 0x1E0B, + 0x1E0C, + 0x1E0D, + 0x1E0E, + 0x1E0F, + 0x1E10, + 0x1E11, + 0x1E12, + 0x1E13, + 0x1E14, + 0x1E15, + 0x1E16, + 0x1E17, + 0x1E18, + 0x1E19, + 0x1E1A, + 0x1E1B, + 0x1E1C, + 0x1E1D, + 0x1E1E, + 0x1E1F, + 0x1E20, + 0x1E21, + 0x1E22, + 0x1E23, + 0x1E24, + 0x1E25, + 0x1E26, + 0x1E27, + 0x1E28, + 0x1E29, + 0x1E2A, + 0x1E2B, + 0x1E2C, + 0x1E2D, + 0x1E2E, + 0x1E2F, + 0x1E30, + 0x1E31, + 0x1E32, + 0x1E33, + 0x1E34, + 0x1E35, + 0x1E36, + 0x1E37, + 0x1E38, + 0x1E39, + 0x1E3A, + 0x1E3B, + 0x1E3C, + 0x1E3D, + 0x1E3E, + 0x1E3F, + 0x1E40, + 0x1E41, + 0x1E42, + 0x1E43, + 0x1E44, + 0x1E45, + 0x1E46, + 0x1E47, + 0x1E48, + 0x1E49, + 0x1E4A, + 0x1E4B, + 0x1E4C, + 0x1E4D, + 0x1E4E, + 0x1E4F, + 0x1E50, + 0x1E51, + 0x1E52, + 0x1E53, + 0x1E54, + 0x1E55, + 0x1E56, + 0x1E57, + 0x1E58, + 0x1E59, + 0x1E5A, + 0x1E5B, + 0x1E5C, + 0x1E5D, + 0x1E5E, + 0x1E5F, + 0x1E60, + 0x1E61, + 0x1E62, + 0x1E63, + 0x1E64, + 0x1E65, + 0x1E66, + 0x1E67, + 0x1E68, + 0x1E69, + 0x1E6A, + 0x1E6B, + 0x1E6C, + 0x1E6D, + 0x1E6E, + 0x1E6F, + 0x1E70, + 0x1E71, + 0x1E72, + 0x1E73, + 0x1E74, + 0x1E75, + 0x1E76, + 0x1E77, + 0x1E78, + 0x1E79, + 0x1E7A, + 0x1E7B, + 0x1E7C, + 0x1E7D, + 0x1E7E, + 0x1E7F, + 0x1E80, + 0x1E81, + 0x1E82, + 0x1E83, + 0x1E84, + 0x1E85, + 0x1E86, + 0x1E87, + 0x1E88, + 0x1E89, + 0x1E8A, + 0x1E8B, + 0x1E8C, + 0x1E8D, + 0x1E8E, + 0x1E8F, + 0x1E90, + 0x1E91, + 0x1E92, + 0x1E93, + 0x1E94, + 0x1E95, + 0x1E9A, + 0x1E9B, + 0x1E9C, + 0x1E9E, + 0x1E9F, + 0x1EA0, + 0x1EA1, + 0x1EA2, + 0x1EA3, + 0x1EA4, + 0x1EA5, + 0x1EA6, + 0x1EA7, + 0x1EA8, + 0x1EA9, + 0x1EAA, + 0x1EAB, + 0x1EAC, + 0x1EAD, + 0x1EAE, + 0x1EAF, + 0x1EB0, + 0x1EB1, + 0x1EB2, + 0x1EB3, + 0x1EB4, + 0x1EB5, + 0x1EB6, + 0x1EB7, + 0x1EB8, + 0x1EB9, + 0x1EBA, + 0x1EBB, + 0x1EBC, + 0x1EBD, + 0x1EBE, + 0x1EBF, + 0x1EC0, + 0x1EC1, + 0x1EC2, + 0x1EC3, + 0x1EC4, + 0x1EC5, + 0x1EC6, + 0x1EC7, + 0x1EC8, + 0x1EC9, + 0x1ECA, + 0x1ECB, + 0x1ECC, + 0x1ECD, + 0x1ECE, + 0x1ECF, + 0x1ED0, + 0x1ED1, + 0x1ED2, + 0x1ED3, + 0x1ED4, + 0x1ED5, + 0x1ED6, + 0x1ED7, + 0x1ED8, + 0x1ED9, + 0x1EDA, + 0x1EDB, + 0x1EDC, + 0x1EDD, + 0x1EDE, + 0x1EDF, + 0x1EE0, + 0x1EE1, + 0x1EE2, + 0x1EE3, + 0x1EE4, + 0x1EE5, + 0x1EE6, + 0x1EE7, + 0x1EE8, + 0x1EE9, + 0x1EEA, + 0x1EEB, + 0x1EEC, + 0x1EED, + 0x1EEE, + 0x1EEF, + 0x1EF0, + 0x1EF1, + 0x1EF2, + 0x1EF3, + 0x1EF4, + 0x1EF5, + 0x1EF6, + 0x1EF7, + 0x1EF8, + 0x1EF9, + 0x1EFA, + 0x1EFB, + 0x1EFC, + 0x1EFD, + 0x1EFE, + 0x1EFF, + 0x1F08, + 0x1F09, + 0x1F0A, + 0x1F0B, + 0x1F0C, + 0x1F0D, + 0x1F0E, + 0x1F0F, + 0x1F10, + 0x1F16, + 0x1F18, + 0x1F19, + 0x1F1A, + 0x1F1B, + 0x1F1C, + 0x1F1D, + 0x1F1E, + 0x1F20, + 0x1F28, + 0x1F29, + 0x1F2A, + 0x1F2B, + 0x1F2C, + 0x1F2D, + 0x1F2E, + 0x1F2F, + 0x1F30, + 0x1F38, + 0x1F39, + 0x1F3A, + 0x1F3B, + 0x1F3C, + 0x1F3D, + 0x1F3E, + 0x1F3F, + 0x1F40, + 0x1F46, + 0x1F48, + 0x1F49, + 0x1F4A, + 0x1F4B, + 0x1F4C, + 0x1F4D, + 0x1F4E, + 0x1F50, + 0x1F58, + 0x1F59, + 0x1F5A, + 0x1F5B, + 0x1F5C, + 0x1F5D, + 0x1F5E, + 0x1F5F, + 0x1F60, + 0x1F68, + 0x1F69, + 0x1F6A, + 0x1F6B, + 0x1F6C, + 0x1F6D, + 0x1F6E, + 0x1F6F, + 0x1F70, + 0x1F71, + 0x1F72, + 0x1F73, + 0x1F74, + 0x1F75, + 0x1F76, + 0x1F77, + 0x1F78, + 0x1F79, + 0x1F7A, + 0x1F7B, + 0x1F7C, + 0x1F7D, + 0x1F7E, + 0x1F80, + 0x1F81, + 0x1F82, + 0x1F83, + 0x1F84, + 0x1F85, + 0x1F86, + 0x1F87, + 0x1F88, + 0x1F89, + 0x1F8A, + 0x1F8B, + 0x1F8C, + 0x1F8D, + 0x1F8E, + 0x1F8F, + 0x1F90, + 0x1F91, + 0x1F92, + 0x1F93, + 0x1F94, + 0x1F95, + 0x1F96, + 0x1F97, + 0x1F98, + 0x1F99, + 0x1F9A, + 0x1F9B, + 0x1F9C, + 0x1F9D, + 0x1F9E, + 0x1F9F, + 0x1FA0, + 0x1FA1, + 0x1FA2, + 0x1FA3, + 0x1FA4, + 0x1FA5, + 0x1FA6, + 0x1FA7, + 0x1FA8, + 0x1FA9, + 0x1FAA, + 0x1FAB, + 0x1FAC, + 0x1FAD, + 0x1FAE, + 0x1FAF, + 0x1FB0, + 0x1FB2, + 0x1FB3, + 0x1FB4, + 0x1FB5, + 0x1FB6, + 0x1FB7, + 0x1FB8, + 0x1FB9, + 0x1FBA, + 0x1FBB, + 0x1FBC, + 0x1FBD, + 0x1FBE, + 0x1FBF, + 0x1FC0, + 0x1FC1, + 0x1FC2, + 0x1FC3, + 0x1FC4, + 0x1FC5, + 0x1FC6, + 0x1FC7, + 0x1FC8, + 0x1FC9, + 0x1FCA, + 0x1FCB, + 0x1FCC, + 0x1FCD, + 0x1FCE, + 0x1FCF, + 0x1FD0, + 0x1FD3, + 0x1FD4, + 0x1FD6, + 0x1FD8, + 0x1FD9, + 0x1FDA, + 0x1FDB, + 0x1FDC, + 0x1FDD, + 0x1FDE, + 0x1FDF, + 0x1FE0, + 0x1FE3, + 0x1FE4, + 0x1FE8, + 0x1FE9, + 0x1FEA, + 0x1FEB, + 0x1FEC, + 0x1FED, + 0x1FEE, + 0x1FEF, + 0x1FF0, + 0x1FF2, + 0x1FF3, + 0x1FF4, + 0x1FF5, + 0x1FF6, + 0x1FF7, + 0x1FF8, + 0x1FF9, + 0x1FFA, + 0x1FFB, + 0x1FFC, + 0x1FFD, + 0x1FFE, + 0x1FFF, + 0x2000, + 0x200B, + 0x200C, + 0x200E, + 0x2010, + 0x2011, + 0x2012, + 0x2017, + 0x2018, + 0x2024, + 0x2027, + 0x2028, + 0x202F, + 0x2030, + 0x2033, + 0x2034, + 0x2035, + 0x2036, + 0x2037, + 0x2038, + 0x203C, + 0x203D, + 0x203E, + 0x203F, + 0x2047, + 0x2048, + 0x2049, + 0x204A, + 0x2057, + 0x2058, + 0x205F, + 0x2060, + 0x2065, + 0x206A, + 0x2070, + 0x2071, + 0x2072, + 0x2074, + 0x2075, + 0x2076, + 0x2077, + 0x2078, + 0x2079, + 0x207A, + 0x207B, + 0x207C, + 0x207D, + 0x207E, + 0x207F, + 0x2080, + 0x2081, + 0x2082, + 0x2083, + 0x2084, + 0x2085, + 0x2086, + 0x2087, + 0x2088, + 0x2089, + 0x208A, + 0x208B, + 0x208C, + 0x208D, + 0x208E, + 0x208F, + 0x2090, + 0x2091, + 0x2092, + 0x2093, + 0x2094, + 0x2095, + 0x2096, + 0x2097, + 0x2098, + 0x2099, + 0x209A, + 0x209B, + 0x209C, + 0x209D, + 0x20A0, + 0x20A8, + 0x20A9, + 0x20C2, + 0x20D0, + 0x20F1, + 0x2100, + 0x2101, + 0x2102, + 0x2103, + 0x2104, + 0x2105, + 0x2106, + 0x2107, + 0x2108, + 0x2109, + 0x210A, + 0x210B, + 0x210F, + 0x2110, + 0x2112, + 0x2114, + 0x2115, + 0x2116, + 0x2117, + 0x2119, + 0x211A, + 0x211B, + 0x211E, + 0x2120, + 0x2121, + 0x2122, + 0x2123, + 0x2124, + 0x2125, + 0x2126, + 0x2127, + 0x2128, + 0x2129, + 0x212A, + 0x212B, + 0x212C, + 0x212D, + 0x212E, + 0x212F, + 0x2131, + 0x2132, + 0x2133, + 0x2134, + 0x2135, + 0x2136, + 0x2137, + 0x2138, + 0x2139, + 0x213A, + 0x213B, + 0x213C, + 0x213D, + 0x213F, + 0x2140, + 0x2141, + 0x2145, + 0x2147, + 0x2148, + 0x2149, + 0x214A, + 0x2150, + 0x2151, + 0x2152, + 0x2153, + 0x2154, + 0x2155, + 0x2156, + 0x2157, + 0x2158, + 0x2159, + 0x215A, + 0x215B, + 0x215C, + 0x215D, + 0x215E, + 0x215F, + 0x2160, + 0x2161, + 0x2162, + 0x2163, + 0x2164, + 0x2165, + 0x2166, + 0x2167, + 0x2168, + 0x2169, + 0x216A, + 0x216B, + 0x216C, + 0x216D, + 0x216E, + 0x216F, + 0x2170, + 0x2171, + 0x2172, + 0x2173, + 0x2174, + 0x2175, + 0x2176, + 0x2177, + 0x2178, + 0x2179, + 0x217A, + 0x217B, + 0x217C, + 0x217D, + 0x217E, + 0x217F, + 0x2180, + 0x2183, + 0x2184, + 0x2189, + 0x218A, + 0x218C, + 0x2190, + 0x222C, + 0x222D, + 0x222E, + 0x222F, + 0x2230, + 0x2231, + 0x2329, + 0x232A, + 0x232B, + 0x242A, + 0x2440, + 0x244B, + 0x2460, + 0x2461, + 0x2462, + 0x2463, + 0x2464, + 0x2465, + 0x2466, + 0x2467, + 0x2468, + 0x2469, + 0x246A, + 0x246B, + 0x246C, + 0x246D, + 0x246E, + 0x246F, + 0x2470, + 0x2471, + 0x2472, + 0x2473, + 0x2474, + 0x2475, + 0x2476, + 0x2477, + 0x2478, + 0x2479, + 0x247A, + 0x247B, + 0x247C, + 0x247D, + 0x247E, + 0x247F, + 0x2480, + 0x2481, + 0x2482, + 0x2483, + 0x2484, + 0x2485, + 0x2486, + 0x2487, + 0x2488, + 0x249C, + 0x249D, + 0x249E, + 0x249F, + 0x24A0, + 0x24A1, + 0x24A2, + 0x24A3, + 0x24A4, + 0x24A5, + 0x24A6, + 0x24A7, + 0x24A8, + 0x24A9, + 0x24AA, + 0x24AB, + 0x24AC, + 0x24AD, + 0x24AE, + 0x24AF, + 0x24B0, + 0x24B1, + 0x24B2, + 0x24B3, + 0x24B4, + 0x24B5, + 0x24B6, + 0x24B7, + 0x24B8, + 0x24B9, + 0x24BA, + 0x24BB, + 0x24BC, + 0x24BD, + 0x24BE, + 0x24BF, + 0x24C0, + 0x24C1, + 0x24C2, + 0x24C3, + 0x24C4, + 0x24C5, + 0x24C6, + 0x24C7, + 0x24C8, + 0x24C9, + 0x24CA, + 0x24CB, + 0x24CC, + 0x24CD, + 0x24CE, + 0x24CF, + 0x24D0, + 0x24D1, + 0x24D2, + 0x24D3, + 0x24D4, + 0x24D5, + 0x24D6, + 0x24D7, + 0x24D8, + 0x24D9, + 0x24DA, + 0x24DB, + 0x24DC, + 0x24DD, + 0x24DE, + 0x24DF, + 0x24E0, + 0x24E1, + 0x24E2, + 0x24E3, + 0x24E4, + 0x24E5, + 0x24E6, + 0x24E7, + 0x24E8, + 0x24E9, + 0x24EA, + 0x24EB, + 0x2A0C, + 0x2A0D, + 0x2A74, + 0x2A75, + 0x2A76, + 0x2A77, + 0x2ADC, + 0x2ADD, + 0x2B74, + 0x2B76, + 0x2C00, + 0x2C01, + 0x2C02, + 0x2C03, + 0x2C04, + 0x2C05, + 0x2C06, + 0x2C07, + 0x2C08, + 0x2C09, + 0x2C0A, + 0x2C0B, + 0x2C0C, + 0x2C0D, + 0x2C0E, + 0x2C0F, + 0x2C10, + 0x2C11, + 0x2C12, + 0x2C13, + 0x2C14, + 0x2C15, + 0x2C16, + 0x2C17, + 0x2C18, + 0x2C19, + 0x2C1A, + 0x2C1B, + 0x2C1C, + 0x2C1D, + 0x2C1E, + 0x2C1F, + 0x2C20, + 0x2C21, + 0x2C22, + 0x2C23, + 0x2C24, + 0x2C25, + 0x2C26, + 0x2C27, + 0x2C28, + 0x2C29, + 0x2C2A, + 0x2C2B, + 0x2C2C, + 0x2C2D, + 0x2C2E, + 0x2C2F, + 0x2C30, + 0x2C60, + 0x2C61, + 0x2C62, + 0x2C63, + 0x2C64, + 0x2C65, + 0x2C67, + 0x2C68, + 0x2C69, + 0x2C6A, + 0x2C6B, + 0x2C6C, + 0x2C6D, + 0x2C6E, + 0x2C6F, + 0x2C70, + 0x2C71, + 0x2C72, + 0x2C73, + 0x2C75, + 0x2C76, + 0x2C7C, + 0x2C7D, + 0x2C7E, + 0x2C7F, + 0x2C80, + 0x2C81, + 0x2C82, + 0x2C83, + 0x2C84, + 0x2C85, + 0x2C86, + 0x2C87, + 0x2C88, + 0x2C89, + 0x2C8A, + 0x2C8B, + 0x2C8C, + 0x2C8D, + 0x2C8E, + 0x2C8F, + 0x2C90, + 0x2C91, + 0x2C92, + 0x2C93, + 0x2C94, + 0x2C95, + 0x2C96, + 0x2C97, + 0x2C98, + 0x2C99, + 0x2C9A, + 0x2C9B, + 0x2C9C, + 0x2C9D, + 0x2C9E, + 0x2C9F, + 0x2CA0, + 0x2CA1, + 0x2CA2, + 0x2CA3, + 0x2CA4, + 0x2CA5, + 0x2CA6, + 0x2CA7, + 0x2CA8, + 0x2CA9, + 0x2CAA, + 0x2CAB, + 0x2CAC, + 0x2CAD, + 0x2CAE, + 0x2CAF, + 0x2CB0, + 0x2CB1, + 0x2CB2, + 0x2CB3, + 0x2CB4, + 0x2CB5, + 0x2CB6, + 0x2CB7, + 0x2CB8, + 0x2CB9, + 0x2CBA, + 0x2CBB, + 0x2CBC, + 0x2CBD, + 0x2CBE, + 0x2CBF, + 0x2CC0, + 0x2CC1, + 0x2CC2, + 0x2CC3, + 0x2CC4, + 0x2CC5, + 0x2CC6, + 0x2CC7, + 0x2CC8, + 0x2CC9, + 0x2CCA, + 0x2CCB, + 0x2CCC, + 0x2CCD, + 0x2CCE, + 0x2CCF, + 0x2CD0, + 0x2CD1, + 0x2CD2, + 0x2CD3, + 0x2CD4, + 0x2CD5, + 0x2CD6, + 0x2CD7, + 0x2CD8, + 0x2CD9, + 0x2CDA, + 0x2CDB, + 0x2CDC, + 0x2CDD, + 0x2CDE, + 0x2CDF, + 0x2CE0, + 0x2CE1, + 0x2CE2, + 0x2CE3, + 0x2CEB, + 0x2CEC, + 0x2CED, + 0x2CEE, + 0x2CF2, + 0x2CF3, + 0x2CF4, + 0x2CF9, + 0x2D26, + 0x2D27, + 0x2D28, + 0x2D2D, + 0x2D2E, + 0x2D30, + 0x2D68, + 0x2D6F, + 0x2D70, + 0x2D71, + 0x2D7F, + 0x2D97, + 0x2DA0, + 0x2DA7, + 0x2DA8, + 0x2DAF, + 0x2DB0, + 0x2DB7, + 0x2DB8, + 0x2DBF, + 0x2DC0, + 0x2DC7, + 0x2DC8, + 0x2DCF, + 0x2DD0, + 0x2DD7, + 0x2DD8, + 0x2DDF, + 0x2DE0, + 0x2E5E, + 0x2E80, + 0x2E9A, + 0x2E9B, + 0x2E9F, + 0x2EA0, + 0x2EF3, + 0x2EF4, + 0x2F00, + 0x2F01, + 0x2F02, + 0x2F03, + 0x2F04, + 0x2F05, + 0x2F06, + 0x2F07, + 0x2F08, + 0x2F09, + 0x2F0A, + 0x2F0B, + 0x2F0C, + 0x2F0D, + 0x2F0E, + 0x2F0F, + 0x2F10, + 0x2F11, + 0x2F12, + 0x2F13, + 0x2F14, + 0x2F15, + 0x2F16, + 0x2F17, + 0x2F18, + 0x2F19, + 0x2F1A, + 0x2F1B, + 0x2F1C, + 0x2F1D, + 0x2F1E, + 0x2F1F, + 0x2F20, + 0x2F21, + 0x2F22, + 0x2F23, + 0x2F24, + 0x2F25, + 0x2F26, + 0x2F27, + 0x2F28, + 0x2F29, + 0x2F2A, + 0x2F2B, + 0x2F2C, + 0x2F2D, + 0x2F2E, + 0x2F2F, + 0x2F30, + 0x2F31, + 0x2F32, + 0x2F33, + 0x2F34, + 0x2F35, + 0x2F36, + 0x2F37, + 0x2F38, + 0x2F39, + 0x2F3A, + 0x2F3B, + 0x2F3C, + 0x2F3D, + 0x2F3E, + 0x2F3F, + 0x2F40, + 0x2F41, + 0x2F42, + 0x2F43, + 0x2F44, + 0x2F45, + 0x2F46, + 0x2F47, + 0x2F48, + 0x2F49, + 0x2F4A, + 0x2F4B, + 0x2F4C, + 0x2F4D, + 0x2F4E, + 0x2F4F, + 0x2F50, + 0x2F51, + 0x2F52, + 0x2F53, + 0x2F54, + 0x2F55, + 0x2F56, + 0x2F57, + 0x2F58, + 0x2F59, + 0x2F5A, + 0x2F5B, + 0x2F5C, + 0x2F5D, + 0x2F5E, + 0x2F5F, + 0x2F60, + 0x2F61, + 0x2F62, + 0x2F63, + 0x2F64, + 0x2F65, + 0x2F66, + 0x2F67, + 0x2F68, + 0x2F69, + 0x2F6A, + 0x2F6B, + 0x2F6C, + 0x2F6D, + 0x2F6E, + 0x2F6F, + 0x2F70, + 0x2F71, + 0x2F72, + 0x2F73, + 0x2F74, + 0x2F75, + 0x2F76, + 0x2F77, + 0x2F78, + 0x2F79, + 0x2F7A, + 0x2F7B, + 0x2F7C, + 0x2F7D, + 0x2F7E, + 0x2F7F, + 0x2F80, + 0x2F81, + 0x2F82, + 0x2F83, + 0x2F84, + 0x2F85, + 0x2F86, + 0x2F87, + 0x2F88, + 0x2F89, + 0x2F8A, + 0x2F8B, + 0x2F8C, + 0x2F8D, + 0x2F8E, + 0x2F8F, + 0x2F90, + 0x2F91, + 0x2F92, + 0x2F93, + 0x2F94, + 0x2F95, + 0x2F96, + 0x2F97, + 0x2F98, + 0x2F99, + 0x2F9A, + 0x2F9B, + 0x2F9C, + 0x2F9D, + 0x2F9E, + 0x2F9F, + 0x2FA0, + 0x2FA1, + 0x2FA2, + 0x2FA3, + 0x2FA4, + 0x2FA5, + 0x2FA6, + 0x2FA7, + 0x2FA8, + 0x2FA9, + 0x2FAA, + 0x2FAB, + 0x2FAC, + 0x2FAD, + 0x2FAE, + 0x2FAF, + 0x2FB0, + 0x2FB1, + 0x2FB2, + 0x2FB3, + 0x2FB4, + 0x2FB5, + 0x2FB6, + 0x2FB7, + 0x2FB8, + 0x2FB9, + 0x2FBA, + 0x2FBB, + 0x2FBC, + 0x2FBD, + 0x2FBE, + 0x2FBF, + 0x2FC0, + 0x2FC1, + 0x2FC2, + 0x2FC3, + 0x2FC4, + 0x2FC5, + 0x2FC6, + 0x2FC7, + 0x2FC8, + 0x2FC9, + 0x2FCA, + 0x2FCB, + 0x2FCC, + 0x2FCD, + 0x2FCE, + 0x2FCF, + 0x2FD0, + 0x2FD1, + 0x2FD2, + 0x2FD3, + 0x2FD4, + 0x2FD5, + 0x2FD6, + 0x3000, + 0x3001, + 0x3002, + 0x3003, + 0x3036, + 0x3037, + 0x3038, + 0x3039, + 0x303A, + 0x303B, + 0x3040, + 0x3041, + 0x3097, + 0x3099, + 0x309B, + 0x309C, + 0x309D, + 0x309F, + 0x30A0, + 0x30FF, + 0x3100, + 0x3105, + 0x3130, + 0x3131, + 0x3132, + 0x3133, + 0x3134, + 0x3135, + 0x3136, + 0x3137, + 0x3138, + 0x3139, + 0x313A, + 0x313B, + 0x313C, + 0x313D, + 0x313E, + 0x313F, + 0x3140, + 0x3141, + 0x3142, + 0x3143, + 0x3144, + 0x3145, + 0x3146, + 0x3147, + 0x3148, + 0x3149, + 0x314A, + 0x314B, + 0x314C, + 0x314D, + 0x314E, + 0x314F, + 0x3150, + 0x3151, + 0x3152, + 0x3153, + 0x3154, + 0x3155, + 0x3156, + 0x3157, + 0x3158, + 0x3159, + 0x315A, + 0x315B, + 0x315C, + 0x315D, + 0x315E, + 0x315F, + 0x3160, + 0x3161, + 0x3162, + 0x3163, + 0x3164, + 0x3165, + 0x3166, + 0x3167, + 0x3168, + 0x3169, + 0x316A, + 0x316B, + 0x316C, + 0x316D, + 0x316E, + 0x316F, + 0x3170, + 0x3171, + 0x3172, + 0x3173, + 0x3174, + 0x3175, + 0x3176, + 0x3177, + 0x3178, + 0x3179, + 0x317A, + 0x317B, + 0x317C, + 0x317D, + 0x317E, + 0x317F, + 0x3180, + 0x3181, + 0x3182, + 0x3183, + 0x3184, + 0x3185, + 0x3186, + 0x3187, + 0x3188, + 0x3189, + 0x318A, + 0x318B, + 0x318C, + 0x318D, + 0x318E, + 0x318F, + 0x3190, + 0x3192, + 0x3193, + 0x3194, + 0x3195, + 0x3196, + 0x3197, + 0x3198, + 0x3199, + 0x319A, + 0x319B, + 0x319C, + 0x319D, + 0x319E, + 0x319F, + 0x31A0, + 0x31E6, + 0x31F0, + 0x3200, + 0x3201, + 0x3202, + 0x3203, + 0x3204, + 0x3205, + 0x3206, + 0x3207, + 0x3208, + 0x3209, + 0x320A, + 0x320B, + 0x320C, + 0x320D, + 0x320E, + 0x320F, + 0x3210, + 0x3211, + 0x3212, + 0x3213, + 0x3214, + 0x3215, + 0x3216, + 0x3217, + 0x3218, + 0x3219, + 0x321A, + 0x321B, + 0x321C, + 0x321D, + 0x321E, + 0x321F, + 0x3220, + 0x3221, + 0x3222, + 0x3223, + 0x3224, + 0x3225, + 0x3226, + 0x3227, + 0x3228, + 0x3229, + 0x322A, + 0x322B, + 0x322C, + 0x322D, + 0x322E, + 0x322F, + 0x3230, + 0x3231, + 0x3232, + 0x3233, + 0x3234, + 0x3235, + 0x3236, + 0x3237, + 0x3238, + 0x3239, + 0x323A, + 0x323B, + 0x323C, + 0x323D, + 0x323E, + 0x323F, + 0x3240, + 0x3241, + 0x3242, + 0x3243, + 0x3244, + 0x3245, + 0x3246, + 0x3247, + 0x3248, + 0x3250, + 0x3251, + 0x3252, + 0x3253, + 0x3254, + 0x3255, + 0x3256, + 0x3257, + 0x3258, + 0x3259, + 0x325A, + 0x325B, + 0x325C, + 0x325D, + 0x325E, + 0x325F, + 0x3260, + 0x3261, + 0x3262, + 0x3263, + 0x3264, + 0x3265, + 0x3266, + 0x3267, + 0x3268, + 0x3269, + 0x326A, + 0x326B, + 0x326C, + 0x326D, + 0x326E, + 0x326F, + 0x3270, + 0x3271, + 0x3272, + 0x3273, + 0x3274, + 0x3275, + 0x3276, + 0x3277, + 0x3278, + 0x3279, + 0x327A, + 0x327B, + 0x327C, + 0x327D, + 0x327E, + 0x327F, + 0x3280, + 0x3281, + 0x3282, + 0x3283, + 0x3284, + 0x3285, + 0x3286, + 0x3287, + 0x3288, + 0x3289, + 0x328A, + 0x328B, + 0x328C, + 0x328D, + 0x328E, + 0x328F, + 0x3290, + 0x3291, + 0x3292, + 0x3293, + 0x3294, + 0x3295, + 0x3296, + 0x3297, + 0x3298, + 0x3299, + 0x329A, + 0x329B, + 0x329C, + 0x329D, + 0x329E, + 0x329F, + 0x32A0, + 0x32A1, + 0x32A2, + 0x32A3, + 0x32A4, + 0x32A5, + 0x32A6, + 0x32A7, + 0x32A8, + 0x32A9, + 0x32AA, + 0x32AB, + 0x32AC, + 0x32AD, + 0x32AE, + 0x32AF, + 0x32B0, + 0x32B1, + 0x32B2, + 0x32B3, + 0x32B4, + 0x32B5, + 0x32B6, + 0x32B7, + 0x32B8, + 0x32B9, + 0x32BA, + 0x32BB, + 0x32BC, + 0x32BD, + 0x32BE, + 0x32BF, + 0x32C0, + 0x32C1, + 0x32C2, + 0x32C3, + 0x32C4, + 0x32C5, + 0x32C6, + 0x32C7, + 0x32C8, + 0x32C9, + 0x32CA, + 0x32CB, + 0x32CC, + 0x32CD, + 0x32CE, + 0x32CF, + 0x32D0, + 0x32D1, + 0x32D2, + 0x32D3, + 0x32D4, + 0x32D5, + 0x32D6, + 0x32D7, + 0x32D8, + 0x32D9, + 0x32DA, + 0x32DB, + 0x32DC, + 0x32DD, + 0x32DE, + 0x32DF, + 0x32E0, + 0x32E1, + 0x32E2, + 0x32E3, + 0x32E4, + 0x32E5, + 0x32E6, + 0x32E7, + 0x32E8, + 0x32E9, + 0x32EA, + 0x32EB, + 0x32EC, + 0x32ED, + 0x32EE, + 0x32EF, + 0x32F0, + 0x32F1, + 0x32F2, + 0x32F3, + 0x32F4, + 0x32F5, + 0x32F6, + 0x32F7, + 0x32F8, + 0x32F9, + 0x32FA, + 0x32FB, + 0x32FC, + 0x32FD, + 0x32FE, + 0x32FF, + 0x3300, + 0x3301, + 0x3302, + 0x3303, + 0x3304, + 0x3305, + 0x3306, + 0x3307, + 0x3308, + 0x3309, + 0x330A, + 0x330B, + 0x330C, + 0x330D, + 0x330E, + 0x330F, + 0x3310, + 0x3311, + 0x3312, + 0x3313, + 0x3314, + 0x3315, + 0x3316, + 0x3317, + 0x3318, + 0x3319, + 0x331A, + 0x331B, + 0x331C, + 0x331D, + 0x331E, + 0x331F, + 0x3320, + 0x3321, + 0x3322, + 0x3323, + 0x3324, + 0x3325, + 0x3326, + 0x3327, + 0x3328, + 0x3329, + 0x332A, + 0x332B, + 0x332C, + 0x332D, + 0x332E, + 0x332F, + 0x3330, + 0x3331, + 0x3332, + 0x3333, + 0x3334, + 0x3335, + 0x3336, + 0x3337, + 0x3338, + 0x3339, + 0x333A, + 0x333B, + 0x333C, + 0x333D, + 0x333E, + 0x333F, + 0x3340, + 0x3341, + 0x3342, + 0x3343, + 0x3344, + 0x3345, + 0x3346, + 0x3347, + 0x3348, + 0x3349, + 0x334A, + 0x334B, + 0x334C, + 0x334D, + 0x334E, + 0x334F, + 0x3350, + 0x3351, + 0x3352, + 0x3353, + 0x3354, + 0x3355, + 0x3356, + 0x3357, + 0x3358, + 0x3359, + 0x335A, + 0x335B, + 0x335C, + 0x335D, + 0x335E, + 0x335F, + 0x3360, + 0x3361, + 0x3362, + 0x3363, + 0x3364, + 0x3365, + 0x3366, + 0x3367, + 0x3368, + 0x3369, + 0x336A, + 0x336B, + 0x336C, + 0x336D, + 0x336E, + 0x336F, + 0x3370, + 0x3371, + 0x3372, + 0x3373, + 0x3374, + 0x3375, + 0x3376, + 0x3377, + 0x3378, + 0x3379, + 0x337A, + 0x337B, + 0x337C, + 0x337D, + 0x337E, + 0x337F, + 0x3380, + 0x3381, + 0x3382, + 0x3383, + 0x3384, + 0x3385, + 0x3386, + 0x3387, + 0x3388, + 0x3389, + 0x338A, + 0x338B, + 0x338C, + 0x338D, + 0x338E, + 0x338F, + 0x3390, + 0x3391, + 0x3392, + 0x3393, + 0x3394, + 0x3395, + 0x3396, + 0x3397, + 0x3398, + 0x3399, + 0x339A, + 0x339B, + 0x339C, + 0x339D, + 0x339E, + 0x339F, + 0x33A0, + 0x33A1, + 0x33A2, + 0x33A3, + 0x33A4, + 0x33A5, + 0x33A6, + 0x33A7, + 0x33A8, + 0x33A9, + 0x33AA, + 0x33AB, + 0x33AC, + 0x33AD, + 0x33AE, + 0x33AF, + 0x33B0, + 0x33B1, + 0x33B2, + 0x33B3, + 0x33B4, + 0x33B5, + 0x33B6, + 0x33B7, + 0x33B8, + 0x33B9, + 0x33BA, + 0x33BB, + 0x33BC, + 0x33BD, + 0x33BE, + 0x33BF, + 0x33C0, + 0x33C1, + 0x33C2, + 0x33C3, + 0x33C4, + 0x33C5, + 0x33C6, + 0x33C7, + 0x33C8, + 0x33C9, + 0x33CA, + 0x33CB, + 0x33CC, + 0x33CD, + 0x33CE, + 0x33CF, + 0x33D0, + 0x33D1, + 0x33D2, + 0x33D3, + 0x33D4, + 0x33D5, + 0x33D6, + 0x33D7, + 0x33D8, + 0x33D9, + 0x33DA, + 0x33DB, + 0x33DC, + 0x33DD, + 0x33DE, + 0x33DF, + 0x33E0, + 0x33E1, + 0x33E2, + 0x33E3, + 0x33E4, + 0x33E5, + 0x33E6, + 0x33E7, + 0x33E8, + 0x33E9, + 0x33EA, + 0x33EB, + 0x33EC, + 0x33ED, + 0x33EE, + 0x33EF, + 0x33F0, + 0x33F1, + 0x33F2, + 0x33F3, + 0x33F4, + 0x33F5, + 0x33F6, + 0x33F7, + 0x33F8, + 0x33F9, + 0x33FA, + 0x33FB, + 0x33FC, + 0x33FD, + 0x33FE, + 0x33FF, + 0x3400, + 0xA48D, + 0xA490, + 0xA4C7, + 0xA4D0, + 0xA62C, + 0xA640, + 0xA641, + 0xA642, + 0xA643, + 0xA644, + 0xA645, + 0xA646, + 0xA647, + 0xA648, + 0xA649, + 0xA64A, + 0xA64B, + 0xA64C, + 0xA64D, + 0xA64E, + 0xA64F, + 0xA650, + 0xA651, + 0xA652, + 0xA653, + 0xA654, + 0xA655, + 0xA656, + 0xA657, + 0xA658, + 0xA659, + 0xA65A, + 0xA65B, + 0xA65C, + 0xA65D, + 0xA65E, + 0xA65F, + 0xA660, + 0xA661, + 0xA662, + 0xA663, + 0xA664, + 0xA665, + 0xA666, + 0xA667, + 0xA668, + 0xA669, + 0xA66A, + 0xA66B, + 0xA66C, + 0xA66D, + 0xA680, + 0xA681, + 0xA682, + 0xA683, + 0xA684, + 0xA685, + 0xA686, + 0xA687, + 0xA688, + 0xA689, + 0xA68A, + 0xA68B, + 0xA68C, + 0xA68D, + 0xA68E, + 0xA68F, + 0xA690, + 0xA691, + 0xA692, + 0xA693, + 0xA694, + 0xA695, + 0xA696, + 0xA697, + 0xA698, + 0xA699, + 0xA69A, + 0xA69B, + 0xA69C, + 0xA69D, + 0xA69E, + 0xA6F8, + 0xA700, + 0xA722, + 0xA723, + 0xA724, + 0xA725, + 0xA726, + 0xA727, + 0xA728, + 0xA729, + 0xA72A, + 0xA72B, + 0xA72C, + 0xA72D, + 0xA72E, + 0xA72F, + 0xA732, + 0xA733, + 0xA734, + 0xA735, + 0xA736, + 0xA737, + 0xA738, + 0xA739, + 0xA73A, + 0xA73B, + 0xA73C, + 0xA73D, + 0xA73E, + 0xA73F, + 0xA740, + 0xA741, + 0xA742, + 0xA743, + 0xA744, + 0xA745, + 0xA746, + 0xA747, + 0xA748, + 0xA749, + 0xA74A, + 0xA74B, + 0xA74C, + 0xA74D, + 0xA74E, + 0xA74F, + 0xA750, + 0xA751, + 0xA752, + 0xA753, + 0xA754, + 0xA755, + 0xA756, + 0xA757, + 0xA758, + 0xA759, + 0xA75A, + 0xA75B, + 0xA75C, + 0xA75D, + 0xA75E, + 0xA75F, + 0xA760, + 0xA761, + 0xA762, + 0xA763, + 0xA764, + 0xA765, + 0xA766, + 0xA767, + 0xA768, + 0xA769, + 0xA76A, + 0xA76B, + 0xA76C, + 0xA76D, + 0xA76E, + 0xA76F, + 0xA770, + 0xA771, + 0xA779, + 0xA77A, + 0xA77B, + 0xA77C, + 0xA77D, + 0xA77E, + 0xA77F, + 0xA780, + 0xA781, + 0xA782, + 0xA783, + 0xA784, + 0xA785, + 0xA786, + 0xA787, + 0xA78B, + 0xA78C, + 0xA78D, + 0xA78E, + 0xA790, + 0xA791, + 0xA792, + 0xA793, + 0xA796, + 0xA797, + 0xA798, + 0xA799, + 0xA79A, + 0xA79B, + 0xA79C, + 0xA79D, + 0xA79E, + 0xA79F, + 0xA7A0, + 0xA7A1, + 0xA7A2, + 0xA7A3, + 0xA7A4, + 0xA7A5, + 0xA7A6, + 0xA7A7, + 0xA7A8, + 0xA7A9, + 0xA7AA, + 0xA7AB, + 0xA7AC, + 0xA7AD, + 0xA7AE, + 0xA7AF, + 0xA7B0, + 0xA7B1, + 0xA7B2, + 0xA7B3, + 0xA7B4, + 0xA7B5, + 0xA7B6, + 0xA7B7, + 0xA7B8, + 0xA7B9, + 0xA7BA, + 0xA7BB, + 0xA7BC, + 0xA7BD, + 0xA7BE, + 0xA7BF, + 0xA7C0, + 0xA7C1, + 0xA7C2, + 0xA7C3, + 0xA7C4, + 0xA7C5, + 0xA7C6, + 0xA7C7, + 0xA7C8, + 0xA7C9, + 0xA7CA, + 0xA7CB, + 0xA7CC, + 0xA7CD, + 0xA7CE, + 0xA7CF, + 0xA7D0, + 0xA7D1, + 0xA7D2, + 0xA7D3, + 0xA7D4, + 0xA7D5, + 0xA7D6, + 0xA7D7, + 0xA7D8, + 0xA7D9, + 0xA7DA, + 0xA7DB, + 0xA7DC, + 0xA7DD, + 0xA7F1, + 0xA7F2, + 0xA7F3, + 0xA7F4, + 0xA7F5, + 0xA7F6, + 0xA7F8, + 0xA7F9, + 0xA7FA, + 0xA82D, + 0xA830, + 0xA83A, + 0xA840, + 0xA878, + 0xA880, + 0xA8C6, + 0xA8CE, + 0xA8DA, + 0xA8E0, + 0xA954, + 0xA95F, + 0xA97D, + 0xA980, + 0xA9CE, + 0xA9CF, + 0xA9DA, + 0xA9DE, + 0xA9FF, + 0xAA00, + 0xAA37, + 0xAA40, + 0xAA4E, + 0xAA50, + 0xAA5A, + 0xAA5C, + 0xAAC3, + 0xAADB, + 0xAAF7, + 0xAB01, + 0xAB07, + 0xAB09, + 0xAB0F, + 0xAB11, + 0xAB17, + 0xAB20, + 0xAB27, + 0xAB28, + 0xAB2F, + 0xAB30, + 0xAB5C, + 0xAB5D, + 0xAB5E, + 0xAB5F, + 0xAB60, + 0xAB69, + 0xAB6A, + 0xAB6C, + 0xAB70, + 0xAB71, + 0xAB72, + 0xAB73, + 0xAB74, + 0xAB75, + 0xAB76, + 0xAB77, + 0xAB78, + 0xAB79, + 0xAB7A, + 0xAB7B, + 0xAB7C, + 0xAB7D, + 0xAB7E, + 0xAB7F, + 0xAB80, + 0xAB81, + 0xAB82, + 0xAB83, + 0xAB84, + 0xAB85, + 0xAB86, + 0xAB87, + 0xAB88, + 0xAB89, + 0xAB8A, + 0xAB8B, + 0xAB8C, + 0xAB8D, + 0xAB8E, + 0xAB8F, + 0xAB90, + 0xAB91, + 0xAB92, + 0xAB93, + 0xAB94, + 0xAB95, + 0xAB96, + 0xAB97, + 0xAB98, + 0xAB99, + 0xAB9A, + 0xAB9B, + 0xAB9C, + 0xAB9D, + 0xAB9E, + 0xAB9F, + 0xABA0, + 0xABA1, + 0xABA2, + 0xABA3, + 0xABA4, + 0xABA5, + 0xABA6, + 0xABA7, + 0xABA8, + 0xABA9, + 0xABAA, + 0xABAB, + 0xABAC, + 0xABAD, + 0xABAE, + 0xABAF, + 0xABB0, + 0xABB1, + 0xABB2, + 0xABB3, + 0xABB4, + 0xABB5, + 0xABB6, + 0xABB7, + 0xABB8, + 0xABB9, + 0xABBA, + 0xABBB, + 0xABBC, + 0xABBD, + 0xABBE, + 0xABBF, + 0xABC0, + 0xABEE, + 0xABF0, + 0xABFA, + 0xAC00, + 0xD7A4, + 0xD7B0, + 0xD7C7, + 0xD7CB, + 0xD7FC, + 0xF900, + 0xF901, + 0xF902, + 0xF903, + 0xF904, + 0xF905, + 0xF906, + 0xF907, + 0xF909, + 0xF90A, + 0xF90B, + 0xF90C, + 0xF90D, + 0xF90E, + 0xF90F, + 0xF910, + 0xF911, + 0xF912, + 0xF913, + 0xF914, + 0xF915, + 0xF916, + 0xF917, + 0xF918, + 0xF919, + 0xF91A, + 0xF91B, + 0xF91C, + 0xF91D, + 0xF91E, + 0xF91F, + 0xF920, + 0xF921, + 0xF922, + 0xF923, + 0xF924, + 0xF925, + 0xF926, + 0xF927, + 0xF928, + 0xF929, + 0xF92A, + 0xF92B, + 0xF92C, + 0xF92D, + 0xF92E, + 0xF92F, + 0xF930, + 0xF931, + 0xF932, + 0xF933, + 0xF934, + 0xF935, + 0xF936, + 0xF937, + 0xF938, + 0xF939, + 0xF93A, + 0xF93B, + 0xF93C, + 0xF93D, + 0xF93E, + 0xF93F, + 0xF940, + 0xF941, + 0xF942, + 0xF943, + 0xF944, + 0xF945, + 0xF946, + 0xF947, + 0xF948, + 0xF949, + 0xF94A, + 0xF94B, + 0xF94C, + 0xF94D, + 0xF94E, + 0xF94F, + 0xF950, + 0xF951, + 0xF952, + 0xF953, + 0xF954, + 0xF955, + 0xF956, + 0xF957, + 0xF958, + 0xF959, + 0xF95A, + 0xF95B, + 0xF95C, + 0xF95D, + 0xF95E, + 0xF95F, + 0xF960, + 0xF961, + 0xF962, + 0xF963, + 0xF964, + 0xF965, + 0xF966, + 0xF967, + 0xF968, + 0xF969, + 0xF96A, + 0xF96B, + 0xF96C, + 0xF96D, + 0xF96E, + 0xF96F, + 0xF970, + 0xF971, + 0xF972, + 0xF973, + 0xF974, + 0xF975, + 0xF976, + 0xF977, + 0xF978, + 0xF979, + 0xF97A, + 0xF97B, + 0xF97C, + 0xF97D, + 0xF97E, + 0xF97F, + 0xF980, + 0xF981, + 0xF982, + 0xF983, + 0xF984, + 0xF985, + 0xF986, + 0xF987, + 0xF988, + 0xF989, + 0xF98A, + 0xF98B, + 0xF98C, + 0xF98D, + 0xF98E, + 0xF98F, + 0xF990, + 0xF991, + 0xF992, + 0xF993, + 0xF994, + 0xF995, + 0xF996, + 0xF997, + 0xF998, + 0xF999, + 0xF99A, + 0xF99B, + 0xF99C, + 0xF99D, + 0xF99E, + 0xF99F, + 0xF9A0, + 0xF9A1, + 0xF9A2, + 0xF9A3, + 0xF9A4, + 0xF9A5, + 0xF9A6, + 0xF9A7, + 0xF9A8, + 0xF9A9, + 0xF9AA, + 0xF9AB, + 0xF9AC, + 0xF9AD, + 0xF9AE, + 0xF9AF, + 0xF9B0, + 0xF9B1, + 0xF9B2, + 0xF9B3, + 0xF9B4, + 0xF9B5, + 0xF9B6, + 0xF9B7, + 0xF9B8, + 0xF9B9, + 0xF9BA, + 0xF9BB, + 0xF9BC, + 0xF9BD, + 0xF9BE, + 0xF9BF, + 0xF9C0, + 0xF9C1, + 0xF9C2, + 0xF9C3, + 0xF9C4, + 0xF9C5, + 0xF9C6, + 0xF9C7, + 0xF9C8, + 0xF9C9, + 0xF9CA, + 0xF9CB, + 0xF9CC, + 0xF9CD, + 0xF9CE, + 0xF9CF, + 0xF9D0, + 0xF9D1, + 0xF9D2, + 0xF9D3, + 0xF9D4, + 0xF9D5, + 0xF9D6, + 0xF9D7, + 0xF9D8, + 0xF9D9, + 0xF9DA, + 0xF9DB, + 0xF9DC, + 0xF9DD, + 0xF9DE, + 0xF9DF, + 0xF9E0, + 0xF9E1, + 0xF9E2, + 0xF9E3, + 0xF9E4, + 0xF9E5, + 0xF9E6, + 0xF9E7, + 0xF9E8, + 0xF9E9, + 0xF9EA, + 0xF9EB, + 0xF9EC, + 0xF9ED, + 0xF9EE, + 0xF9EF, + 0xF9F0, + 0xF9F1, + 0xF9F2, + 0xF9F3, + 0xF9F4, + 0xF9F5, + 0xF9F6, + 0xF9F7, + 0xF9F8, + 0xF9F9, + 0xF9FA, + 0xF9FB, + 0xF9FC, + 0xF9FD, + 0xF9FE, + 0xF9FF, + 0xFA00, + 0xFA01, + 0xFA02, + 0xFA03, + 0xFA04, + 0xFA05, + 0xFA06, + 0xFA07, + 0xFA08, + 0xFA09, + 0xFA0A, + 0xFA0B, + 0xFA0C, + 0xFA0D, + 0xFA0E, + 0xFA10, + 0xFA11, + 0xFA12, + 0xFA13, + 0xFA15, + 0xFA16, + 0xFA17, + 0xFA18, + 0xFA19, + 0xFA1A, + 0xFA1B, + 0xFA1C, + 0xFA1D, + 0xFA1E, + 0xFA1F, + 0xFA20, + 0xFA21, + 0xFA22, + 0xFA23, + 0xFA25, + 0xFA26, + 0xFA27, + 0xFA2A, + 0xFA2B, + 0xFA2C, + 0xFA2D, + 0xFA2E, + 0xFA2F, + 0xFA30, + 0xFA31, + 0xFA32, + 0xFA33, + 0xFA34, + 0xFA35, + 0xFA36, + 0xFA37, + 0xFA38, + 0xFA39, + 0xFA3A, + 0xFA3B, + 0xFA3C, + 0xFA3D, + 0xFA3E, + 0xFA3F, + 0xFA40, + 0xFA41, + 0xFA42, + 0xFA43, + 0xFA44, + 0xFA45, + 0xFA46, + 0xFA47, + 0xFA48, + 0xFA49, + 0xFA4A, + 0xFA4B, + 0xFA4C, + 0xFA4D, + 0xFA4E, + 0xFA4F, + 0xFA50, + 0xFA51, + 0xFA52, + 0xFA53, + 0xFA54, + 0xFA55, + 0xFA56, + 0xFA57, + 0xFA58, + 0xFA59, + 0xFA5A, + 0xFA5B, + 0xFA5C, + 0xFA5D, + 0xFA5F, + 0xFA60, + 0xFA61, + 0xFA62, + 0xFA63, + 0xFA64, + 0xFA65, + 0xFA66, + 0xFA67, + 0xFA68, + 0xFA69, + 0xFA6A, + 0xFA6B, + 0xFA6C, + 0xFA6D, + 0xFA6E, + 0xFA70, + 0xFA71, + 0xFA72, + 0xFA73, + 0xFA74, + 0xFA75, + 0xFA76, + 0xFA77, + 0xFA78, + 0xFA79, + 0xFA7A, + 0xFA7B, + 0xFA7C, + 0xFA7D, + 0xFA7E, + 0xFA7F, + 0xFA80, + 0xFA81, + 0xFA82, + 0xFA83, + 0xFA84, + 0xFA85, + 0xFA86, + 0xFA87, + 0xFA88, + 0xFA89, + 0xFA8A, + 0xFA8B, + 0xFA8C, + 0xFA8D, + 0xFA8E, + 0xFA8F, + 0xFA90, + 0xFA91, + 0xFA92, + 0xFA93, + 0xFA94, + 0xFA95, + 0xFA96, + 0xFA97, + 0xFA98, + 0xFA99, + 0xFA9A, + 0xFA9B, + 0xFA9C, + 0xFA9D, + 0xFA9E, + 0xFA9F, + 0xFAA0, + 0xFAA1, + 0xFAA2, + 0xFAA3, + 0xFAA4, + 0xFAA5, + 0xFAA6, + 0xFAA7, + 0xFAA8, + 0xFAA9, + 0xFAAA, + 0xFAAB, + 0xFAAC, + 0xFAAD, + 0xFAAE, + 0xFAAF, + 0xFAB0, + 0xFAB1, + 0xFAB2, + 0xFAB3, + 0xFAB4, + 0xFAB5, + 0xFAB6, + 0xFAB7, + 0xFAB8, + 0xFAB9, + 0xFABA, + 0xFABB, + 0xFABC, + 0xFABD, + 0xFABE, + 0xFABF, + 0xFAC0, + 0xFAC1, + 0xFAC2, + 0xFAC3, + 0xFAC4, + 0xFAC5, + 0xFAC6, + 0xFAC7, + 0xFAC8, + 0xFAC9, + 0xFACA, + 0xFACB, + 0xFACC, + 0xFACD, + 0xFACE, + 0xFACF, + 0xFAD0, + 0xFAD1, + 0xFAD2, + 0xFAD3, + 0xFAD4, + 0xFAD5, + 0xFAD6, + 0xFAD7, + 0xFAD8, + 0xFAD9, + 0xFADA, + 0xFB00, + 0xFB01, + 0xFB02, + 0xFB03, + 0xFB04, + 0xFB05, + 0xFB07, + 0xFB13, + 0xFB14, + 0xFB15, + 0xFB16, + 0xFB17, + 0xFB18, + 0xFB1D, + 0xFB1E, + 0xFB1F, + 0xFB20, + 0xFB21, + 0xFB22, + 0xFB23, + 0xFB24, + 0xFB25, + 0xFB26, + 0xFB27, + 0xFB28, + 0xFB29, + 0xFB2A, + 0xFB2B, + 0xFB2C, + 0xFB2D, + 0xFB2E, + 0xFB2F, + 0xFB30, + 0xFB31, + 0xFB32, + 0xFB33, + 0xFB34, + 0xFB35, + 0xFB36, + 0xFB37, + 0xFB38, + 0xFB39, + 0xFB3A, + 0xFB3B, + 0xFB3C, + 0xFB3D, + 0xFB3E, + 0xFB3F, + 0xFB40, + 0xFB41, + 0xFB42, + 0xFB43, + 0xFB44, + 0xFB45, + 0xFB46, + 0xFB47, + 0xFB48, + 0xFB49, + 0xFB4A, + 0xFB4B, + 0xFB4C, + 0xFB4D, + 0xFB4E, + 0xFB4F, + 0xFB50, + 0xFB52, + 0xFB56, + 0xFB5A, + 0xFB5E, + 0xFB62, + 0xFB66, + 0xFB6A, + 0xFB6E, + 0xFB72, + 0xFB76, + 0xFB7A, + 0xFB7E, + 0xFB82, + 0xFB84, + 0xFB86, + 0xFB88, + 0xFB8A, + 0xFB8C, + 0xFB8E, + 0xFB92, + 0xFB96, + 0xFB9A, + 0xFB9E, + 0xFBA0, + 0xFBA4, + 0xFBA6, + 0xFBAA, + 0xFBAE, + 0xFBB0, + 0xFBB2, + 0xFBD3, + 0xFBD7, + 0xFBD9, + 0xFBDB, + 0xFBDD, + 0xFBDE, + 0xFBE0, + 0xFBE2, + 0xFBE4, + 0xFBE8, + 0xFBEA, + 0xFBEC, + 0xFBEE, + 0xFBF0, + 0xFBF2, + 0xFBF4, + 0xFBF6, + 0xFBF9, + 0xFBFC, + 0xFC00, + 0xFC01, + 0xFC02, + 0xFC03, + 0xFC04, + 0xFC05, + 0xFC06, + 0xFC07, + 0xFC08, + 0xFC09, + 0xFC0A, + 0xFC0B, + 0xFC0C, + 0xFC0D, + 0xFC0E, + 0xFC0F, + 0xFC10, + 0xFC11, + 0xFC12, + 0xFC13, + 0xFC14, + 0xFC15, + 0xFC16, + 0xFC17, + 0xFC18, + 0xFC19, + 0xFC1A, + 0xFC1B, + 0xFC1C, + 0xFC1D, + 0xFC1E, + 0xFC1F, + 0xFC20, + 0xFC21, + 0xFC22, + 0xFC23, + 0xFC24, + 0xFC25, + 0xFC26, + 0xFC27, + 0xFC28, + 0xFC29, + 0xFC2A, + 0xFC2B, + 0xFC2C, + 0xFC2D, + 0xFC2E, + 0xFC2F, + 0xFC30, + 0xFC31, + 0xFC32, + 0xFC33, + 0xFC34, + 0xFC35, + 0xFC36, + 0xFC37, + 0xFC38, + 0xFC39, + 0xFC3A, + 0xFC3B, + 0xFC3C, + 0xFC3D, + 0xFC3E, + 0xFC3F, + 0xFC40, + 0xFC41, + 0xFC42, + 0xFC43, + 0xFC44, + 0xFC45, + 0xFC46, + 0xFC47, + 0xFC48, + 0xFC49, + 0xFC4A, + 0xFC4B, + 0xFC4C, + 0xFC4D, + 0xFC4E, + 0xFC4F, + 0xFC50, + 0xFC51, + 0xFC52, + 0xFC53, + 0xFC54, + 0xFC55, + 0xFC56, + 0xFC57, + 0xFC58, + 0xFC59, + 0xFC5A, + 0xFC5B, + 0xFC5C, + 0xFC5D, + 0xFC5E, + 0xFC5F, + 0xFC60, + 0xFC61, + 0xFC62, + 0xFC63, + 0xFC64, + 0xFC65, + 0xFC66, + 0xFC67, + 0xFC68, + 0xFC69, + 0xFC6A, + 0xFC6B, + 0xFC6C, + 0xFC6D, + 0xFC6E, + 0xFC6F, + 0xFC70, + 0xFC71, + 0xFC72, + 0xFC73, + 0xFC74, + 0xFC75, + 0xFC76, + 0xFC77, + 0xFC78, + 0xFC79, + 0xFC7A, + 0xFC7B, + 0xFC7C, + 0xFC7D, + 0xFC7E, + 0xFC7F, + 0xFC80, + 0xFC81, + 0xFC82, + 0xFC83, + 0xFC84, + 0xFC85, + 0xFC86, + 0xFC87, + 0xFC88, + 0xFC89, + 0xFC8A, + 0xFC8B, + 0xFC8C, + 0xFC8D, + 0xFC8E, + 0xFC8F, + 0xFC90, + 0xFC91, + 0xFC92, + 0xFC93, + 0xFC94, + 0xFC95, + 0xFC96, + 0xFC97, + 0xFC98, + 0xFC99, + 0xFC9A, + 0xFC9B, + 0xFC9C, + 0xFC9D, + 0xFC9E, + 0xFC9F, + 0xFCA0, + 0xFCA1, + 0xFCA2, + 0xFCA3, + 0xFCA4, + 0xFCA5, + 0xFCA6, + 0xFCA7, + 0xFCA8, + 0xFCA9, + 0xFCAA, + 0xFCAB, + 0xFCAC, + 0xFCAD, + 0xFCAE, + 0xFCAF, + 0xFCB0, + 0xFCB1, + 0xFCB2, + 0xFCB3, + 0xFCB4, + 0xFCB5, + 0xFCB6, + 0xFCB7, + 0xFCB8, + 0xFCB9, + 0xFCBA, + 0xFCBB, + 0xFCBC, + 0xFCBD, + 0xFCBE, + 0xFCBF, + 0xFCC0, + 0xFCC1, + 0xFCC2, + 0xFCC3, + 0xFCC4, + 0xFCC5, + 0xFCC6, + 0xFCC7, + 0xFCC8, + 0xFCC9, + 0xFCCA, + 0xFCCB, + 0xFCCC, + 0xFCCD, + 0xFCCE, + 0xFCCF, + 0xFCD0, + 0xFCD1, + 0xFCD2, + 0xFCD3, + 0xFCD4, + 0xFCD5, + 0xFCD6, + 0xFCD7, + 0xFCD8, + 0xFCD9, + 0xFCDA, + 0xFCDB, + 0xFCDC, + 0xFCDD, + 0xFCDE, + 0xFCDF, + 0xFCE0, + 0xFCE1, + 0xFCE2, + 0xFCE3, + 0xFCE4, + 0xFCE5, + 0xFCE6, + 0xFCE7, + 0xFCE8, + 0xFCE9, + 0xFCEA, + 0xFCEB, + 0xFCEC, + 0xFCED, + 0xFCEE, + 0xFCEF, + 0xFCF0, + 0xFCF1, + 0xFCF2, + 0xFCF3, + 0xFCF4, + 0xFCF5, + 0xFCF6, + 0xFCF7, + 0xFCF8, + 0xFCF9, + 0xFCFA, + 0xFCFB, + 0xFCFC, + 0xFCFD, + 0xFCFE, + 0xFCFF, + 0xFD00, + 0xFD01, + 0xFD02, + 0xFD03, + 0xFD04, + 0xFD05, + 0xFD06, + 0xFD07, + 0xFD08, + 0xFD09, + 0xFD0A, + 0xFD0B, + 0xFD0C, + 0xFD0D, + 0xFD0E, + 0xFD0F, + 0xFD10, + 0xFD11, + 0xFD12, + 0xFD13, + 0xFD14, + 0xFD15, + 0xFD16, + 0xFD17, + 0xFD18, + 0xFD19, + 0xFD1A, + 0xFD1B, + 0xFD1C, + 0xFD1D, + 0xFD1E, + 0xFD1F, + 0xFD20, + 0xFD21, + 0xFD22, + 0xFD23, + 0xFD24, + 0xFD25, + 0xFD26, + 0xFD27, + 0xFD28, + 0xFD29, + 0xFD2A, + 0xFD2B, + 0xFD2C, + 0xFD2D, + 0xFD2E, + 0xFD2F, + 0xFD30, + 0xFD31, + 0xFD32, + 0xFD33, + 0xFD34, + 0xFD35, + 0xFD36, + 0xFD37, + 0xFD38, + 0xFD39, + 0xFD3A, + 0xFD3B, + 0xFD3C, + 0xFD3E, + 0xFD50, + 0xFD51, + 0xFD53, + 0xFD54, + 0xFD55, + 0xFD56, + 0xFD57, + 0xFD58, + 0xFD5A, + 0xFD5B, + 0xFD5C, + 0xFD5D, + 0xFD5E, + 0xFD5F, + 0xFD61, + 0xFD62, + 0xFD64, + 0xFD66, + 0xFD67, + 0xFD69, + 0xFD6A, + 0xFD6C, + 0xFD6E, + 0xFD6F, + 0xFD71, + 0xFD73, + 0xFD74, + 0xFD75, + 0xFD76, + 0xFD78, + 0xFD79, + 0xFD7A, + 0xFD7B, + 0xFD7C, + 0xFD7E, + 0xFD7F, + 0xFD80, + 0xFD81, + 0xFD82, + 0xFD83, + 0xFD85, + 0xFD87, + 0xFD89, + 0xFD8A, + 0xFD8B, + 0xFD8C, + 0xFD8D, + 0xFD8E, + 0xFD8F, + 0xFD90, + 0xFD92, + 0xFD93, + 0xFD94, + 0xFD95, + 0xFD96, + 0xFD97, + 0xFD99, + 0xFD9A, + 0xFD9B, + 0xFD9C, + 0xFD9E, + 0xFD9F, + 0xFDA0, + 0xFDA1, + 0xFDA2, + 0xFDA3, + 0xFDA4, + 0xFDA5, + 0xFDA6, + 0xFDA7, + 0xFDA8, + 0xFDA9, + 0xFDAA, + 0xFDAB, + 0xFDAC, + 0xFDAD, + 0xFDAE, + 0xFDAF, + 0xFDB0, + 0xFDB1, + 0xFDB2, + 0xFDB3, + 0xFDB4, + 0xFDB5, + 0xFDB6, + 0xFDB7, + 0xFDB8, + 0xFDB9, + 0xFDBA, + 0xFDBB, + 0xFDBC, + 0xFDBD, + 0xFDBE, + 0xFDBF, + 0xFDC0, + 0xFDC1, + 0xFDC2, + 0xFDC3, + 0xFDC4, + 0xFDC5, + 0xFDC6, + 0xFDC7, + 0xFDC8, + 0xFDD0, + 0xFDF0, + 0xFDF1, + 0xFDF2, + 0xFDF3, + 0xFDF4, + 0xFDF5, + 0xFDF6, + 0xFDF7, + 0xFDF8, + 0xFDF9, + 0xFDFA, + 0xFDFB, + 0xFDFC, + 0xFDFD, + 0xFE00, + 0xFE10, + 0xFE11, + 0xFE12, + 0xFE13, + 0xFE14, + 0xFE15, + 0xFE16, + 0xFE17, + 0xFE18, + 0xFE19, + 0xFE20, + 0xFE30, + 0xFE31, + 0xFE32, + 0xFE33, + 0xFE35, + 0xFE36, + 0xFE37, + 0xFE38, + 0xFE39, + 0xFE3A, + 0xFE3B, + 0xFE3C, + 0xFE3D, + 0xFE3E, + 0xFE3F, + 0xFE40, + 0xFE41, + 0xFE42, + 0xFE43, + 0xFE44, + 0xFE45, + 0xFE47, + 0xFE48, + 0xFE49, + 0xFE4D, + 0xFE50, + 0xFE51, + 0xFE52, + 0xFE54, + 0xFE55, + 0xFE56, + 0xFE57, + 0xFE58, + 0xFE59, + 0xFE5A, + 0xFE5B, + 0xFE5C, + 0xFE5D, + 0xFE5E, + 0xFE5F, + 0xFE60, + 0xFE61, + 0xFE62, + 0xFE63, + 0xFE64, + 0xFE65, + 0xFE66, + 0xFE67, + 0xFE68, + 0xFE69, + 0xFE6A, + 0xFE6B, + 0xFE6C, + 0xFE70, + 0xFE71, + 0xFE72, + 0xFE73, + 0xFE74, + 0xFE75, + 0xFE76, + 0xFE77, + 0xFE78, + 0xFE79, + 0xFE7A, + 0xFE7B, + 0xFE7C, + 0xFE7D, + 0xFE7E, + 0xFE7F, + 0xFE80, + 0xFE81, + 0xFE83, + 0xFE85, + 0xFE87, + 0xFE89, + 0xFE8D, + 0xFE8F, + 0xFE93, + 0xFE95, + 0xFE99, + 0xFE9D, + 0xFEA1, + 0xFEA5, + 0xFEA9, + 0xFEAB, + 0xFEAD, + 0xFEAF, + 0xFEB1, + 0xFEB5, + 0xFEB9, + 0xFEBD, + 0xFEC1, + 0xFEC5, + 0xFEC9, + 0xFECD, + 0xFED1, + 0xFED5, + 0xFED9, + 0xFEDD, + 0xFEE1, + 0xFEE5, + 0xFEE9, + 0xFEED, + 0xFEEF, + 0xFEF1, + 0xFEF5, + 0xFEF7, + 0xFEF9, + 0xFEFB, + 0xFEFD, + 0xFEFF, + 0xFF00, + 0xFF01, + 0xFF02, + 0xFF03, + 0xFF04, + 0xFF05, + 0xFF06, + 0xFF07, + 0xFF08, + 0xFF09, + 0xFF0A, + 0xFF0B, + 0xFF0C, + 0xFF0D, + 0xFF0E, + 0xFF0F, + 0xFF10, + 0xFF11, + 0xFF12, + 0xFF13, + 0xFF14, + 0xFF15, + 0xFF16, + 0xFF17, + 0xFF18, + 0xFF19, + 0xFF1A, + 0xFF1B, + 0xFF1C, + 0xFF1D, + 0xFF1E, + 0xFF1F, + 0xFF20, + 0xFF21, + 0xFF22, + 0xFF23, + 0xFF24, + 0xFF25, + 0xFF26, + 0xFF27, + 0xFF28, + 0xFF29, + 0xFF2A, + 0xFF2B, + 0xFF2C, + 0xFF2D, + 0xFF2E, + 0xFF2F, + 0xFF30, + 0xFF31, + 0xFF32, + 0xFF33, + 0xFF34, + 0xFF35, + 0xFF36, + 0xFF37, + 0xFF38, + 0xFF39, + 0xFF3A, + 0xFF3B, + 0xFF3C, + 0xFF3D, + 0xFF3E, + 0xFF3F, + 0xFF40, + 0xFF41, + 0xFF42, + 0xFF43, + 0xFF44, + 0xFF45, + 0xFF46, + 0xFF47, + 0xFF48, + 0xFF49, + 0xFF4A, + 0xFF4B, + 0xFF4C, + 0xFF4D, + 0xFF4E, + 0xFF4F, + 0xFF50, + 0xFF51, + 0xFF52, + 0xFF53, + 0xFF54, + 0xFF55, + 0xFF56, + 0xFF57, + 0xFF58, + 0xFF59, + 0xFF5A, + 0xFF5B, + 0xFF5C, + 0xFF5D, + 0xFF5E, + 0xFF5F, + 0xFF60, + 0xFF61, + 0xFF62, + 0xFF63, + 0xFF64, + 0xFF65, + 0xFF66, + 0xFF67, + 0xFF68, + 0xFF69, + 0xFF6A, + 0xFF6B, + 0xFF6C, + 0xFF6D, + 0xFF6E, + 0xFF6F, + 0xFF70, + 0xFF71, + 0xFF72, + 0xFF73, + 0xFF74, + 0xFF75, + 0xFF76, + 0xFF77, + 0xFF78, + 0xFF79, + 0xFF7A, + 0xFF7B, + 0xFF7C, + 0xFF7D, + 0xFF7E, + 0xFF7F, + 0xFF80, + 0xFF81, + 0xFF82, + 0xFF83, + 0xFF84, + 0xFF85, + 0xFF86, + 0xFF87, + 0xFF88, + 0xFF89, + 0xFF8A, + 0xFF8B, + 0xFF8C, + 0xFF8D, + 0xFF8E, + 0xFF8F, + 0xFF90, + 0xFF91, + 0xFF92, + 0xFF93, + 0xFF94, + 0xFF95, + 0xFF96, + 0xFF97, + 0xFF98, + 0xFF99, + 0xFF9A, + 0xFF9B, + 0xFF9C, + 0xFF9D, + 0xFF9E, + 0xFF9F, + 0xFFA0, + 0xFFA1, + 0xFFA2, + 0xFFA3, + 0xFFA4, + 0xFFA5, + 0xFFA6, + 0xFFA7, + 0xFFA8, + 0xFFA9, + 0xFFAA, + 0xFFAB, + 0xFFAC, + 0xFFAD, + 0xFFAE, + 0xFFAF, + 0xFFB0, + 0xFFB1, + 0xFFB2, + 0xFFB3, + 0xFFB4, + 0xFFB5, + 0xFFB6, + 0xFFB7, + 0xFFB8, + 0xFFB9, + 0xFFBA, + 0xFFBB, + 0xFFBC, + 0xFFBD, + 0xFFBE, + 0xFFBF, + 0xFFC2, + 0xFFC3, + 0xFFC4, + 0xFFC5, + 0xFFC6, + 0xFFC7, + 0xFFC8, + 0xFFCA, + 0xFFCB, + 0xFFCC, + 0xFFCD, + 0xFFCE, + 0xFFCF, + 0xFFD0, + 0xFFD2, + 0xFFD3, + 0xFFD4, + 0xFFD5, + 0xFFD6, + 0xFFD7, + 0xFFD8, + 0xFFDA, + 0xFFDB, + 0xFFDC, + 0xFFDD, + 0xFFE0, + 0xFFE1, + 0xFFE2, + 0xFFE3, + 0xFFE4, + 0xFFE5, + 0xFFE6, + 0xFFE7, + 0xFFE8, + 0xFFE9, + 0xFFEA, + 0xFFEB, + 0xFFEC, + 0xFFED, + 0xFFEE, + 0xFFEF, + 0x10000, + 0x1000C, + 0x1000D, + 0x10027, + 0x10028, + 0x1003B, + 0x1003C, + 0x1003E, + 0x1003F, + 0x1004E, + 0x10050, + 0x1005E, + 0x10080, + 0x100FB, + 0x10100, + 0x10103, + 0x10107, + 0x10134, + 0x10137, + 0x1018F, + 0x10190, + 0x1019D, + 0x101A0, + 0x101A1, + 0x101D0, + 0x101FE, + 0x10280, + 0x1029D, + 0x102A0, + 0x102D1, + 0x102E0, + 0x102FC, + 0x10300, + 0x10324, + 0x1032D, + 0x1034B, + 0x10350, + 0x1037B, + 0x10380, + 0x1039E, + 0x1039F, + 0x103C4, + 0x103C8, + 0x103D6, + 0x10400, + 0x10401, + 0x10402, + 0x10403, + 0x10404, + 0x10405, + 0x10406, + 0x10407, + 0x10408, + 0x10409, + 0x1040A, + 0x1040B, + 0x1040C, + 0x1040D, + 0x1040E, + 0x1040F, + 0x10410, + 0x10411, + 0x10412, + 0x10413, + 0x10414, + 0x10415, + 0x10416, + 0x10417, + 0x10418, + 0x10419, + 0x1041A, + 0x1041B, + 0x1041C, + 0x1041D, + 0x1041E, + 0x1041F, + 0x10420, + 0x10421, + 0x10422, + 0x10423, + 0x10424, + 0x10425, + 0x10426, + 0x10427, + 0x10428, + 0x1049E, + 0x104A0, + 0x104AA, + 0x104B0, + 0x104B1, + 0x104B2, + 0x104B3, + 0x104B4, + 0x104B5, + 0x104B6, + 0x104B7, + 0x104B8, + 0x104B9, + 0x104BA, + 0x104BB, + 0x104BC, + 0x104BD, + 0x104BE, + 0x104BF, + 0x104C0, + 0x104C1, + 0x104C2, + 0x104C3, + 0x104C4, + 0x104C5, + 0x104C6, + 0x104C7, + 0x104C8, + 0x104C9, + 0x104CA, + 0x104CB, + 0x104CC, + 0x104CD, + 0x104CE, + 0x104CF, + 0x104D0, + 0x104D1, + 0x104D2, + 0x104D3, + 0x104D4, + 0x104D8, + 0x104FC, + 0x10500, + 0x10528, + 0x10530, + 0x10564, + 0x1056F, + 0x10570, + 0x10571, + 0x10572, + 0x10573, + 0x10574, + 0x10575, + 0x10576, + 0x10577, + 0x10578, + 0x10579, + 0x1057A, + 0x1057B, + 0x1057C, + 0x1057D, + 0x1057E, + 0x1057F, + 0x10580, + 0x10581, + 0x10582, + 0x10583, + 0x10584, + 0x10585, + 0x10586, + 0x10587, + 0x10588, + 0x10589, + 0x1058A, + 0x1058B, + 0x1058C, + 0x1058D, + 0x1058E, + 0x1058F, + 0x10590, + 0x10591, + 0x10592, + 0x10593, + 0x10594, + 0x10595, + 0x10596, + 0x10597, + 0x105A2, + 0x105A3, + 0x105B2, + 0x105B3, + 0x105BA, + 0x105BB, + 0x105BD, + 0x105C0, + 0x105F4, + 0x10600, + 0x10737, + 0x10740, + 0x10756, + 0x10760, + 0x10768, + 0x10780, + 0x10781, + 0x10782, + 0x10783, + 0x10784, + 0x10785, + 0x10786, + 0x10787, + 0x10788, + 0x10789, + 0x1078A, + 0x1078B, + 0x1078C, + 0x1078D, + 0x1078E, + 0x1078F, + 0x10790, + 0x10791, + 0x10792, + 0x10793, + 0x10794, + 0x10795, + 0x10796, + 0x10797, + 0x10798, + 0x10799, + 0x1079A, + 0x1079B, + 0x1079C, + 0x1079D, + 0x1079E, + 0x1079F, + 0x107A0, + 0x107A1, + 0x107A2, + 0x107A3, + 0x107A4, + 0x107A5, + 0x107A6, + 0x107A7, + 0x107A8, + 0x107A9, + 0x107AA, + 0x107AB, + 0x107AC, + 0x107AD, + 0x107AE, + 0x107AF, + 0x107B0, + 0x107B1, + 0x107B2, + 0x107B3, + 0x107B4, + 0x107B5, + 0x107B6, + 0x107B7, + 0x107B8, + 0x107B9, + 0x107BA, + 0x107BB, + 0x10800, + 0x10806, + 0x10808, + 0x10809, + 0x1080A, + 0x10836, + 0x10837, + 0x10839, + 0x1083C, + 0x1083D, + 0x1083F, + 0x10856, + 0x10857, + 0x1089F, + 0x108A7, + 0x108B0, + 0x108E0, + 0x108F3, + 0x108F4, + 0x108F6, + 0x108FB, + 0x1091C, + 0x1091F, + 0x1093A, + 0x1093F, + 0x1095A, + 0x10980, + 0x109B8, + 0x109BC, + 0x109D0, + 0x109D2, + 0x10A04, + 0x10A05, + 0x10A07, + 0x10A0C, + 0x10A14, + 0x10A15, + 0x10A18, + 0x10A19, + 0x10A36, + 0x10A38, + 0x10A3B, + 0x10A3F, + 0x10A49, + 0x10A50, + 0x10A59, + 0x10A60, + 0x10AA0, + 0x10AC0, + 0x10AE7, + 0x10AEB, + 0x10AF7, + 0x10B00, + 0x10B36, + 0x10B39, + 0x10B56, + 0x10B58, + 0x10B73, + 0x10B78, + 0x10B92, + 0x10B99, + 0x10B9D, + 0x10BA9, + 0x10BB0, + 0x10C00, + 0x10C49, + 0x10C80, + 0x10C81, + 0x10C82, + 0x10C83, + 0x10C84, + 0x10C85, + 0x10C86, + 0x10C87, + 0x10C88, + 0x10C89, + 0x10C8A, + 0x10C8B, + 0x10C8C, + 0x10C8D, + 0x10C8E, + 0x10C8F, + 0x10C90, + 0x10C91, + 0x10C92, + 0x10C93, + 0x10C94, + 0x10C95, + 0x10C96, + 0x10C97, + 0x10C98, + 0x10C99, + 0x10C9A, + 0x10C9B, + 0x10C9C, + 0x10C9D, + 0x10C9E, + 0x10C9F, + 0x10CA0, + 0x10CA1, + 0x10CA2, + 0x10CA3, + 0x10CA4, + 0x10CA5, + 0x10CA6, + 0x10CA7, + 0x10CA8, + 0x10CA9, + 0x10CAA, + 0x10CAB, + 0x10CAC, + 0x10CAD, + 0x10CAE, + 0x10CAF, + 0x10CB0, + 0x10CB1, + 0x10CB2, + 0x10CB3, + 0x10CC0, + 0x10CF3, + 0x10CFA, + 0x10D28, + 0x10D30, + 0x10D3A, + 0x10D40, + 0x10D50, + 0x10D51, + 0x10D52, + 0x10D53, + 0x10D54, + 0x10D55, + 0x10D56, + 0x10D57, + 0x10D58, + 0x10D59, + 0x10D5A, + 0x10D5B, + 0x10D5C, + 0x10D5D, + 0x10D5E, + 0x10D5F, + 0x10D60, + 0x10D61, + 0x10D62, + 0x10D63, + 0x10D64, + 0x10D65, + 0x10D66, + 0x10D69, + 0x10D86, + 0x10D8E, + 0x10D90, + 0x10E60, + 0x10E7F, + 0x10E80, + 0x10EAA, + 0x10EAB, + 0x10EAE, + 0x10EB0, + 0x10EB2, + 0x10EC2, + 0x10EC8, + 0x10ED0, + 0x10ED9, + 0x10EFA, + 0x10F28, + 0x10F30, + 0x10F5A, + 0x10F70, + 0x10F8A, + 0x10FB0, + 0x10FCC, + 0x10FE0, + 0x10FF7, + 0x11000, + 0x1104E, + 0x11052, + 0x11076, + 0x1107F, + 0x110BD, + 0x110BE, + 0x110C3, + 0x110D0, + 0x110E9, + 0x110F0, + 0x110FA, + 0x11100, + 0x11135, + 0x11136, + 0x11148, + 0x11150, + 0x11177, + 0x11180, + 0x111E0, + 0x111E1, + 0x111F5, + 0x11200, + 0x11212, + 0x11213, + 0x11242, + 0x11280, + 0x11287, + 0x11288, + 0x11289, + 0x1128A, + 0x1128E, + 0x1128F, + 0x1129E, + 0x1129F, + 0x112AA, + 0x112B0, + 0x112EB, + 0x112F0, + 0x112FA, + 0x11300, + 0x11304, + 0x11305, + 0x1130D, + 0x1130F, + 0x11311, + 0x11313, + 0x11329, + 0x1132A, + 0x11331, + 0x11332, + 0x11334, + 0x11335, + 0x1133A, + 0x1133B, + 0x11345, + 0x11347, + 0x11349, + 0x1134B, + 0x1134E, + 0x11350, + 0x11351, + 0x11357, + 0x11358, + 0x1135D, + 0x11364, + 0x11366, + 0x1136D, + 0x11370, + 0x11375, + 0x11380, + 0x1138A, + 0x1138B, + 0x1138C, + 0x1138E, + 0x1138F, + 0x11390, + 0x113B6, + 0x113B7, + 0x113C1, + 0x113C2, + 0x113C3, + 0x113C5, + 0x113C6, + 0x113C7, + 0x113CB, + 0x113CC, + 0x113D6, + 0x113D7, + 0x113D9, + 0x113E1, + 0x113E3, + 0x11400, + 0x1145C, + 0x1145D, + 0x11462, + 0x11480, + 0x114C8, + 0x114D0, + 0x114DA, + 0x11580, + 0x115B6, + 0x115B8, + 0x115DE, + 0x11600, + 0x11645, + 0x11650, + 0x1165A, + 0x11660, + 0x1166D, + 0x11680, + 0x116BA, + 0x116C0, + 0x116CA, + 0x116D0, + 0x116E4, + 0x11700, + 0x1171B, + 0x1171D, + 0x1172C, + 0x11730, + 0x11747, + 0x11800, + 0x1183C, + 0x118A0, + 0x118A1, + 0x118A2, + 0x118A3, + 0x118A4, + 0x118A5, + 0x118A6, + 0x118A7, + 0x118A8, + 0x118A9, + 0x118AA, + 0x118AB, + 0x118AC, + 0x118AD, + 0x118AE, + 0x118AF, + 0x118B0, + 0x118B1, + 0x118B2, + 0x118B3, + 0x118B4, + 0x118B5, + 0x118B6, + 0x118B7, + 0x118B8, + 0x118B9, + 0x118BA, + 0x118BB, + 0x118BC, + 0x118BD, + 0x118BE, + 0x118BF, + 0x118C0, + 0x118F3, + 0x118FF, + 0x11907, + 0x11909, + 0x1190A, + 0x1190C, + 0x11914, + 0x11915, + 0x11917, + 0x11918, + 0x11936, + 0x11937, + 0x11939, + 0x1193B, + 0x11947, + 0x11950, + 0x1195A, + 0x119A0, + 0x119A8, + 0x119AA, + 0x119D8, + 0x119DA, + 0x119E5, + 0x11A00, + 0x11A48, + 0x11A50, + 0x11AA3, + 0x11AB0, + 0x11AF9, + 0x11B00, + 0x11B0A, + 0x11B60, + 0x11B68, + 0x11BC0, + 0x11BE2, + 0x11BF0, + 0x11BFA, + 0x11C00, + 0x11C09, + 0x11C0A, + 0x11C37, + 0x11C38, + 0x11C46, + 0x11C50, + 0x11C6D, + 0x11C70, + 0x11C90, + 0x11C92, + 0x11CA8, + 0x11CA9, + 0x11CB7, + 0x11D00, + 0x11D07, + 0x11D08, + 0x11D0A, + 0x11D0B, + 0x11D37, + 0x11D3A, + 0x11D3B, + 0x11D3C, + 0x11D3E, + 0x11D3F, + 0x11D48, + 0x11D50, + 0x11D5A, + 0x11D60, + 0x11D66, + 0x11D67, + 0x11D69, + 0x11D6A, + 0x11D8F, + 0x11D90, + 0x11D92, + 0x11D93, + 0x11D99, + 0x11DA0, + 0x11DAA, + 0x11DB0, + 0x11DDC, + 0x11DE0, + 0x11DEA, + 0x11EE0, + 0x11EF9, + 0x11F00, + 0x11F11, + 0x11F12, + 0x11F3B, + 0x11F3E, + 0x11F5B, + 0x11FB0, + 0x11FB1, + 0x11FC0, + 0x11FF2, + 0x11FFF, + 0x1239A, + 0x12400, + 0x1246F, + 0x12470, + 0x12475, + 0x12480, + 0x12544, + 0x12F90, + 0x12FF3, + 0x13000, + 0x13430, + 0x13440, + 0x13456, + 0x13460, + 0x143FB, + 0x14400, + 0x14647, + 0x16100, + 0x1613A, + 0x16800, + 0x16A39, + 0x16A40, + 0x16A5F, + 0x16A60, + 0x16A6A, + 0x16A6E, + 0x16ABF, + 0x16AC0, + 0x16ACA, + 0x16AD0, + 0x16AEE, + 0x16AF0, + 0x16AF6, + 0x16B00, + 0x16B46, + 0x16B50, + 0x16B5A, + 0x16B5B, + 0x16B62, + 0x16B63, + 0x16B78, + 0x16B7D, + 0x16B90, + 0x16D40, + 0x16D7A, + 0x16E40, + 0x16E41, + 0x16E42, + 0x16E43, + 0x16E44, + 0x16E45, + 0x16E46, + 0x16E47, + 0x16E48, + 0x16E49, + 0x16E4A, + 0x16E4B, + 0x16E4C, + 0x16E4D, + 0x16E4E, + 0x16E4F, + 0x16E50, + 0x16E51, + 0x16E52, + 0x16E53, + 0x16E54, + 0x16E55, + 0x16E56, + 0x16E57, + 0x16E58, + 0x16E59, + 0x16E5A, + 0x16E5B, + 0x16E5C, + 0x16E5D, + 0x16E5E, + 0x16E5F, + 0x16E60, + 0x16E9B, + 0x16EA0, + 0x16EA1, + 0x16EA2, + 0x16EA3, + 0x16EA4, + 0x16EA5, + 0x16EA6, + 0x16EA7, + 0x16EA8, + 0x16EA9, + 0x16EAA, + 0x16EAB, + 0x16EAC, + 0x16EAD, + 0x16EAE, + 0x16EAF, + 0x16EB0, + 0x16EB1, + 0x16EB2, + 0x16EB3, + 0x16EB4, + 0x16EB5, + 0x16EB6, + 0x16EB7, + 0x16EB8, + 0x16EB9, + 0x16EBB, + 0x16ED4, + 0x16F00, + 0x16F4B, + 0x16F4F, + 0x16F88, + 0x16F8F, + 0x16FA0, + 0x16FE0, + 0x16FE5, + 0x16FF0, + 0x16FF7, + 0x17000, + 0x18CD6, + 0x18CFF, + 0x18D1F, + 0x18D80, + 0x18DF3, + 0x1AFF0, + 0x1AFF4, + 0x1AFF5, + 0x1AFFC, + 0x1AFFD, + 0x1AFFF, + 0x1B000, + 0x1B123, + 0x1B132, + 0x1B133, + 0x1B150, + 0x1B153, + 0x1B155, + 0x1B156, + 0x1B164, + 0x1B168, + 0x1B170, + 0x1B2FC, + 0x1BC00, + 0x1BC6B, + 0x1BC70, + 0x1BC7D, + 0x1BC80, + 0x1BC89, + 0x1BC90, + 0x1BC9A, + 0x1BC9C, + 0x1BCA0, + 0x1BCA4, + 0x1CC00, + 0x1CCD6, + 0x1CCD7, + 0x1CCD8, + 0x1CCD9, + 0x1CCDA, + 0x1CCDB, + 0x1CCDC, + 0x1CCDD, + 0x1CCDE, + 0x1CCDF, + 0x1CCE0, + 0x1CCE1, + 0x1CCE2, + 0x1CCE3, + 0x1CCE4, + 0x1CCE5, + 0x1CCE6, + 0x1CCE7, + 0x1CCE8, + 0x1CCE9, + 0x1CCEA, + 0x1CCEB, + 0x1CCEC, + 0x1CCED, + 0x1CCEE, + 0x1CCEF, + 0x1CCF0, + 0x1CCF1, + 0x1CCF2, + 0x1CCF3, + 0x1CCF4, + 0x1CCF5, + 0x1CCF6, + 0x1CCF7, + 0x1CCF8, + 0x1CCF9, + 0x1CCFA, + 0x1CCFD, + 0x1CD00, + 0x1CEB4, + 0x1CEBA, + 0x1CED1, + 0x1CEE0, + 0x1CEF1, + 0x1CF00, + 0x1CF2E, + 0x1CF30, + 0x1CF47, + 0x1CF50, + 0x1CFC4, + 0x1D000, + 0x1D0F6, + 0x1D100, + 0x1D127, + 0x1D129, + 0x1D15E, + 0x1D15F, + 0x1D160, + 0x1D161, + 0x1D162, + 0x1D163, + 0x1D164, + 0x1D165, + 0x1D173, + 0x1D17B, + 0x1D1BB, + 0x1D1BC, + 0x1D1BD, + 0x1D1BE, + 0x1D1BF, + 0x1D1C0, + 0x1D1C1, + 0x1D1EB, + 0x1D200, + 0x1D246, + 0x1D2C0, + 0x1D2D4, + 0x1D2E0, + 0x1D2F4, + 0x1D300, + 0x1D357, + 0x1D360, + 0x1D379, + 0x1D400, + 0x1D401, + 0x1D402, + 0x1D403, + 0x1D404, + 0x1D405, + 0x1D406, + 0x1D407, + 0x1D408, + 0x1D409, + 0x1D40A, + 0x1D40B, + 0x1D40C, + 0x1D40D, + 0x1D40E, + 0x1D40F, + 0x1D410, + 0x1D411, + 0x1D412, + 0x1D413, + 0x1D414, + 0x1D415, + 0x1D416, + 0x1D417, + 0x1D418, + 0x1D419, + 0x1D41A, + 0x1D41B, + 0x1D41C, + 0x1D41D, + 0x1D41E, + 0x1D41F, + 0x1D420, + 0x1D421, + 0x1D422, + 0x1D423, + 0x1D424, + 0x1D425, + 0x1D426, + 0x1D427, + 0x1D428, + 0x1D429, + 0x1D42A, + 0x1D42B, + 0x1D42C, + 0x1D42D, + 0x1D42E, + 0x1D42F, + 0x1D430, + 0x1D431, + 0x1D432, + 0x1D433, + 0x1D434, + 0x1D435, + 0x1D436, + 0x1D437, + 0x1D438, + 0x1D439, + 0x1D43A, + 0x1D43B, + 0x1D43C, + 0x1D43D, + 0x1D43E, + 0x1D43F, + 0x1D440, + 0x1D441, + 0x1D442, + 0x1D443, + 0x1D444, + 0x1D445, + 0x1D446, + 0x1D447, + 0x1D448, + 0x1D449, + 0x1D44A, + 0x1D44B, + 0x1D44C, + 0x1D44D, + 0x1D44E, + 0x1D44F, + 0x1D450, + 0x1D451, + 0x1D452, + 0x1D453, + 0x1D454, + 0x1D455, + 0x1D456, + 0x1D457, + 0x1D458, + 0x1D459, + 0x1D45A, + 0x1D45B, + 0x1D45C, + 0x1D45D, + 0x1D45E, + 0x1D45F, + 0x1D460, + 0x1D461, + 0x1D462, + 0x1D463, + 0x1D464, + 0x1D465, + 0x1D466, + 0x1D467, + 0x1D468, + 0x1D469, + 0x1D46A, + 0x1D46B, + 0x1D46C, + 0x1D46D, + 0x1D46E, + 0x1D46F, + 0x1D470, + 0x1D471, + 0x1D472, + 0x1D473, + 0x1D474, + 0x1D475, + 0x1D476, + 0x1D477, + 0x1D478, + 0x1D479, + 0x1D47A, + 0x1D47B, + 0x1D47C, + 0x1D47D, + 0x1D47E, + 0x1D47F, + 0x1D480, + 0x1D481, + 0x1D482, + 0x1D483, + 0x1D484, + 0x1D485, + 0x1D486, + 0x1D487, + 0x1D488, + 0x1D489, + 0x1D48A, + 0x1D48B, + 0x1D48C, + 0x1D48D, + 0x1D48E, + 0x1D48F, + 0x1D490, + 0x1D491, + 0x1D492, + 0x1D493, + 0x1D494, + 0x1D495, + 0x1D496, + 0x1D497, + 0x1D498, + 0x1D499, + 0x1D49A, + 0x1D49B, + 0x1D49C, + 0x1D49D, + 0x1D49E, + 0x1D49F, + 0x1D4A0, + 0x1D4A2, + 0x1D4A3, + 0x1D4A5, + 0x1D4A6, + 0x1D4A7, + 0x1D4A9, + 0x1D4AA, + 0x1D4AB, + 0x1D4AC, + 0x1D4AD, + 0x1D4AE, + 0x1D4AF, + 0x1D4B0, + 0x1D4B1, + 0x1D4B2, + 0x1D4B3, + 0x1D4B4, + 0x1D4B5, + 0x1D4B6, + 0x1D4B7, + 0x1D4B8, + 0x1D4B9, + 0x1D4BA, + 0x1D4BB, + 0x1D4BC, + 0x1D4BD, + 0x1D4BE, + 0x1D4BF, + 0x1D4C0, + 0x1D4C1, + 0x1D4C2, + 0x1D4C3, + 0x1D4C4, + 0x1D4C5, + 0x1D4C6, + 0x1D4C7, + 0x1D4C8, + 0x1D4C9, + 0x1D4CA, + 0x1D4CB, + 0x1D4CC, + 0x1D4CD, + 0x1D4CE, + 0x1D4CF, + 0x1D4D0, + 0x1D4D1, + 0x1D4D2, + 0x1D4D3, + 0x1D4D4, + 0x1D4D5, + 0x1D4D6, + 0x1D4D7, + 0x1D4D8, + 0x1D4D9, + 0x1D4DA, + 0x1D4DB, + 0x1D4DC, + 0x1D4DD, + 0x1D4DE, + 0x1D4DF, + 0x1D4E0, + 0x1D4E1, + 0x1D4E2, + 0x1D4E3, + 0x1D4E4, + 0x1D4E5, + 0x1D4E6, + 0x1D4E7, + 0x1D4E8, + 0x1D4E9, + 0x1D4EA, + 0x1D4EB, + 0x1D4EC, + 0x1D4ED, + 0x1D4EE, + 0x1D4EF, + 0x1D4F0, + 0x1D4F1, + 0x1D4F2, + 0x1D4F3, + 0x1D4F4, + 0x1D4F5, + 0x1D4F6, + 0x1D4F7, + 0x1D4F8, + 0x1D4F9, + 0x1D4FA, + 0x1D4FB, + 0x1D4FC, + 0x1D4FD, + 0x1D4FE, + 0x1D4FF, + 0x1D500, + 0x1D501, + 0x1D502, + 0x1D503, + 0x1D504, + 0x1D505, + 0x1D506, + 0x1D507, + 0x1D508, + 0x1D509, + 0x1D50A, + 0x1D50B, + 0x1D50D, + 0x1D50E, + 0x1D50F, + 0x1D510, + 0x1D511, + 0x1D512, + 0x1D513, + 0x1D514, + 0x1D515, + 0x1D516, + 0x1D517, + 0x1D518, + 0x1D519, + 0x1D51A, + 0x1D51B, + 0x1D51C, + 0x1D51D, + 0x1D51E, + 0x1D51F, + 0x1D520, + 0x1D521, + 0x1D522, + 0x1D523, + 0x1D524, + 0x1D525, + 0x1D526, + 0x1D527, + 0x1D528, + 0x1D529, + 0x1D52A, + 0x1D52B, + 0x1D52C, + 0x1D52D, + 0x1D52E, + 0x1D52F, + 0x1D530, + 0x1D531, + 0x1D532, + 0x1D533, + 0x1D534, + 0x1D535, + 0x1D536, + 0x1D537, + 0x1D538, + 0x1D539, + 0x1D53A, + 0x1D53B, + 0x1D53C, + 0x1D53D, + 0x1D53E, + 0x1D53F, + 0x1D540, + 0x1D541, + 0x1D542, + 0x1D543, + 0x1D544, + 0x1D545, + 0x1D546, + 0x1D547, + 0x1D54A, + 0x1D54B, + 0x1D54C, + 0x1D54D, + 0x1D54E, + 0x1D54F, + 0x1D550, + 0x1D551, + 0x1D552, + 0x1D553, + 0x1D554, + 0x1D555, + 0x1D556, + 0x1D557, + 0x1D558, + 0x1D559, + 0x1D55A, + 0x1D55B, + 0x1D55C, + 0x1D55D, + 0x1D55E, + 0x1D55F, + 0x1D560, + 0x1D561, + 0x1D562, + 0x1D563, + 0x1D564, + 0x1D565, + 0x1D566, + 0x1D567, + 0x1D568, + 0x1D569, + 0x1D56A, + 0x1D56B, + 0x1D56C, + 0x1D56D, + 0x1D56E, + 0x1D56F, + 0x1D570, + 0x1D571, + 0x1D572, + 0x1D573, + 0x1D574, + 0x1D575, + 0x1D576, + 0x1D577, + 0x1D578, + 0x1D579, + 0x1D57A, + 0x1D57B, + 0x1D57C, + 0x1D57D, + 0x1D57E, + 0x1D57F, + 0x1D580, + 0x1D581, + 0x1D582, + 0x1D583, + 0x1D584, + 0x1D585, + 0x1D586, + 0x1D587, + 0x1D588, + 0x1D589, + 0x1D58A, + 0x1D58B, + 0x1D58C, + 0x1D58D, + 0x1D58E, + 0x1D58F, + 0x1D590, + 0x1D591, + 0x1D592, + 0x1D593, + 0x1D594, + 0x1D595, + 0x1D596, + 0x1D597, + 0x1D598, + 0x1D599, + 0x1D59A, + 0x1D59B, + 0x1D59C, + 0x1D59D, + 0x1D59E, + 0x1D59F, + 0x1D5A0, + 0x1D5A1, + 0x1D5A2, + 0x1D5A3, + 0x1D5A4, + 0x1D5A5, + 0x1D5A6, + 0x1D5A7, + 0x1D5A8, + 0x1D5A9, + 0x1D5AA, + 0x1D5AB, + 0x1D5AC, + 0x1D5AD, + 0x1D5AE, + 0x1D5AF, + 0x1D5B0, + 0x1D5B1, + 0x1D5B2, + 0x1D5B3, + 0x1D5B4, + 0x1D5B5, + 0x1D5B6, + 0x1D5B7, + 0x1D5B8, + 0x1D5B9, + 0x1D5BA, + 0x1D5BB, + 0x1D5BC, + 0x1D5BD, + 0x1D5BE, + 0x1D5BF, + 0x1D5C0, + 0x1D5C1, + 0x1D5C2, + 0x1D5C3, + 0x1D5C4, + 0x1D5C5, + 0x1D5C6, + 0x1D5C7, + 0x1D5C8, + 0x1D5C9, + 0x1D5CA, + 0x1D5CB, + 0x1D5CC, + 0x1D5CD, + 0x1D5CE, + 0x1D5CF, + 0x1D5D0, + 0x1D5D1, + 0x1D5D2, + 0x1D5D3, + 0x1D5D4, + 0x1D5D5, + 0x1D5D6, + 0x1D5D7, + 0x1D5D8, + 0x1D5D9, + 0x1D5DA, + 0x1D5DB, + 0x1D5DC, + 0x1D5DD, + 0x1D5DE, + 0x1D5DF, + 0x1D5E0, + 0x1D5E1, + 0x1D5E2, + 0x1D5E3, + 0x1D5E4, + 0x1D5E5, + 0x1D5E6, + 0x1D5E7, + 0x1D5E8, + 0x1D5E9, + 0x1D5EA, + 0x1D5EB, + 0x1D5EC, + 0x1D5ED, + 0x1D5EE, + 0x1D5EF, + 0x1D5F0, + 0x1D5F1, + 0x1D5F2, + 0x1D5F3, + 0x1D5F4, + 0x1D5F5, + 0x1D5F6, + 0x1D5F7, + 0x1D5F8, + 0x1D5F9, + 0x1D5FA, + 0x1D5FB, + 0x1D5FC, + 0x1D5FD, + 0x1D5FE, + 0x1D5FF, + 0x1D600, + 0x1D601, + 0x1D602, + 0x1D603, + 0x1D604, + 0x1D605, + 0x1D606, + 0x1D607, + 0x1D608, + 0x1D609, + 0x1D60A, + 0x1D60B, + 0x1D60C, + 0x1D60D, + 0x1D60E, + 0x1D60F, + 0x1D610, + 0x1D611, + 0x1D612, + 0x1D613, + 0x1D614, + 0x1D615, + 0x1D616, + 0x1D617, + 0x1D618, + 0x1D619, + 0x1D61A, + 0x1D61B, + 0x1D61C, + 0x1D61D, + 0x1D61E, + 0x1D61F, + 0x1D620, + 0x1D621, + 0x1D622, + 0x1D623, + 0x1D624, + 0x1D625, + 0x1D626, + 0x1D627, + 0x1D628, + 0x1D629, + 0x1D62A, + 0x1D62B, + 0x1D62C, + 0x1D62D, + 0x1D62E, + 0x1D62F, + 0x1D630, + 0x1D631, + 0x1D632, + 0x1D633, + 0x1D634, + 0x1D635, + 0x1D636, + 0x1D637, + 0x1D638, + 0x1D639, + 0x1D63A, + 0x1D63B, + 0x1D63C, + 0x1D63D, + 0x1D63E, + 0x1D63F, + 0x1D640, + 0x1D641, + 0x1D642, + 0x1D643, + 0x1D644, + 0x1D645, + 0x1D646, + 0x1D647, + 0x1D648, + 0x1D649, + 0x1D64A, + 0x1D64B, + 0x1D64C, + 0x1D64D, + 0x1D64E, + 0x1D64F, + 0x1D650, + 0x1D651, + 0x1D652, + 0x1D653, + 0x1D654, + 0x1D655, + 0x1D656, + 0x1D657, + 0x1D658, + 0x1D659, + 0x1D65A, + 0x1D65B, + 0x1D65C, + 0x1D65D, + 0x1D65E, + 0x1D65F, + 0x1D660, + 0x1D661, + 0x1D662, + 0x1D663, + 0x1D664, + 0x1D665, + 0x1D666, + 0x1D667, + 0x1D668, + 0x1D669, + 0x1D66A, + 0x1D66B, + 0x1D66C, + 0x1D66D, + 0x1D66E, + 0x1D66F, + 0x1D670, + 0x1D671, + 0x1D672, + 0x1D673, + 0x1D674, + 0x1D675, + 0x1D676, + 0x1D677, + 0x1D678, + 0x1D679, + 0x1D67A, + 0x1D67B, + 0x1D67C, + 0x1D67D, + 0x1D67E, + 0x1D67F, + 0x1D680, + 0x1D681, + 0x1D682, + 0x1D683, + 0x1D684, + 0x1D685, + 0x1D686, + 0x1D687, + 0x1D688, + 0x1D689, + 0x1D68A, + 0x1D68B, + 0x1D68C, + 0x1D68D, + 0x1D68E, + 0x1D68F, + 0x1D690, + 0x1D691, + 0x1D692, + 0x1D693, + 0x1D694, + 0x1D695, + 0x1D696, + 0x1D697, + 0x1D698, + 0x1D699, + 0x1D69A, + 0x1D69B, + 0x1D69C, + 0x1D69D, + 0x1D69E, + 0x1D69F, + 0x1D6A0, + 0x1D6A1, + 0x1D6A2, + 0x1D6A3, + 0x1D6A4, + 0x1D6A5, + 0x1D6A6, + 0x1D6A8, + 0x1D6A9, + 0x1D6AA, + 0x1D6AB, + 0x1D6AC, + 0x1D6AD, + 0x1D6AE, + 0x1D6AF, + 0x1D6B0, + 0x1D6B1, + 0x1D6B2, + 0x1D6B3, + 0x1D6B4, + 0x1D6B5, + 0x1D6B6, + 0x1D6B7, + 0x1D6B8, + 0x1D6B9, + 0x1D6BA, + 0x1D6BB, + 0x1D6BC, + 0x1D6BD, + 0x1D6BE, + 0x1D6BF, + 0x1D6C0, + 0x1D6C1, + 0x1D6C2, + 0x1D6C3, + 0x1D6C4, + 0x1D6C5, + 0x1D6C6, + 0x1D6C7, + 0x1D6C8, + 0x1D6C9, + 0x1D6CA, + 0x1D6CB, + 0x1D6CC, + 0x1D6CD, + 0x1D6CE, + 0x1D6CF, + 0x1D6D0, + 0x1D6D1, + 0x1D6D2, + 0x1D6D3, + 0x1D6D5, + 0x1D6D6, + 0x1D6D7, + 0x1D6D8, + 0x1D6D9, + 0x1D6DA, + 0x1D6DB, + 0x1D6DC, + 0x1D6DD, + 0x1D6DE, + 0x1D6DF, + 0x1D6E0, + 0x1D6E1, + 0x1D6E2, + 0x1D6E3, + 0x1D6E4, + 0x1D6E5, + 0x1D6E6, + 0x1D6E7, + 0x1D6E8, + 0x1D6E9, + 0x1D6EA, + 0x1D6EB, + 0x1D6EC, + 0x1D6ED, + 0x1D6EE, + 0x1D6EF, + 0x1D6F0, + 0x1D6F1, + 0x1D6F2, + 0x1D6F3, + 0x1D6F4, + 0x1D6F5, + 0x1D6F6, + 0x1D6F7, + 0x1D6F8, + 0x1D6F9, + 0x1D6FA, + 0x1D6FB, + 0x1D6FC, + 0x1D6FD, + 0x1D6FE, + 0x1D6FF, + 0x1D700, + 0x1D701, + 0x1D702, + 0x1D703, + 0x1D704, + 0x1D705, + 0x1D706, + 0x1D707, + 0x1D708, + 0x1D709, + 0x1D70A, + 0x1D70B, + 0x1D70C, + 0x1D70D, + 0x1D70F, + 0x1D710, + 0x1D711, + 0x1D712, + 0x1D713, + 0x1D714, + 0x1D715, + 0x1D716, + 0x1D717, + 0x1D718, + 0x1D719, + 0x1D71A, + 0x1D71B, + 0x1D71C, + 0x1D71D, + 0x1D71E, + 0x1D71F, + 0x1D720, + 0x1D721, + 0x1D722, + 0x1D723, + 0x1D724, + 0x1D725, + 0x1D726, + 0x1D727, + 0x1D728, + 0x1D729, + 0x1D72A, + 0x1D72B, + 0x1D72C, + 0x1D72D, + 0x1D72E, + 0x1D72F, + 0x1D730, + 0x1D731, + 0x1D732, + 0x1D733, + 0x1D734, + 0x1D735, + 0x1D736, + 0x1D737, + 0x1D738, + 0x1D739, + 0x1D73A, + 0x1D73B, + 0x1D73C, + 0x1D73D, + 0x1D73E, + 0x1D73F, + 0x1D740, + 0x1D741, + 0x1D742, + 0x1D743, + 0x1D744, + 0x1D745, + 0x1D746, + 0x1D747, + 0x1D749, + 0x1D74A, + 0x1D74B, + 0x1D74C, + 0x1D74D, + 0x1D74E, + 0x1D74F, + 0x1D750, + 0x1D751, + 0x1D752, + 0x1D753, + 0x1D754, + 0x1D755, + 0x1D756, + 0x1D757, + 0x1D758, + 0x1D759, + 0x1D75A, + 0x1D75B, + 0x1D75C, + 0x1D75D, + 0x1D75E, + 0x1D75F, + 0x1D760, + 0x1D761, + 0x1D762, + 0x1D763, + 0x1D764, + 0x1D765, + 0x1D766, + 0x1D767, + 0x1D768, + 0x1D769, + 0x1D76A, + 0x1D76B, + 0x1D76C, + 0x1D76D, + 0x1D76E, + 0x1D76F, + 0x1D770, + 0x1D771, + 0x1D772, + 0x1D773, + 0x1D774, + 0x1D775, + 0x1D776, + 0x1D777, + 0x1D778, + 0x1D779, + 0x1D77A, + 0x1D77B, + 0x1D77C, + 0x1D77D, + 0x1D77E, + 0x1D77F, + 0x1D780, + 0x1D781, + 0x1D783, + 0x1D784, + 0x1D785, + 0x1D786, + 0x1D787, + 0x1D788, + 0x1D789, + 0x1D78A, + 0x1D78B, + 0x1D78C, + 0x1D78D, + 0x1D78E, + 0x1D78F, + 0x1D790, + 0x1D791, + 0x1D792, + 0x1D793, + 0x1D794, + 0x1D795, + 0x1D796, + 0x1D797, + 0x1D798, + 0x1D799, + 0x1D79A, + 0x1D79B, + 0x1D79C, + 0x1D79D, + 0x1D79E, + 0x1D79F, + 0x1D7A0, + 0x1D7A1, + 0x1D7A2, + 0x1D7A3, + 0x1D7A4, + 0x1D7A5, + 0x1D7A6, + 0x1D7A7, + 0x1D7A8, + 0x1D7A9, + 0x1D7AA, + 0x1D7AB, + 0x1D7AC, + 0x1D7AD, + 0x1D7AE, + 0x1D7AF, + 0x1D7B0, + 0x1D7B1, + 0x1D7B2, + 0x1D7B3, + 0x1D7B4, + 0x1D7B5, + 0x1D7B6, + 0x1D7B7, + 0x1D7B8, + 0x1D7B9, + 0x1D7BA, + 0x1D7BB, + 0x1D7BD, + 0x1D7BE, + 0x1D7BF, + 0x1D7C0, + 0x1D7C1, + 0x1D7C2, + 0x1D7C3, + 0x1D7C4, + 0x1D7C5, + 0x1D7C6, + 0x1D7C7, + 0x1D7C8, + 0x1D7C9, + 0x1D7CA, + 0x1D7CC, + 0x1D7CE, + 0x1D7CF, + 0x1D7D0, + 0x1D7D1, + 0x1D7D2, + 0x1D7D3, + 0x1D7D4, + 0x1D7D5, + 0x1D7D6, + 0x1D7D7, + 0x1D7D8, + 0x1D7D9, + 0x1D7DA, + 0x1D7DB, + 0x1D7DC, + 0x1D7DD, + 0x1D7DE, + 0x1D7DF, + 0x1D7E0, + 0x1D7E1, + 0x1D7E2, + 0x1D7E3, + 0x1D7E4, + 0x1D7E5, + 0x1D7E6, + 0x1D7E7, + 0x1D7E8, + 0x1D7E9, + 0x1D7EA, + 0x1D7EB, + 0x1D7EC, + 0x1D7ED, + 0x1D7EE, + 0x1D7EF, + 0x1D7F0, + 0x1D7F1, + 0x1D7F2, + 0x1D7F3, + 0x1D7F4, + 0x1D7F5, + 0x1D7F6, + 0x1D7F7, + 0x1D7F8, + 0x1D7F9, + 0x1D7FA, + 0x1D7FB, + 0x1D7FC, + 0x1D7FD, + 0x1D7FE, + 0x1D7FF, + 0x1D800, + 0x1DA8C, + 0x1DA9B, + 0x1DAA0, + 0x1DAA1, + 0x1DAB0, + 0x1DF00, + 0x1DF1F, + 0x1DF25, + 0x1DF2B, + 0x1E000, + 0x1E007, + 0x1E008, + 0x1E019, + 0x1E01B, + 0x1E022, + 0x1E023, + 0x1E025, + 0x1E026, + 0x1E02B, + 0x1E030, + 0x1E031, + 0x1E032, + 0x1E033, + 0x1E034, + 0x1E035, + 0x1E036, + 0x1E037, + 0x1E038, + 0x1E039, + 0x1E03A, + 0x1E03B, + 0x1E03C, + 0x1E03D, + 0x1E03E, + 0x1E03F, + 0x1E040, + 0x1E041, + 0x1E042, + 0x1E043, + 0x1E044, + 0x1E045, + 0x1E046, + 0x1E047, + 0x1E048, + 0x1E049, + 0x1E04A, + 0x1E04B, + 0x1E04C, + 0x1E04D, + 0x1E04E, + 0x1E04F, + 0x1E050, + 0x1E051, + 0x1E052, + 0x1E053, + 0x1E054, + 0x1E055, + 0x1E056, + 0x1E057, + 0x1E058, + 0x1E059, + 0x1E05A, + 0x1E05B, + 0x1E05C, + 0x1E05D, + 0x1E05E, + 0x1E05F, + 0x1E060, + 0x1E061, + 0x1E062, + 0x1E063, + 0x1E064, + 0x1E065, + 0x1E066, + 0x1E067, + 0x1E068, + 0x1E069, + 0x1E06A, + 0x1E06B, + 0x1E06C, + 0x1E06D, + 0x1E06E, + 0x1E08F, + 0x1E090, + 0x1E100, + 0x1E12D, + 0x1E130, + 0x1E13E, + 0x1E140, + 0x1E14A, + 0x1E14E, + 0x1E150, + 0x1E290, + 0x1E2AF, + 0x1E2C0, + 0x1E2FA, + 0x1E2FF, + 0x1E300, + 0x1E4D0, + 0x1E4FA, + 0x1E5D0, + 0x1E5FB, + 0x1E5FF, + 0x1E600, + 0x1E6C0, + 0x1E6DF, + 0x1E6E0, + 0x1E6F6, + 0x1E6FE, + 0x1E700, + 0x1E7E0, + 0x1E7E7, + 0x1E7E8, + 0x1E7EC, + 0x1E7ED, + 0x1E7EF, + 0x1E7F0, + 0x1E7FF, + 0x1E800, + 0x1E8C5, + 0x1E8C7, + 0x1E8D7, + 0x1E900, + 0x1E901, + 0x1E902, + 0x1E903, + 0x1E904, + 0x1E905, + 0x1E906, + 0x1E907, + 0x1E908, + 0x1E909, + 0x1E90A, + 0x1E90B, + 0x1E90C, + 0x1E90D, + 0x1E90E, + 0x1E90F, + 0x1E910, + 0x1E911, + 0x1E912, + 0x1E913, + 0x1E914, + 0x1E915, + 0x1E916, + 0x1E917, + 0x1E918, + 0x1E919, + 0x1E91A, + 0x1E91B, + 0x1E91C, + 0x1E91D, + 0x1E91E, + 0x1E91F, + 0x1E920, + 0x1E921, + 0x1E922, + 0x1E94C, + 0x1E950, + 0x1E95A, + 0x1E95E, + 0x1E960, + 0x1EC71, + 0x1ECB5, + 0x1ED01, + 0x1ED3E, + 0x1EE00, + 0x1EE01, + 0x1EE02, + 0x1EE03, + 0x1EE04, + 0x1EE05, + 0x1EE06, + 0x1EE07, + 0x1EE08, + 0x1EE09, + 0x1EE0A, + 0x1EE0B, + 0x1EE0C, + 0x1EE0D, + 0x1EE0E, + 0x1EE0F, + 0x1EE10, + 0x1EE11, + 0x1EE12, + 0x1EE13, + 0x1EE14, + 0x1EE15, + 0x1EE16, + 0x1EE17, + 0x1EE18, + 0x1EE19, + 0x1EE1A, + 0x1EE1B, + 0x1EE1C, + 0x1EE1D, + 0x1EE1E, + 0x1EE1F, + 0x1EE20, + 0x1EE21, + 0x1EE22, + 0x1EE23, + 0x1EE24, + 0x1EE25, + 0x1EE27, + 0x1EE28, + 0x1EE29, + 0x1EE2A, + 0x1EE2B, + 0x1EE2C, + 0x1EE2D, + 0x1EE2E, + 0x1EE2F, + 0x1EE30, + 0x1EE31, + 0x1EE32, + 0x1EE33, + 0x1EE34, + 0x1EE35, + 0x1EE36, + 0x1EE37, + 0x1EE38, + 0x1EE39, + 0x1EE3A, + 0x1EE3B, + 0x1EE3C, + 0x1EE42, + 0x1EE43, + 0x1EE47, + 0x1EE48, + 0x1EE49, + 0x1EE4A, + 0x1EE4B, + 0x1EE4C, + 0x1EE4D, + 0x1EE4E, + 0x1EE4F, + 0x1EE50, + 0x1EE51, + 0x1EE52, + 0x1EE53, + 0x1EE54, + 0x1EE55, + 0x1EE57, + 0x1EE58, + 0x1EE59, + 0x1EE5A, + 0x1EE5B, + 0x1EE5C, + 0x1EE5D, + 0x1EE5E, + 0x1EE5F, + 0x1EE60, + 0x1EE61, + 0x1EE62, + 0x1EE63, + 0x1EE64, + 0x1EE65, + 0x1EE67, + 0x1EE68, + 0x1EE69, + 0x1EE6A, + 0x1EE6B, + 0x1EE6C, + 0x1EE6D, + 0x1EE6E, + 0x1EE6F, + 0x1EE70, + 0x1EE71, + 0x1EE72, + 0x1EE73, + 0x1EE74, + 0x1EE75, + 0x1EE76, + 0x1EE77, + 0x1EE78, + 0x1EE79, + 0x1EE7A, + 0x1EE7B, + 0x1EE7C, + 0x1EE7D, + 0x1EE7E, + 0x1EE7F, + 0x1EE80, + 0x1EE81, + 0x1EE82, + 0x1EE83, + 0x1EE84, + 0x1EE85, + 0x1EE86, + 0x1EE87, + 0x1EE88, + 0x1EE89, + 0x1EE8A, + 0x1EE8B, + 0x1EE8C, + 0x1EE8D, + 0x1EE8E, + 0x1EE8F, + 0x1EE90, + 0x1EE91, + 0x1EE92, + 0x1EE93, + 0x1EE94, + 0x1EE95, + 0x1EE96, + 0x1EE97, + 0x1EE98, + 0x1EE99, + 0x1EE9A, + 0x1EE9B, + 0x1EE9C, + 0x1EEA1, + 0x1EEA2, + 0x1EEA3, + 0x1EEA4, + 0x1EEA5, + 0x1EEA6, + 0x1EEA7, + 0x1EEA8, + 0x1EEA9, + 0x1EEAA, + 0x1EEAB, + 0x1EEAC, + 0x1EEAD, + 0x1EEAE, + 0x1EEAF, + 0x1EEB0, + 0x1EEB1, + 0x1EEB2, + 0x1EEB3, + 0x1EEB4, + 0x1EEB5, + 0x1EEB6, + 0x1EEB7, + 0x1EEB8, + 0x1EEB9, + 0x1EEBA, + 0x1EEBB, + 0x1EEBC, + 0x1EEF0, + 0x1EEF2, + 0x1F000, + 0x1F02C, + 0x1F030, + 0x1F094, + 0x1F0A0, + 0x1F0AF, + 0x1F0B1, + 0x1F0C0, + 0x1F0C1, + 0x1F0D0, + 0x1F0D1, + 0x1F0F6, + 0x1F101, + 0x1F102, + 0x1F103, + 0x1F104, + 0x1F105, + 0x1F106, + 0x1F107, + 0x1F108, + 0x1F109, + 0x1F10A, + 0x1F10B, + 0x1F110, + 0x1F111, + 0x1F112, + 0x1F113, + 0x1F114, + 0x1F115, + 0x1F116, + 0x1F117, + 0x1F118, + 0x1F119, + 0x1F11A, + 0x1F11B, + 0x1F11C, + 0x1F11D, + 0x1F11E, + 0x1F11F, + 0x1F120, + 0x1F121, + 0x1F122, + 0x1F123, + 0x1F124, + 0x1F125, + 0x1F126, + 0x1F127, + 0x1F128, + 0x1F129, + 0x1F12A, + 0x1F12B, + 0x1F12C, + 0x1F12D, + 0x1F12E, + 0x1F12F, + 0x1F130, + 0x1F131, + 0x1F132, + 0x1F133, + 0x1F134, + 0x1F135, + 0x1F136, + 0x1F137, + 0x1F138, + 0x1F139, + 0x1F13A, + 0x1F13B, + 0x1F13C, + 0x1F13D, + 0x1F13E, + 0x1F13F, + 0x1F140, + 0x1F141, + 0x1F142, + 0x1F143, + 0x1F144, + 0x1F145, + 0x1F146, + 0x1F147, + 0x1F148, + 0x1F149, + 0x1F14A, + 0x1F14B, + 0x1F14C, + 0x1F14D, + 0x1F14E, + 0x1F14F, + 0x1F150, + 0x1F16A, + 0x1F16B, + 0x1F16C, + 0x1F16D, + 0x1F190, + 0x1F191, + 0x1F1AE, + 0x1F1E6, + 0x1F200, + 0x1F201, + 0x1F202, + 0x1F203, + 0x1F210, + 0x1F211, + 0x1F212, + 0x1F213, + 0x1F214, + 0x1F215, + 0x1F216, + 0x1F217, + 0x1F218, + 0x1F219, + 0x1F21A, + 0x1F21B, + 0x1F21C, + 0x1F21D, + 0x1F21E, + 0x1F21F, + 0x1F220, + 0x1F221, + 0x1F222, + 0x1F223, + 0x1F224, + 0x1F225, + 0x1F226, + 0x1F227, + 0x1F228, + 0x1F229, + 0x1F22A, + 0x1F22B, + 0x1F22C, + 0x1F22D, + 0x1F22E, + 0x1F22F, + 0x1F230, + 0x1F231, + 0x1F232, + 0x1F233, + 0x1F234, + 0x1F235, + 0x1F236, + 0x1F237, + 0x1F238, + 0x1F239, + 0x1F23A, + 0x1F23B, + 0x1F23C, + 0x1F240, + 0x1F241, + 0x1F242, + 0x1F243, + 0x1F244, + 0x1F245, + 0x1F246, + 0x1F247, + 0x1F248, + 0x1F249, + 0x1F250, + 0x1F251, + 0x1F252, + 0x1F260, + 0x1F266, + 0x1F300, + 0x1F6D9, + 0x1F6DC, + 0x1F6ED, + 0x1F6F0, + 0x1F6FD, + 0x1F700, + 0x1F7DA, + 0x1F7E0, + 0x1F7EC, + 0x1F7F0, + 0x1F7F1, + 0x1F800, + 0x1F80C, + 0x1F810, + 0x1F848, + 0x1F850, + 0x1F85A, + 0x1F860, + 0x1F888, + 0x1F890, + 0x1F8AE, + 0x1F8B0, + 0x1F8BC, + 0x1F8C0, + 0x1F8C2, + 0x1F8D0, + 0x1F8D9, + 0x1F900, + 0x1FA58, + 0x1FA60, + 0x1FA6E, + 0x1FA70, + 0x1FA7D, + 0x1FA80, + 0x1FA8B, + 0x1FA8E, + 0x1FAC7, + 0x1FAC8, + 0x1FAC9, + 0x1FACD, + 0x1FADD, + 0x1FADF, + 0x1FAEB, + 0x1FAEF, + 0x1FAF9, + 0x1FB00, + 0x1FB93, + 0x1FB94, + 0x1FBF0, + 0x1FBF1, + 0x1FBF2, + 0x1FBF3, + 0x1FBF4, + 0x1FBF5, + 0x1FBF6, + 0x1FBF7, + 0x1FBF8, + 0x1FBF9, + 0x1FBFA, + 0x1FBFB, + 0x20000, + 0x2A6E0, + 0x2A700, + 0x2B81E, + 0x2B820, + 0x2CEAE, + 0x2CEB0, + 0x2EBE1, + 0x2EBF0, + 0x2EE5E, + 0x2F800, + 0x2F801, + 0x2F802, + 0x2F803, + 0x2F804, + 0x2F805, + 0x2F806, + 0x2F807, + 0x2F808, + 0x2F809, + 0x2F80A, + 0x2F80B, + 0x2F80C, + 0x2F80D, + 0x2F80E, + 0x2F80F, + 0x2F810, + 0x2F811, + 0x2F812, + 0x2F813, + 0x2F814, + 0x2F815, + 0x2F816, + 0x2F817, + 0x2F818, + 0x2F819, + 0x2F81A, + 0x2F81B, + 0x2F81C, + 0x2F81D, + 0x2F81E, + 0x2F81F, + 0x2F820, + 0x2F821, + 0x2F822, + 0x2F823, + 0x2F824, + 0x2F825, + 0x2F826, + 0x2F827, + 0x2F828, + 0x2F829, + 0x2F82A, + 0x2F82B, + 0x2F82C, + 0x2F82D, + 0x2F82E, + 0x2F82F, + 0x2F830, + 0x2F831, + 0x2F834, + 0x2F835, + 0x2F836, + 0x2F837, + 0x2F838, + 0x2F839, + 0x2F83A, + 0x2F83B, + 0x2F83C, + 0x2F83D, + 0x2F83E, + 0x2F83F, + 0x2F840, + 0x2F841, + 0x2F842, + 0x2F843, + 0x2F844, + 0x2F845, + 0x2F847, + 0x2F848, + 0x2F849, + 0x2F84A, + 0x2F84B, + 0x2F84C, + 0x2F84D, + 0x2F84E, + 0x2F84F, + 0x2F850, + 0x2F851, + 0x2F852, + 0x2F853, + 0x2F854, + 0x2F855, + 0x2F856, + 0x2F857, + 0x2F858, + 0x2F859, + 0x2F85A, + 0x2F85B, + 0x2F85C, + 0x2F85D, + 0x2F85E, + 0x2F85F, + 0x2F860, + 0x2F861, + 0x2F862, + 0x2F863, + 0x2F864, + 0x2F865, + 0x2F866, + 0x2F867, + 0x2F868, + 0x2F869, + 0x2F86A, + 0x2F86C, + 0x2F86D, + 0x2F86E, + 0x2F86F, + 0x2F870, + 0x2F871, + 0x2F872, + 0x2F873, + 0x2F874, + 0x2F875, + 0x2F876, + 0x2F877, + 0x2F878, + 0x2F879, + 0x2F87A, + 0x2F87B, + 0x2F87C, + 0x2F87D, + 0x2F87E, + 0x2F87F, + 0x2F880, + 0x2F881, + 0x2F882, + 0x2F883, + 0x2F884, + 0x2F885, + 0x2F886, + 0x2F887, + 0x2F888, + 0x2F889, + 0x2F88A, + 0x2F88B, + 0x2F88C, + 0x2F88D, + 0x2F88E, + 0x2F88F, + 0x2F890, + 0x2F891, + 0x2F893, + 0x2F894, + 0x2F896, + 0x2F897, + 0x2F898, + 0x2F899, + 0x2F89A, + 0x2F89B, + 0x2F89C, + 0x2F89D, + 0x2F89E, + 0x2F89F, + 0x2F8A0, + 0x2F8A1, + 0x2F8A2, + 0x2F8A3, + 0x2F8A4, + 0x2F8A5, + 0x2F8A6, + 0x2F8A7, + 0x2F8A8, + 0x2F8A9, + 0x2F8AA, + 0x2F8AB, + 0x2F8AC, + 0x2F8AD, + 0x2F8AE, + 0x2F8AF, + 0x2F8B0, + 0x2F8B1, + 0x2F8B2, + 0x2F8B3, + 0x2F8B4, + 0x2F8B5, + 0x2F8B6, + 0x2F8B7, + 0x2F8B8, + 0x2F8B9, + 0x2F8BA, + 0x2F8BB, + 0x2F8BC, + 0x2F8BD, + 0x2F8BE, + 0x2F8BF, + 0x2F8C0, + 0x2F8C1, + 0x2F8C2, + 0x2F8C3, + 0x2F8C4, + 0x2F8C5, + 0x2F8C6, + 0x2F8C7, + 0x2F8C8, + 0x2F8C9, + 0x2F8CA, + 0x2F8CB, + 0x2F8CC, + 0x2F8CD, + 0x2F8CE, + 0x2F8CF, + 0x2F8D0, + 0x2F8D1, + 0x2F8D2, + 0x2F8D3, + 0x2F8D4, + 0x2F8D5, + 0x2F8D6, + 0x2F8D7, + 0x2F8D8, + 0x2F8D9, + 0x2F8DA, + 0x2F8DB, + 0x2F8DC, + 0x2F8DD, + 0x2F8DE, + 0x2F8DF, + 0x2F8E0, + 0x2F8E1, + 0x2F8E2, + 0x2F8E3, + 0x2F8E4, + 0x2F8E5, + 0x2F8E6, + 0x2F8E7, + 0x2F8E8, + 0x2F8E9, + 0x2F8EA, + 0x2F8EB, + 0x2F8EC, + 0x2F8ED, + 0x2F8EE, + 0x2F8EF, + 0x2F8F0, + 0x2F8F1, + 0x2F8F2, + 0x2F8F3, + 0x2F8F4, + 0x2F8F5, + 0x2F8F6, + 0x2F8F7, + 0x2F8F8, + 0x2F8F9, + 0x2F8FA, + 0x2F8FB, + 0x2F8FC, + 0x2F8FD, + 0x2F8FE, + 0x2F8FF, + 0x2F900, + 0x2F901, + 0x2F902, + 0x2F903, + 0x2F904, + 0x2F905, + 0x2F906, + 0x2F907, + 0x2F908, + 0x2F909, + 0x2F90A, + 0x2F90B, + 0x2F90C, + 0x2F90D, + 0x2F90E, + 0x2F90F, + 0x2F910, + 0x2F911, + 0x2F912, + 0x2F913, + 0x2F914, + 0x2F915, + 0x2F916, + 0x2F917, + 0x2F918, + 0x2F919, + 0x2F91A, + 0x2F91B, + 0x2F91C, + 0x2F91D, + 0x2F91E, + 0x2F91F, + 0x2F920, + 0x2F921, + 0x2F922, + 0x2F923, + 0x2F924, + 0x2F925, + 0x2F926, + 0x2F927, + 0x2F928, + 0x2F929, + 0x2F92A, + 0x2F92B, + 0x2F92C, + 0x2F92E, + 0x2F92F, + 0x2F930, + 0x2F931, + 0x2F932, + 0x2F933, + 0x2F934, + 0x2F935, + 0x2F936, + 0x2F937, + 0x2F938, + 0x2F939, + 0x2F93A, + 0x2F93B, + 0x2F93C, + 0x2F93D, + 0x2F93E, + 0x2F93F, + 0x2F940, + 0x2F941, + 0x2F942, + 0x2F943, + 0x2F944, + 0x2F945, + 0x2F946, + 0x2F948, + 0x2F949, + 0x2F94A, + 0x2F94B, + 0x2F94C, + 0x2F94D, + 0x2F94E, + 0x2F94F, + 0x2F950, + 0x2F951, + 0x2F952, + 0x2F953, + 0x2F954, + 0x2F955, + 0x2F956, + 0x2F957, + 0x2F958, + 0x2F959, + 0x2F95A, + 0x2F95B, + 0x2F95C, + 0x2F95D, + 0x2F95F, + 0x2F960, + 0x2F961, + 0x2F962, + 0x2F963, + 0x2F964, + 0x2F965, + 0x2F966, + 0x2F967, + 0x2F968, + 0x2F969, + 0x2F96A, + 0x2F96B, + 0x2F96C, + 0x2F96D, + 0x2F96E, + 0x2F96F, + 0x2F970, + 0x2F971, + 0x2F972, + 0x2F973, + 0x2F974, + 0x2F975, + 0x2F976, + 0x2F977, + 0x2F978, + 0x2F979, + 0x2F97A, + 0x2F97B, + 0x2F97C, + 0x2F97D, + 0x2F97E, + 0x2F97F, + 0x2F980, + 0x2F981, + 0x2F982, + 0x2F983, + 0x2F984, + 0x2F985, + 0x2F986, + 0x2F987, + 0x2F988, + 0x2F989, + 0x2F98A, + 0x2F98B, + 0x2F98C, + 0x2F98D, + 0x2F98E, + 0x2F98F, + 0x2F990, + 0x2F991, + 0x2F992, + 0x2F993, + 0x2F994, + 0x2F995, + 0x2F996, + 0x2F997, + 0x2F998, + 0x2F999, + 0x2F99A, + 0x2F99B, + 0x2F99C, + 0x2F99D, + 0x2F99E, + 0x2F99F, + 0x2F9A0, + 0x2F9A1, + 0x2F9A2, + 0x2F9A3, + 0x2F9A4, + 0x2F9A5, + 0x2F9A6, + 0x2F9A7, + 0x2F9A8, + 0x2F9A9, + 0x2F9AA, + 0x2F9AB, + 0x2F9AC, + 0x2F9AD, + 0x2F9AE, + 0x2F9AF, + 0x2F9B0, + 0x2F9B1, + 0x2F9B2, + 0x2F9B3, + 0x2F9B4, + 0x2F9B5, + 0x2F9B6, + 0x2F9B7, + 0x2F9B8, + 0x2F9B9, + 0x2F9BA, + 0x2F9BB, + 0x2F9BC, + 0x2F9BD, + 0x2F9BE, + 0x2F9BF, + 0x2F9C0, + 0x2F9C1, + 0x2F9C2, + 0x2F9C3, + 0x2F9C4, + 0x2F9C5, + 0x2F9C6, + 0x2F9C7, + 0x2F9C8, + 0x2F9C9, + 0x2F9CA, + 0x2F9CB, + 0x2F9CC, + 0x2F9CD, + 0x2F9CE, + 0x2F9CF, + 0x2F9D0, + 0x2F9D1, + 0x2F9D2, + 0x2F9D3, + 0x2F9D4, + 0x2F9D5, + 0x2F9D6, + 0x2F9D7, + 0x2F9D8, + 0x2F9D9, + 0x2F9DA, + 0x2F9DB, + 0x2F9DC, + 0x2F9DD, + 0x2F9DE, + 0x2F9DF, + 0x2F9E0, + 0x2F9E1, + 0x2F9E2, + 0x2F9E3, + 0x2F9E4, + 0x2F9E5, + 0x2F9E6, + 0x2F9E7, + 0x2F9E8, + 0x2F9E9, + 0x2F9EA, + 0x2F9EB, + 0x2F9EC, + 0x2F9ED, + 0x2F9EE, + 0x2F9EF, + 0x2F9F0, + 0x2F9F1, + 0x2F9F2, + 0x2F9F3, + 0x2F9F4, + 0x2F9F5, + 0x2F9F6, + 0x2F9F7, + 0x2F9F8, + 0x2F9F9, + 0x2F9FA, + 0x2F9FB, + 0x2F9FC, + 0x2F9FD, + 0x2F9FE, + 0x2FA00, + 0x2FA01, + 0x2FA02, + 0x2FA03, + 0x2FA04, + 0x2FA05, + 0x2FA06, + 0x2FA07, + 0x2FA08, + 0x2FA09, + 0x2FA0A, + 0x2FA0B, + 0x2FA0C, + 0x2FA0D, + 0x2FA0E, + 0x2FA0F, + 0x2FA10, + 0x2FA11, + 0x2FA12, + 0x2FA13, + 0x2FA14, + 0x2FA15, + 0x2FA16, + 0x2FA17, + 0x2FA18, + 0x2FA19, + 0x2FA1A, + 0x2FA1B, + 0x2FA1C, + 0x2FA1D, + 0x2FA1E, + 0x30000, + 0x3134B, + 0x31350, + 0x3347A, + 0xE0100, + 0xE01F0, + ), +) + +uts46_statuses: bytes = ( + b"VVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVV" + b"VMMMMMMMMMMMMMMMMMMMMMMMMMMVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVV" + b"XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXMVVVVVVVMVMVVIVMVVMMMMVVMMMVMMMV" + b"MMMMMMMMMMMMMMMMMMMMMMMVMMMMMMMDVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVV" + b"MVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMMVMVMVMVMVMMV" + b"MVMVMVMMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMMVMVMVMVMM" + b"VMVMMVMMMVMMMMVMMVMMMVMMVMMVMVMVMMVMVMVMMVMMMVMVMMVMVMMMMVMVMVMV" + b"MVMVMVMVMVMVMVMVMVMVMVMVMVMMVMMMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVM" + b"VMVMVMVMVMVMVMVMVMVMVMVMVMVMMVMMVMVMMMMVMVMVMVMVMMMMMMMMMVMMMMMM" + b"VMMMMMVMMVMMMVIVMVMVMVMVXMVMMXMMMMMMMXMXMMVMMMMMMMMMMMMMMMMMXMMM" + b"MMMMMMVDVMMMMMMMMVMVMVMVMVMVMVMVMVMVMVMVMVMMMVMMVMVMMVMMMMMMMMMM" + b"MMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMVMVMVMVMVMVMVMVMVMVMVMV" + b"MVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMV" + b"MVMMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVM" + b"VMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVXMMMMMMMMMMMMMM" + b"MMMMMMMMMMMMMMMMMMMMMMMMXVMVXVXVXVXVXVXVMMMMVXVXVXVXVXVXVXVXVXVX" + b"VXVXVMMMMMMMMVXVXVXVXVXVXVXVXVXVXVXMMXMVXVXVXVXVXVXVXVMXVMXVXVXV" + b"XVXVXVXMMMVXMXVXVXVXVXVXVXVXVXVXVXVXVXVXVXVXVXVXVXVXVXVXVXVXVXVX" + b"VXMMXVXVXVXVXVXVXVXVXVXVXVXVXVXVXVXVXVXVXVXVXVXVXVXVXVXVXVXVXVXV" + b"XVXVXVXVXVXVXVXVXVXVXVXVXVXVXVXVXVXVXVXVXVXVXVXVXVXVXVXVXVXVXVXV" + b"XVMVXVXVXVXVXVXVXVMVXVXVXVXVXMMVXVMVMVXVMVMVMVMVMVXVMVMMMMMVMVMV" + b"XVMVMVMVMVMVXVXVXVMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMXMXMXVMV" + b"IVXVXVXVXVXVXVXVXVXVXVXVXVXVXVXVXVXVXVXMMMMMMXVXVXVXVXVXVXVXVXVX" + b"VIVXVXVXVIVXVXVXVXVXVXVXVXVXVXVXVXVXVXVXVXVXVXVXVXVXVXVXVXVXVMMM" + b"MMMMMMVXMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMXMMMVXVXVMMMV" + b"MMMMMMMMMMMVMMMMMMMMMMMMMMMMMMVMMMMMMMMMMMMMMMMMMMMMMMMMMMMVMVMM" + b"MMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMVMVMVMVMVMVMVMVMVMVMVMVMVMVMV" + b"MVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMV" + b"MVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMMVMVM" + b"VMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVM" + b"VMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMMMMMMMMVXMMMMMMXVMMMMMMMMVMMMMMM" + b"MMVXMMMMMMXVXMXMXMXMVMMMMMMMMVMVMVMVMVMVMVMXMMMMMMMMMMMMMMMMMMMM" + b"MMMMMMMMMMMMMMMMMMMMMMMMMMMMVMMMXVMMMMMMMMMMMMMMXVMMMMMMMMMVMXVM" + b"MMMXMMMVMVMMMMMMMMXMMMXVMMMMMMMMXMIDXVMVMVXVXMVMMVMMVMVMVMMMVMVM" + b"IXIMMXMMMMMMMMMMMMMMMMMMMMMMMMMMMXMMMMMMMMMMMMMXVMVXVXMMMMVMMMVM" + b"MMMMMVMMVMMMVMMMVMVMVMVMMMMVMMMMMMMMMMVMMMMMVMMMMVMMMMMMMMMMMMMM" + b"MMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMVMVMVXVMMVMMVMMVXVXMMMMMMMMMMM" + b"MMMMMMMMMMMMMMMMMMMMMMMMMMMMMXMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMM" + b"MMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMVMVMMMVMVXVMMMMMMMM" + b"MMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMVMVMMMVMVMVMVMMMMVMVMVMM" + b"MMMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMV" + b"MVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVXVXVXVXVXMVXVXVXVXVX" + b"VXVXVXVXVXVXVXVMVMXMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMM" + b"MMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMM" + b"MMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMM" + b"MMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMXMVMVMVMMMVXVXVMMVMVMXV" + b"XMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMIMMMMMMMMMMM" + b"MMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMXVMMMMMMMMMMMMMMVXVMMMMMMMMMMMMMM" + b"MMMMMMMMMMMMMMMMMXMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMVMMMMM" + b"MMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMVMMMMMMMMMMMMMMMMMMMMM" + b"MMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMM" + b"MMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMM" + b"MMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMM" + b"MMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMM" + b"MMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMXMMMMXMMMMMMMMMMMMM" + b"MMMXMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMVXVXVXMVMVMVMVMVMVMVM" + b"VMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMMVXV" + b"MVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMV" + b"MVMVMVMVMVMVMVMVMVMMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMMMMMVM" + b"MMMMVMVMVMVMVMVMVMVMMMMVMVMMVMVMVMVMVMVMVMVMXMMMMMVMMVXVXVXVXVXV" + b"XVXVXVXVXVXVXVXVXVXVXVXVXVXVXVMMMMVMVXMMMMMMMMMMMMMMMMMMMMMMMMMM" + b"MMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMVXVXVXVXVX" + b"MMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMM" + b"MMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMM" + b"MMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMM" + b"MMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMM" + b"MMMMMMMMMMMMMVMVMVMMMMMMMMMMVMVMVMMVMMMMMMMMMMMMMMMMMMMMMMMMMMMM" + b"MMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMXMMMMMMMMMMMMMMMMMMMMMMMM" + b"MMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMM" + b"MMMMMMMMMMMMMMMMMMXMMMMMMXMMMMMXMVMMMMMMMMMMMMMMMMMMMMMMMMXMMMMM" + b"XMXMMXMMXMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMVMMMMMMMMMMMMMM" + b"MMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMM" + b"MMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMM" + b"MMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMM" + b"MMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMM" + b"MMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMM" + b"MMVMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMVMMMMMMMMMMM" + b"MMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMVXMMMMMMMMMMMMMVIMMXMMM" + b"MMMXVXMMMMMMMMMMMMMMMMMMMVMMMMMMXMMMMMMMMMMMMMMMMMMMXMMMMXMMMVMX" + b"MMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMXIXMMMMMMMMMMM" + b"MMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMM" + b"MMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMM" + b"MMMMMMMMMMMMMMMMMMMMIMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMXMMMMMMXMMMMM" + b"MXMMMMMMXMMMXMMMMMMMXMMMMMMMXVXVXVXVXVXVXVXVXVXVXVXVXVXVXVXVXVXV" + b"XVXVXVXVXMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMVXVXMMMMMMMMMMM" + b"MMMMMMMMMMMMMMMMMMMMMMMMMXVXVXVXVMMMMMMMMMMMXMMMMMMMMMMMMMMMXMMM" + b"MMMMXMMXVXVXVXVXVXVXVXVXVMMMMMXMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMM" + b"MMMMMMMMMXMMMMMMMMMXVXVXVXVXVXVXVXVXVXVXVXVXVXVXVXVXVXVXVXVXVXVX" + b"VXVXVXVXVXVXVXVXVXVXVXMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMM" + b"MMMMMMMMMXVXVXVXVMMMMMMMMMMMMMMMMMMMMMMXVXVXVXVXVXVXVXVXVXVXVXVX" + b"VXVXVXVXVXVXVXVXVXVXVXVXVXVXVXVXVXVXVXVXVXVXVXVXVXVXVXVXVXVXVXVX" + b"VXVXVXVXVXVXVXVXVXVXVXVXVXVXVXVXVXVXVXVXVXVXVXVXVXVXVXVXVXVXVXMM" + b"MMMMMMMMMMMMMMMMMMMMMMMMMMMMMMVXVXVXVXVXVXVXVXVXVXVXVXVXVXVXVXVX" + b"VXVXVXVXVXVXVXVXVXVXVXVXVXVXVXVXVXVXVXVXVXVXVXVXVXVXVXVXVXVXVXVX" + b"VXVXVXVXVXVXVXVXVXVXVXVXVXVXVXVXVXVXVXVXVXMMMMMMMMMMMMMMMMMMMMMM" + b"MMMMMMMMMMVXMMMMMMMMMMMMMMMMMMMMMMMMMXVXVXVXVXVXVXVXVXVXVXVXVXVX" + b"VXVXVXVXVXVXVXVXVXVIXVMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMVXVXVX" + b"VXVXVXVXVXVXVMMMMMMMVIVMMMMMMVXVXVXVXVXVXMMMMMMMMMMMMMMMMMMMMMMM" + b"MMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMXM" + b"MMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMM" + b"MMMMMMXMMXMXMMXMMMMXMMMMMMMMMMMMXMXMMMMMMMXMMMMMMMMMMMMMMMMMMMMM" + b"MMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMXMMMMXMMMMMMMMXMMMMM" + b"MMXMMMMMMMMMMMMMMMMMMMMMMMMMMMMXMMMMXMMMMMXMXMMMMMMMXMMMMMMMMMMM" + b"MMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMM" + b"MMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMM" + b"MMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMM" + b"MMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMM" + b"MMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMM" + b"MMMMMMMMMXMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMM" + b"MMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMM" + b"MMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMM" + b"MMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMM" + b"MMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMXMMMMMMMMMMMMMMMMMMMMMMM" + b"MMMMMMMMMMMMMMMMMMMMMMMMMMMVXVXVXVXVXVXVXVXVXVXMMMMMMMMMMMMMMMMM" + b"MMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMXVXVXVXVXVXVXVXVXVX" + b"VXVXVXVXVXVXVXVXVXVXVXMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMVXVXVXVX" + b"VXMMMMXMMMMMMMMMMMMMMMMMMMMMMMMMMMXMMXMXMXMMMMMMMMMMXMMMMXMXMXMX" + b"MXMXMXMMMXMMXMXMXMXMXMXMXMMXMXMMMMXMMMMMMMXMMMMXMMMMXMXMMMMMMMMM" + b"MXMMMMMMMMMMMMMMMMMXMMMXMMMMMXMMMMMMMMMMMMMMMMMXVXVXVXVXVXVXVXMM" + b"MMMMMMMMVMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMVMMMMMMMMMMMMMMMMMMMMMMM" + b"MMMMMMMMMVMMMVMVXVMMMXMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMM" + b"MMXMMMMMMMMMXMMXVXVXVXVXVXVXVXVXVXVXVXVXVXVXVXVXVXVXVXVXVXVXVXVX" + b"VXVMMMMMMMMMMVXVXVXVXVXVXMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMM" + b"MMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMM" + b"MMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMM" + b"MMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMM" + b"MMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMM" + b"MMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMM" + b"MMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMM" + b"MMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMM" + b"MMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMXVXVXIX" +) + +uts46_replacements: tuple[str | None, ...] = ( + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + "a", + "b", + "c", + "d", + "e", + "f", + "g", + "h", + "i", + "j", + "k", + "l", + "m", + "n", + "o", + "p", + "q", + "r", + "s", + "t", + "u", + "v", + "w", + "x", + "y", + "z", + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + " ", + None, + None, + None, + None, + None, + None, + None, + " ̈", + None, + "a", + None, + None, + None, + None, + " ̄", + None, + None, + "2", + "3", + " ́", + "μ", + None, + None, + " ̧", + "1", + "o", + None, + "1⁄4", + "1⁄2", + "3⁄4", + None, + "à", + "á", + "â", + "ã", + "ä", + "å", + "æ", + "ç", + "è", + "é", + "ê", + "ë", + "ì", + "í", + "î", + "ï", + "ð", + "ñ", + "ò", + "ó", + "ô", + "õ", + "ö", + None, + "ø", + "ù", + "ú", + "û", + "ü", + "ý", + "þ", + "ss", + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + "ā", + None, + "ă", + None, + "ą", + None, + "ć", + None, + "ĉ", + None, + "ċ", + None, + "č", + None, + "ď", + None, + "đ", + None, + "ē", + None, + "ĕ", + None, + "ė", + None, + "ę", + None, + "ě", + None, + "ĝ", + None, + "ğ", + None, + "ġ", + None, + "ģ", + None, + "ĥ", + None, + "ħ", + None, + "ĩ", + None, + "ī", + None, + "ĭ", + None, + "į", + None, + "i̇", + None, + "ij", + "ĵ", + None, + "ķ", + None, + "ĺ", + None, + "ļ", + None, + "ľ", + None, + "l·", + "ł", + None, + "ń", + None, + "ņ", + None, + "ň", + None, + "ʼn", + "ŋ", + None, + "ō", + None, + "ŏ", + None, + "ő", + None, + "œ", + None, + "ŕ", + None, + "ŗ", + None, + "ř", + None, + "ś", + None, + "ŝ", + None, + "ş", + None, + "š", + None, + "ţ", + None, + "ť", + None, + "ŧ", + None, + "ũ", + None, + "ū", + None, + "ŭ", + None, + "ů", + None, + "ű", + None, + "ų", + None, + "ŵ", + None, + "ŷ", + None, + "ÿ", + "ź", + None, + "ż", + None, + "ž", + None, + "s", + None, + "ɓ", + "ƃ", + None, + "ƅ", + None, + "ɔ", + "ƈ", + None, + "ɖ", + "ɗ", + "ƌ", + None, + "ǝ", + "ə", + "ɛ", + "ƒ", + None, + "ɠ", + "ɣ", + None, + "ɩ", + "ɨ", + "ƙ", + None, + "ɯ", + "ɲ", + None, + "ɵ", + "ơ", + None, + "ƣ", + None, + "ƥ", + None, + "ʀ", + "ƨ", + None, + "ʃ", + None, + "ƭ", + None, + "ʈ", + "ư", + None, + "ʊ", + "ʋ", + "ƴ", + None, + "ƶ", + None, + "ʒ", + "ƹ", + None, + "ƽ", + None, + "dž", + "lj", + "nj", + "ǎ", + None, + "ǐ", + None, + "ǒ", + None, + "ǔ", + None, + "ǖ", + None, + "ǘ", + None, + "ǚ", + None, + "ǜ", + None, + "ǟ", + None, + "ǡ", + None, + "ǣ", + None, + "ǥ", + None, + "ǧ", + None, + "ǩ", + None, + "ǫ", + None, + "ǭ", + None, + "ǯ", + None, + "dz", + "ǵ", + None, + "ƕ", + "ƿ", + "ǹ", + None, + "ǻ", + None, + "ǽ", + None, + "ǿ", + None, + "ȁ", + None, + "ȃ", + None, + "ȅ", + None, + "ȇ", + None, + "ȉ", + None, + "ȋ", + None, + "ȍ", + None, + "ȏ", + None, + "ȑ", + None, + "ȓ", + None, + "ȕ", + None, + "ȗ", + None, + "ș", + None, + "ț", + None, + "ȝ", + None, + "ȟ", + None, + "ƞ", + None, + "ȣ", + None, + "ȥ", + None, + "ȧ", + None, + "ȩ", + None, + "ȫ", + None, + "ȭ", + None, + "ȯ", + None, + "ȱ", + None, + "ȳ", + None, + "ⱥ", + "ȼ", + None, + "ƚ", + "ⱦ", + None, + "ɂ", + None, + "ƀ", + "ʉ", + "ʌ", + "ɇ", + None, + "ɉ", + None, + "ɋ", + None, + "ɍ", + None, + "ɏ", + None, + "h", + "ɦ", + "j", + "r", + "ɹ", + "ɻ", + "ʁ", + "w", + "y", + None, + " ̆", + " ̇", + " ̊", + " ̨", + " ̃", + " ̋", + None, + "ɣ", + "l", + "s", + "x", + "ʕ", + None, + "̀", + "́", + None, + "̓", + "̈́", + "ι", + None, + None, + None, + "ͱ", + None, + "ͳ", + None, + "ʹ", + None, + "ͷ", + None, + None, + " ι", + None, + ";", + "ϳ", + None, + " ́", + " ̈́", + "ά", + "·", + "έ", + "ή", + "ί", + None, + "ό", + None, + "ύ", + "ώ", + None, + "α", + "β", + "γ", + "δ", + "ε", + "ζ", + "η", + "θ", + "ι", + "κ", + "λ", + "μ", + "ν", + "ξ", + "ο", + "π", + "ρ", + None, + "σ", + "τ", + "υ", + "φ", + "χ", + "ψ", + "ω", + "ϊ", + "ϋ", + None, + "σ", + None, + "ϗ", + "β", + "θ", + "υ", + "ύ", + "ϋ", + "φ", + "π", + None, + "ϙ", + None, + "ϛ", + None, + "ϝ", + None, + "ϟ", + None, + "ϡ", + None, + "ϣ", + None, + "ϥ", + None, + "ϧ", + None, + "ϩ", + None, + "ϫ", + None, + "ϭ", + None, + "ϯ", + None, + "κ", + "ρ", + "σ", + None, + "θ", + "ε", + None, + "ϸ", + None, + "σ", + "ϻ", + None, + "ͻ", + "ͼ", + "ͽ", + "ѐ", + "ё", + "ђ", + "ѓ", + "є", + "ѕ", + "і", + "ї", + "ј", + "љ", + "њ", + "ћ", + "ќ", + "ѝ", + "ў", + "џ", + "а", + "б", + "в", + "г", + "д", + "е", + "ж", + "з", + "и", + "й", + "к", + "л", + "м", + "н", + "о", + "п", + "р", + "с", + "т", + "у", + "ф", + "х", + "ц", + "ч", + "ш", + "щ", + "ъ", + "ы", + "ь", + "э", + "ю", + "я", + None, + "ѡ", + None, + "ѣ", + None, + "ѥ", + None, + "ѧ", + None, + "ѩ", + None, + "ѫ", + None, + "ѭ", + None, + "ѯ", + None, + "ѱ", + None, + "ѳ", + None, + "ѵ", + None, + "ѷ", + None, + "ѹ", + None, + "ѻ", + None, + "ѽ", + None, + "ѿ", + None, + "ҁ", + None, + "ҋ", + None, + "ҍ", + None, + "ҏ", + None, + "ґ", + None, + "ғ", + None, + "ҕ", + None, + "җ", + None, + "ҙ", + None, + "қ", + None, + "ҝ", + None, + "ҟ", + None, + "ҡ", + None, + "ң", + None, + "ҥ", + None, + "ҧ", + None, + "ҩ", + None, + "ҫ", + None, + "ҭ", + None, + "ү", + None, + "ұ", + None, + "ҳ", + None, + "ҵ", + None, + "ҷ", + None, + "ҹ", + None, + "һ", + None, + "ҽ", + None, + "ҿ", + None, + "ӏ", + "ӂ", + None, + "ӄ", + None, + "ӆ", + None, + "ӈ", + None, + "ӊ", + None, + "ӌ", + None, + "ӎ", + None, + "ӑ", + None, + "ӓ", + None, + "ӕ", + None, + "ӗ", + None, + "ә", + None, + "ӛ", + None, + "ӝ", + None, + "ӟ", + None, + "ӡ", + None, + "ӣ", + None, + "ӥ", + None, + "ӧ", + None, + "ө", + None, + "ӫ", + None, + "ӭ", + None, + "ӯ", + None, + "ӱ", + None, + "ӳ", + None, + "ӵ", + None, + "ӷ", + None, + "ӹ", + None, + "ӻ", + None, + "ӽ", + None, + "ӿ", + None, + "ԁ", + None, + "ԃ", + None, + "ԅ", + None, + "ԇ", + None, + "ԉ", + None, + "ԋ", + None, + "ԍ", + None, + "ԏ", + None, + "ԑ", + None, + "ԓ", + None, + "ԕ", + None, + "ԗ", + None, + "ԙ", + None, + "ԛ", + None, + "ԝ", + None, + "ԟ", + None, + "ԡ", + None, + "ԣ", + None, + "ԥ", + None, + "ԧ", + None, + "ԩ", + None, + "ԫ", + None, + "ԭ", + None, + "ԯ", + None, + None, + "ա", + "բ", + "գ", + "դ", + "ե", + "զ", + "է", + "ը", + "թ", + "ժ", + "ի", + "լ", + "խ", + "ծ", + "կ", + "հ", + "ձ", + "ղ", + "ճ", + "մ", + "յ", + "ն", + "շ", + "ո", + "չ", + "պ", + "ջ", + "ռ", + "ս", + "վ", + "տ", + "ր", + "ց", + "ւ", + "փ", + "ք", + "օ", + "ֆ", + None, + None, + "եւ", + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + "اٴ", + "وٴ", + "ۇٴ", + "يٴ", + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + "क़", + "ख़", + "ग़", + "ज़", + "ड़", + "ढ़", + "फ़", + "य़", + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + "ড়", + "ঢ়", + None, + "য়", + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + "ਲ਼", + None, + None, + "ਸ਼", + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + "ਖ਼", + "ਗ਼", + "ਜ਼", + None, + None, + "ਫ਼", + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + "ଡ଼", + "ଢ଼", + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + "ํา", + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + "ໍາ", + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + "ຫນ", + "ຫມ", + None, + None, + None, + "་", + None, + "གྷ", + None, + None, + None, + "ཌྷ", + None, + "དྷ", + None, + "བྷ", + None, + "ཛྷ", + None, + "ཀྵ", + None, + None, + None, + "ཱི", + None, + "ཱུ", + "ྲྀ", + "ྲཱྀ", + "ླྀ", + "ླཱྀ", + None, + "ཱྀ", + None, + "ྒྷ", + None, + None, + None, + "ྜྷ", + None, + "ྡྷ", + None, + "ྦྷ", + None, + "ྫྷ", + None, + "ྐྵ", + None, + None, + None, + None, + None, + None, + None, + "ⴀ", + "ⴁ", + "ⴂ", + "ⴃ", + "ⴄ", + "ⴅ", + "ⴆ", + "ⴇ", + "ⴈ", + "ⴉ", + "ⴊ", + "ⴋ", + "ⴌ", + "ⴍ", + "ⴎ", + "ⴏ", + "ⴐ", + "ⴑ", + "ⴒ", + "ⴓ", + "ⴔ", + "ⴕ", + "ⴖ", + "ⴗ", + "ⴘ", + "ⴙ", + "ⴚ", + "ⴛ", + "ⴜ", + "ⴝ", + "ⴞ", + "ⴟ", + "ⴠ", + "ⴡ", + "ⴢ", + "ⴣ", + "ⴤ", + "ⴥ", + None, + "ⴧ", + None, + "ⴭ", + None, + None, + "ნ", + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + "Ᏸ", + "Ᏹ", + "Ᏺ", + "Ᏻ", + "Ᏼ", + "Ᏽ", + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + "в", + "д", + "о", + "с", + "т", + "ъ", + "ѣ", + "ꙋ", + "ᲊ", + None, + None, + "ა", + "ბ", + "გ", + "დ", + "ე", + "ვ", + "ზ", + "თ", + "ი", + "კ", + "ლ", + "მ", + "ნ", + "ო", + "პ", + "ჟ", + "რ", + "ს", + "ტ", + "უ", + "ფ", + "ქ", + "ღ", + "ყ", + "შ", + "ჩ", + "ც", + "ძ", + "წ", + "ჭ", + "ხ", + "ჯ", + "ჰ", + "ჱ", + "ჲ", + "ჳ", + "ჴ", + "ჵ", + "ჶ", + "ჷ", + "ჸ", + "ჹ", + "ჺ", + None, + "ჽ", + "ჾ", + "ჿ", + None, + None, + None, + None, + None, + "a", + "æ", + "b", + None, + "d", + "e", + "ǝ", + "g", + "h", + "i", + "j", + "k", + "l", + "m", + "n", + None, + "o", + "ȣ", + "p", + "r", + "t", + "u", + "w", + "a", + "ɐ", + "ɑ", + "ᴂ", + "b", + "d", + "e", + "ə", + "ɛ", + "ɜ", + "g", + None, + "k", + "m", + "ŋ", + "o", + "ɔ", + "ᴖ", + "ᴗ", + "p", + "t", + "u", + "ᴝ", + "ɯ", + "v", + "ᴥ", + "β", + "γ", + "δ", + "φ", + "χ", + "i", + "r", + "u", + "v", + "β", + "γ", + "ρ", + "φ", + "χ", + None, + "н", + None, + "ɒ", + "c", + "ɕ", + "ð", + "ɜ", + "f", + "ɟ", + "ɡ", + "ɥ", + "ɨ", + "ɩ", + "ɪ", + "ᵻ", + "ʝ", + "ɭ", + "ᶅ", + "ʟ", + "ɱ", + "ɰ", + "ɲ", + "ɳ", + "ɴ", + "ɵ", + "ɸ", + "ʂ", + "ʃ", + "ƫ", + "ʉ", + "ʊ", + "ᴜ", + "ʋ", + "ʌ", + "z", + "ʐ", + "ʑ", + "ʒ", + "θ", + None, + "ḁ", + None, + "ḃ", + None, + "ḅ", + None, + "ḇ", + None, + "ḉ", + None, + "ḋ", + None, + "ḍ", + None, + "ḏ", + None, + "ḑ", + None, + "ḓ", + None, + "ḕ", + None, + "ḗ", + None, + "ḙ", + None, + "ḛ", + None, + "ḝ", + None, + "ḟ", + None, + "ḡ", + None, + "ḣ", + None, + "ḥ", + None, + "ḧ", + None, + "ḩ", + None, + "ḫ", + None, + "ḭ", + None, + "ḯ", + None, + "ḱ", + None, + "ḳ", + None, + "ḵ", + None, + "ḷ", + None, + "ḹ", + None, + "ḻ", + None, + "ḽ", + None, + "ḿ", + None, + "ṁ", + None, + "ṃ", + None, + "ṅ", + None, + "ṇ", + None, + "ṉ", + None, + "ṋ", + None, + "ṍ", + None, + "ṏ", + None, + "ṑ", + None, + "ṓ", + None, + "ṕ", + None, + "ṗ", + None, + "ṙ", + None, + "ṛ", + None, + "ṝ", + None, + "ṟ", + None, + "ṡ", + None, + "ṣ", + None, + "ṥ", + None, + "ṧ", + None, + "ṩ", + None, + "ṫ", + None, + "ṭ", + None, + "ṯ", + None, + "ṱ", + None, + "ṳ", + None, + "ṵ", + None, + "ṷ", + None, + "ṹ", + None, + "ṻ", + None, + "ṽ", + None, + "ṿ", + None, + "ẁ", + None, + "ẃ", + None, + "ẅ", + None, + "ẇ", + None, + "ẉ", + None, + "ẋ", + None, + "ẍ", + None, + "ẏ", + None, + "ẑ", + None, + "ẓ", + None, + "ẕ", + None, + "aʾ", + "ṡ", + None, + "ß", + None, + "ạ", + None, + "ả", + None, + "ấ", + None, + "ầ", + None, + "ẩ", + None, + "ẫ", + None, + "ậ", + None, + "ắ", + None, + "ằ", + None, + "ẳ", + None, + "ẵ", + None, + "ặ", + None, + "ẹ", + None, + "ẻ", + None, + "ẽ", + None, + "ế", + None, + "ề", + None, + "ể", + None, + "ễ", + None, + "ệ", + None, + "ỉ", + None, + "ị", + None, + "ọ", + None, + "ỏ", + None, + "ố", + None, + "ồ", + None, + "ổ", + None, + "ỗ", + None, + "ộ", + None, + "ớ", + None, + "ờ", + None, + "ở", + None, + "ỡ", + None, + "ợ", + None, + "ụ", + None, + "ủ", + None, + "ứ", + None, + "ừ", + None, + "ử", + None, + "ữ", + None, + "ự", + None, + "ỳ", + None, + "ỵ", + None, + "ỷ", + None, + "ỹ", + None, + "ỻ", + None, + "ỽ", + None, + "ỿ", + None, + "ἀ", + "ἁ", + "ἂ", + "ἃ", + "ἄ", + "ἅ", + "ἆ", + "ἇ", + None, + None, + "ἐ", + "ἑ", + "ἒ", + "ἓ", + "ἔ", + "ἕ", + None, + None, + "ἠ", + "ἡ", + "ἢ", + "ἣ", + "ἤ", + "ἥ", + "ἦ", + "ἧ", + None, + "ἰ", + "ἱ", + "ἲ", + "ἳ", + "ἴ", + "ἵ", + "ἶ", + "ἷ", + None, + None, + "ὀ", + "ὁ", + "ὂ", + "ὃ", + "ὄ", + "ὅ", + None, + None, + None, + "ὑ", + None, + "ὓ", + None, + "ὕ", + None, + "ὗ", + None, + "ὠ", + "ὡ", + "ὢ", + "ὣ", + "ὤ", + "ὥ", + "ὦ", + "ὧ", + None, + "ά", + None, + "έ", + None, + "ή", + None, + "ί", + None, + "ό", + None, + "ύ", + None, + "ώ", + None, + "ἀι", + "ἁι", + "ἂι", + "ἃι", + "ἄι", + "ἅι", + "ἆι", + "ἇι", + "ἀι", + "ἁι", + "ἂι", + "ἃι", + "ἄι", + "ἅι", + "ἆι", + "ἇι", + "ἠι", + "ἡι", + "ἢι", + "ἣι", + "ἤι", + "ἥι", + "ἦι", + "ἧι", + "ἠι", + "ἡι", + "ἢι", + "ἣι", + "ἤι", + "ἥι", + "ἦι", + "ἧι", + "ὠι", + "ὡι", + "ὢι", + "ὣι", + "ὤι", + "ὥι", + "ὦι", + "ὧι", + "ὠι", + "ὡι", + "ὢι", + "ὣι", + "ὤι", + "ὥι", + "ὦι", + "ὧι", + None, + "ὰι", + "αι", + "άι", + None, + None, + "ᾶι", + "ᾰ", + "ᾱ", + "ὰ", + "ά", + "αι", + " ̓", + "ι", + " ̓", + " ͂", + " ̈͂", + "ὴι", + "ηι", + "ήι", + None, + None, + "ῆι", + "ὲ", + "έ", + "ὴ", + "ή", + "ηι", + " ̓̀", + " ̓́", + " ̓͂", + None, + "ΐ", + None, + None, + "ῐ", + "ῑ", + "ὶ", + "ί", + None, + " ̔̀", + " ̔́", + " ̔͂", + None, + "ΰ", + None, + "ῠ", + "ῡ", + "ὺ", + "ύ", + "ῥ", + " ̈̀", + " ̈́", + "`", + None, + "ὼι", + "ωι", + "ώι", + None, + None, + "ῶι", + "ὸ", + "ό", + "ὼ", + "ώ", + "ωι", + " ́", + " ̔", + None, + " ", + None, + "", + None, + None, + "‐", + None, + " ̳", + None, + None, + None, + None, + " ", + None, + "′′", + "′′′", + None, + "‵‵", + "‵‵‵", + None, + "!!", + None, + " ̅", + None, + "??", + "?!", + "!?", + None, + "′′′′", + None, + " ", + None, + None, + None, + "0", + "i", + None, + "4", + "5", + "6", + "7", + "8", + "9", + "+", + "−", + "=", + "(", + ")", + "n", + "0", + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "+", + "−", + "=", + "(", + ")", + None, + "a", + "e", + "o", + "x", + "ə", + "h", + "k", + "l", + "m", + "n", + "p", + "s", + "t", + None, + None, + "rs", + None, + None, + None, + None, + "a/c", + "a/s", + "c", + "°c", + None, + "c/o", + "c/u", + "ɛ", + None, + "°f", + "g", + "h", + "ħ", + "i", + "l", + None, + "n", + "no", + None, + "p", + "q", + "r", + None, + "sm", + "tel", + "tm", + None, + "z", + None, + "ω", + None, + "z", + None, + "k", + "å", + "b", + "c", + None, + "e", + "f", + "ⅎ", + "m", + "o", + "א", + "ב", + "ג", + "ד", + "i", + None, + "fax", + "π", + "γ", + "π", + "∑", + None, + "d", + "e", + "i", + "j", + None, + "1⁄7", + "1⁄9", + "1⁄10", + "1⁄3", + "2⁄3", + "1⁄5", + "2⁄5", + "3⁄5", + "4⁄5", + "1⁄6", + "5⁄6", + "1⁄8", + "3⁄8", + "5⁄8", + "7⁄8", + "1⁄", + "i", + "ii", + "iii", + "iv", + "v", + "vi", + "vii", + "viii", + "ix", + "x", + "xi", + "xii", + "l", + "c", + "d", + "m", + "i", + "ii", + "iii", + "iv", + "v", + "vi", + "vii", + "viii", + "ix", + "x", + "xi", + "xii", + "l", + "c", + "d", + "m", + None, + "ↄ", + None, + "0⁄3", + None, + None, + None, + "∫∫", + "∫∫∫", + None, + "∮∮", + "∮∮∮", + None, + "〈", + "〉", + None, + None, + None, + None, + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10", + "11", + "12", + "13", + "14", + "15", + "16", + "17", + "18", + "19", + "20", + "(1)", + "(2)", + "(3)", + "(4)", + "(5)", + "(6)", + "(7)", + "(8)", + "(9)", + "(10)", + "(11)", + "(12)", + "(13)", + "(14)", + "(15)", + "(16)", + "(17)", + "(18)", + "(19)", + "(20)", + None, + "(a)", + "(b)", + "(c)", + "(d)", + "(e)", + "(f)", + "(g)", + "(h)", + "(i)", + "(j)", + "(k)", + "(l)", + "(m)", + "(n)", + "(o)", + "(p)", + "(q)", + "(r)", + "(s)", + "(t)", + "(u)", + "(v)", + "(w)", + "(x)", + "(y)", + "(z)", + "a", + "b", + "c", + "d", + "e", + "f", + "g", + "h", + "i", + "j", + "k", + "l", + "m", + "n", + "o", + "p", + "q", + "r", + "s", + "t", + "u", + "v", + "w", + "x", + "y", + "z", + "a", + "b", + "c", + "d", + "e", + "f", + "g", + "h", + "i", + "j", + "k", + "l", + "m", + "n", + "o", + "p", + "q", + "r", + "s", + "t", + "u", + "v", + "w", + "x", + "y", + "z", + "0", + None, + "∫∫∫∫", + None, + "::=", + "==", + "===", + None, + "⫝̸", + None, + None, + None, + "ⰰ", + "ⰱ", + "ⰲ", + "ⰳ", + "ⰴ", + "ⰵ", + "ⰶ", + "ⰷ", + "ⰸ", + "ⰹ", + "ⰺ", + "ⰻ", + "ⰼ", + "ⰽ", + "ⰾ", + "ⰿ", + "ⱀ", + "ⱁ", + "ⱂ", + "ⱃ", + "ⱄ", + "ⱅ", + "ⱆ", + "ⱇ", + "ⱈ", + "ⱉ", + "ⱊ", + "ⱋ", + "ⱌ", + "ⱍ", + "ⱎ", + "ⱏ", + "ⱐ", + "ⱑ", + "ⱒ", + "ⱓ", + "ⱔ", + "ⱕ", + "ⱖ", + "ⱗ", + "ⱘ", + "ⱙ", + "ⱚ", + "ⱛ", + "ⱜ", + "ⱝ", + "ⱞ", + "ⱟ", + None, + "ⱡ", + None, + "ɫ", + "ᵽ", + "ɽ", + None, + "ⱨ", + None, + "ⱪ", + None, + "ⱬ", + None, + "ɑ", + "ɱ", + "ɐ", + "ɒ", + None, + "ⱳ", + None, + "ⱶ", + None, + "j", + "v", + "ȿ", + "ɀ", + "ⲁ", + None, + "ⲃ", + None, + "ⲅ", + None, + "ⲇ", + None, + "ⲉ", + None, + "ⲋ", + None, + "ⲍ", + None, + "ⲏ", + None, + "ⲑ", + None, + "ⲓ", + None, + "ⲕ", + None, + "ⲗ", + None, + "ⲙ", + None, + "ⲛ", + None, + "ⲝ", + None, + "ⲟ", + None, + "ⲡ", + None, + "ⲣ", + None, + "ⲥ", + None, + "ⲧ", + None, + "ⲩ", + None, + "ⲫ", + None, + "ⲭ", + None, + "ⲯ", + None, + "ⲱ", + None, + "ⲳ", + None, + "ⲵ", + None, + "ⲷ", + None, + "ⲹ", + None, + "ⲻ", + None, + "ⲽ", + None, + "ⲿ", + None, + "ⳁ", + None, + "ⳃ", + None, + "ⳅ", + None, + "ⳇ", + None, + "ⳉ", + None, + "ⳋ", + None, + "ⳍ", + None, + "ⳏ", + None, + "ⳑ", + None, + "ⳓ", + None, + "ⳕ", + None, + "ⳗ", + None, + "ⳙ", + None, + "ⳛ", + None, + "ⳝ", + None, + "ⳟ", + None, + "ⳡ", + None, + "ⳣ", + None, + "ⳬ", + None, + "ⳮ", + None, + "ⳳ", + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + "ⵡ", + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + "母", + None, + "龟", + None, + "一", + "丨", + "丶", + "丿", + "乙", + "亅", + "二", + "亠", + "人", + "儿", + "入", + "八", + "冂", + "冖", + "冫", + "几", + "凵", + "刀", + "力", + "勹", + "匕", + "匚", + "匸", + "十", + "卜", + "卩", + "厂", + "厶", + "又", + "口", + "囗", + "土", + "士", + "夂", + "夊", + "夕", + "大", + "女", + "子", + "宀", + "寸", + "小", + "尢", + "尸", + "屮", + "山", + "巛", + "工", + "己", + "巾", + "干", + "幺", + "广", + "廴", + "廾", + "弋", + "弓", + "彐", + "彡", + "彳", + "心", + "戈", + "戶", + "手", + "支", + "攴", + "文", + "斗", + "斤", + "方", + "无", + "日", + "曰", + "月", + "木", + "欠", + "止", + "歹", + "殳", + "毋", + "比", + "毛", + "氏", + "气", + "水", + "火", + "爪", + "父", + "爻", + "爿", + "片", + "牙", + "牛", + "犬", + "玄", + "玉", + "瓜", + "瓦", + "甘", + "生", + "用", + "田", + "疋", + "疒", + "癶", + "白", + "皮", + "皿", + "目", + "矛", + "矢", + "石", + "示", + "禸", + "禾", + "穴", + "立", + "竹", + "米", + "糸", + "缶", + "网", + "羊", + "羽", + "老", + "而", + "耒", + "耳", + "聿", + "肉", + "臣", + "自", + "至", + "臼", + "舌", + "舛", + "舟", + "艮", + "色", + "艸", + "虍", + "虫", + "血", + "行", + "衣", + "襾", + "見", + "角", + "言", + "谷", + "豆", + "豕", + "豸", + "貝", + "赤", + "走", + "足", + "身", + "車", + "辛", + "辰", + "辵", + "邑", + "酉", + "釆", + "里", + "金", + "長", + "門", + "阜", + "隶", + "隹", + "雨", + "靑", + "非", + "面", + "革", + "韋", + "韭", + "音", + "頁", + "風", + "飛", + "食", + "首", + "香", + "馬", + "骨", + "高", + "髟", + "鬥", + "鬯", + "鬲", + "鬼", + "魚", + "鳥", + "鹵", + "鹿", + "麥", + "麻", + "黃", + "黍", + "黑", + "黹", + "黽", + "鼎", + "鼓", + "鼠", + "鼻", + "齊", + "齒", + "龍", + "龜", + "龠", + None, + " ", + None, + ".", + None, + "〒", + None, + "十", + "卄", + "卅", + None, + None, + None, + None, + None, + " ゙", + " ゚", + None, + "より", + None, + "コト", + None, + None, + None, + "ᄀ", + "ᄁ", + "ᆪ", + "ᄂ", + "ᆬ", + "ᆭ", + "ᄃ", + "ᄄ", + "ᄅ", + "ᆰ", + "ᆱ", + "ᆲ", + "ᆳ", + "ᆴ", + "ᆵ", + "ᄚ", + "ᄆ", + "ᄇ", + "ᄈ", + "ᄡ", + "ᄉ", + "ᄊ", + "ᄋ", + "ᄌ", + "ᄍ", + "ᄎ", + "ᄏ", + "ᄐ", + "ᄑ", + "ᄒ", + "ᅡ", + "ᅢ", + "ᅣ", + "ᅤ", + "ᅥ", + "ᅦ", + "ᅧ", + "ᅨ", + "ᅩ", + "ᅪ", + "ᅫ", + "ᅬ", + "ᅭ", + "ᅮ", + "ᅯ", + "ᅰ", + "ᅱ", + "ᅲ", + "ᅳ", + "ᅴ", + "ᅵ", + None, + "ᄔ", + "ᄕ", + "ᇇ", + "ᇈ", + "ᇌ", + "ᇎ", + "ᇓ", + "ᇗ", + "ᇙ", + "ᄜ", + "ᇝ", + "ᇟ", + "ᄝ", + "ᄞ", + "ᄠ", + "ᄢ", + "ᄣ", + "ᄧ", + "ᄩ", + "ᄫ", + "ᄬ", + "ᄭ", + "ᄮ", + "ᄯ", + "ᄲ", + "ᄶ", + "ᅀ", + "ᅇ", + "ᅌ", + "ᇱ", + "ᇲ", + "ᅗ", + "ᅘ", + "ᅙ", + "ᆄ", + "ᆅ", + "ᆈ", + "ᆑ", + "ᆒ", + "ᆔ", + "ᆞ", + "ᆡ", + None, + None, + "一", + "二", + "三", + "四", + "上", + "中", + "下", + "甲", + "乙", + "丙", + "丁", + "天", + "地", + "人", + None, + None, + None, + "(ᄀ)", + "(ᄂ)", + "(ᄃ)", + "(ᄅ)", + "(ᄆ)", + "(ᄇ)", + "(ᄉ)", + "(ᄋ)", + "(ᄌ)", + "(ᄎ)", + "(ᄏ)", + "(ᄐ)", + "(ᄑ)", + "(ᄒ)", + "(가)", + "(나)", + "(다)", + "(라)", + "(마)", + "(바)", + "(사)", + "(아)", + "(자)", + "(차)", + "(카)", + "(타)", + "(파)", + "(하)", + "(주)", + "(오전)", + "(오후)", + None, + "(一)", + "(二)", + "(三)", + "(四)", + "(五)", + "(六)", + "(七)", + "(八)", + "(九)", + "(十)", + "(月)", + "(火)", + "(水)", + "(木)", + "(金)", + "(土)", + "(日)", + "(株)", + "(有)", + "(社)", + "(名)", + "(特)", + "(財)", + "(祝)", + "(労)", + "(代)", + "(呼)", + "(学)", + "(監)", + "(企)", + "(資)", + "(協)", + "(祭)", + "(休)", + "(自)", + "(至)", + "問", + "幼", + "文", + "箏", + None, + "pte", + "21", + "22", + "23", + "24", + "25", + "26", + "27", + "28", + "29", + "30", + "31", + "32", + "33", + "34", + "35", + "ᄀ", + "ᄂ", + "ᄃ", + "ᄅ", + "ᄆ", + "ᄇ", + "ᄉ", + "ᄋ", + "ᄌ", + "ᄎ", + "ᄏ", + "ᄐ", + "ᄑ", + "ᄒ", + "가", + "나", + "다", + "라", + "마", + "바", + "사", + "아", + "자", + "차", + "카", + "타", + "파", + "하", + "참고", + "주의", + "우", + None, + "一", + "二", + "三", + "四", + "五", + "六", + "七", + "八", + "九", + "十", + "月", + "火", + "水", + "木", + "金", + "土", + "日", + "株", + "有", + "社", + "名", + "特", + "財", + "祝", + "労", + "秘", + "男", + "女", + "適", + "優", + "印", + "注", + "項", + "休", + "写", + "正", + "上", + "中", + "下", + "左", + "右", + "医", + "宗", + "学", + "監", + "企", + "資", + "協", + "夜", + "36", + "37", + "38", + "39", + "40", + "41", + "42", + "43", + "44", + "45", + "46", + "47", + "48", + "49", + "50", + "1月", + "2月", + "3月", + "4月", + "5月", + "6月", + "7月", + "8月", + "9月", + "10月", + "11月", + "12月", + "hg", + "erg", + "ev", + "ltd", + "ア", + "イ", + "ウ", + "エ", + "オ", + "カ", + "キ", + "ク", + "ケ", + "コ", + "サ", + "シ", + "ス", + "セ", + "ソ", + "タ", + "チ", + "ツ", + "テ", + "ト", + "ナ", + "ニ", + "ヌ", + "ネ", + "ノ", + "ハ", + "ヒ", + "フ", + "ヘ", + "ホ", + "マ", + "ミ", + "ム", + "メ", + "モ", + "ヤ", + "ユ", + "ヨ", + "ラ", + "リ", + "ル", + "レ", + "ロ", + "ワ", + "ヰ", + "ヱ", + "ヲ", + "令和", + "アパート", + "アルファ", + "アンペア", + "アール", + "イニング", + "インチ", + "ウォン", + "エスクード", + "エーカー", + "オンス", + "オーム", + "カイリ", + "カラット", + "カロリー", + "ガロン", + "ガンマ", + "ギガ", + "ギニー", + "キュリー", + "ギルダー", + "キロ", + "キログラム", + "キロメートル", + "キロワット", + "グラム", + "グラムトン", + "クルゼイロ", + "クローネ", + "ケース", + "コルナ", + "コーポ", + "サイクル", + "サンチーム", + "シリング", + "センチ", + "セント", + "ダース", + "デシ", + "ドル", + "トン", + "ナノ", + "ノット", + "ハイツ", + "パーセント", + "パーツ", + "バーレル", + "ピアストル", + "ピクル", + "ピコ", + "ビル", + "ファラッド", + "フィート", + "ブッシェル", + "フラン", + "ヘクタール", + "ペソ", + "ペニヒ", + "ヘルツ", + "ペンス", + "ページ", + "ベータ", + "ポイント", + "ボルト", + "ホン", + "ポンド", + "ホール", + "ホーン", + "マイクロ", + "マイル", + "マッハ", + "マルク", + "マンション", + "ミクロン", + "ミリ", + "ミリバール", + "メガ", + "メガトン", + "メートル", + "ヤード", + "ヤール", + "ユアン", + "リットル", + "リラ", + "ルピー", + "ルーブル", + "レム", + "レントゲン", + "ワット", + "0点", + "1点", + "2点", + "3点", + "4点", + "5点", + "6点", + "7点", + "8点", + "9点", + "10点", + "11点", + "12点", + "13点", + "14点", + "15点", + "16点", + "17点", + "18点", + "19点", + "20点", + "21点", + "22点", + "23点", + "24点", + "hpa", + "da", + "au", + "bar", + "ov", + "pc", + "dm", + "dm2", + "dm3", + "iu", + "平成", + "昭和", + "大正", + "明治", + "株式会社", + "pa", + "na", + "μa", + "ma", + "ka", + "kb", + "mb", + "gb", + "cal", + "kcal", + "pf", + "nf", + "μf", + "μg", + "mg", + "kg", + "hz", + "khz", + "mhz", + "ghz", + "thz", + "μl", + "ml", + "dl", + "kl", + "fm", + "nm", + "μm", + "mm", + "cm", + "km", + "mm2", + "cm2", + "m2", + "km2", + "mm3", + "cm3", + "m3", + "km3", + "m∕s", + "m∕s2", + "pa", + "kpa", + "mpa", + "gpa", + "rad", + "rad∕s", + "rad∕s2", + "ps", + "ns", + "μs", + "ms", + "pv", + "nv", + "μv", + "mv", + "kv", + "mv", + "pw", + "nw", + "μw", + "mw", + "kw", + "mw", + "kω", + "mω", + None, + "bq", + "cc", + "cd", + "c∕kg", + None, + "db", + "gy", + "ha", + "hp", + "in", + "kk", + "km", + "kt", + "lm", + "ln", + "log", + "lx", + "mb", + "mil", + "mol", + "ph", + None, + "ppm", + "pr", + "sr", + "sv", + "wb", + "v∕m", + "a∕m", + "1日", + "2日", + "3日", + "4日", + "5日", + "6日", + "7日", + "8日", + "9日", + "10日", + "11日", + "12日", + "13日", + "14日", + "15日", + "16日", + "17日", + "18日", + "19日", + "20日", + "21日", + "22日", + "23日", + "24日", + "25日", + "26日", + "27日", + "28日", + "29日", + "30日", + "31日", + "gal", + None, + None, + None, + None, + None, + None, + "ꙁ", + None, + "ꙃ", + None, + "ꙅ", + None, + "ꙇ", + None, + "ꙉ", + None, + "ꙋ", + None, + "ꙍ", + None, + "ꙏ", + None, + "ꙑ", + None, + "ꙓ", + None, + "ꙕ", + None, + "ꙗ", + None, + "ꙙ", + None, + "ꙛ", + None, + "ꙝ", + None, + "ꙟ", + None, + "ꙡ", + None, + "ꙣ", + None, + "ꙥ", + None, + "ꙧ", + None, + "ꙩ", + None, + "ꙫ", + None, + "ꙭ", + None, + "ꚁ", + None, + "ꚃ", + None, + "ꚅ", + None, + "ꚇ", + None, + "ꚉ", + None, + "ꚋ", + None, + "ꚍ", + None, + "ꚏ", + None, + "ꚑ", + None, + "ꚓ", + None, + "ꚕ", + None, + "ꚗ", + None, + "ꚙ", + None, + "ꚛ", + None, + "ъ", + "ь", + None, + None, + None, + "ꜣ", + None, + "ꜥ", + None, + "ꜧ", + None, + "ꜩ", + None, + "ꜫ", + None, + "ꜭ", + None, + "ꜯ", + None, + "ꜳ", + None, + "ꜵ", + None, + "ꜷ", + None, + "ꜹ", + None, + "ꜻ", + None, + "ꜽ", + None, + "ꜿ", + None, + "ꝁ", + None, + "ꝃ", + None, + "ꝅ", + None, + "ꝇ", + None, + "ꝉ", + None, + "ꝋ", + None, + "ꝍ", + None, + "ꝏ", + None, + "ꝑ", + None, + "ꝓ", + None, + "ꝕ", + None, + "ꝗ", + None, + "ꝙ", + None, + "ꝛ", + None, + "ꝝ", + None, + "ꝟ", + None, + "ꝡ", + None, + "ꝣ", + None, + "ꝥ", + None, + "ꝧ", + None, + "ꝩ", + None, + "ꝫ", + None, + "ꝭ", + None, + "ꝯ", + None, + "ꝯ", + None, + "ꝺ", + None, + "ꝼ", + None, + "ᵹ", + "ꝿ", + None, + "ꞁ", + None, + "ꞃ", + None, + "ꞅ", + None, + "ꞇ", + None, + "ꞌ", + None, + "ɥ", + None, + "ꞑ", + None, + "ꞓ", + None, + "ꞗ", + None, + "ꞙ", + None, + "ꞛ", + None, + "ꞝ", + None, + "ꞟ", + None, + "ꞡ", + None, + "ꞣ", + None, + "ꞥ", + None, + "ꞧ", + None, + "ꞩ", + None, + "ɦ", + "ɜ", + "ɡ", + "ɬ", + "ɪ", + None, + "ʞ", + "ʇ", + "ʝ", + "ꭓ", + "ꞵ", + None, + "ꞷ", + None, + "ꞹ", + None, + "ꞻ", + None, + "ꞽ", + None, + "ꞿ", + None, + "ꟁ", + None, + "ꟃ", + None, + "ꞔ", + "ʂ", + "ᶎ", + "ꟈ", + None, + "ꟊ", + None, + "ɤ", + "ꟍ", + None, + "꟏", + None, + "ꟑ", + None, + "ꟓ", + None, + "ꟕ", + None, + "ꟗ", + None, + "ꟙ", + None, + "ꟛ", + None, + "ƛ", + None, + "s", + "c", + "f", + "q", + "ꟶ", + None, + "ħ", + "œ", + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + "ꜧ", + "ꬷ", + "ɫ", + "ꭒ", + None, + "ʍ", + None, + None, + "Ꭰ", + "Ꭱ", + "Ꭲ", + "Ꭳ", + "Ꭴ", + "Ꭵ", + "Ꭶ", + "Ꭷ", + "Ꭸ", + "Ꭹ", + "Ꭺ", + "Ꭻ", + "Ꭼ", + "Ꭽ", + "Ꭾ", + "Ꭿ", + "Ꮀ", + "Ꮁ", + "Ꮂ", + "Ꮃ", + "Ꮄ", + "Ꮅ", + "Ꮆ", + "Ꮇ", + "Ꮈ", + "Ꮉ", + "Ꮊ", + "Ꮋ", + "Ꮌ", + "Ꮍ", + "Ꮎ", + "Ꮏ", + "Ꮐ", + "Ꮑ", + "Ꮒ", + "Ꮓ", + "Ꮔ", + "Ꮕ", + "Ꮖ", + "Ꮗ", + "Ꮘ", + "Ꮙ", + "Ꮚ", + "Ꮛ", + "Ꮜ", + "Ꮝ", + "Ꮞ", + "Ꮟ", + "Ꮠ", + "Ꮡ", + "Ꮢ", + "Ꮣ", + "Ꮤ", + "Ꮥ", + "Ꮦ", + "Ꮧ", + "Ꮨ", + "Ꮩ", + "Ꮪ", + "Ꮫ", + "Ꮬ", + "Ꮭ", + "Ꮮ", + "Ꮯ", + "Ꮰ", + "Ꮱ", + "Ꮲ", + "Ꮳ", + "Ꮴ", + "Ꮵ", + "Ꮶ", + "Ꮷ", + "Ꮸ", + "Ꮹ", + "Ꮺ", + "Ꮻ", + "Ꮼ", + "Ꮽ", + "Ꮾ", + "Ꮿ", + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + "豈", + "更", + "車", + "賈", + "滑", + "串", + "句", + "龜", + "契", + "金", + "喇", + "奈", + "懶", + "癩", + "羅", + "蘿", + "螺", + "裸", + "邏", + "樂", + "洛", + "烙", + "珞", + "落", + "酪", + "駱", + "亂", + "卵", + "欄", + "爛", + "蘭", + "鸞", + "嵐", + "濫", + "藍", + "襤", + "拉", + "臘", + "蠟", + "廊", + "朗", + "浪", + "狼", + "郎", + "來", + "冷", + "勞", + "擄", + "櫓", + "爐", + "盧", + "老", + "蘆", + "虜", + "路", + "露", + "魯", + "鷺", + "碌", + "祿", + "綠", + "菉", + "錄", + "鹿", + "論", + "壟", + "弄", + "籠", + "聾", + "牢", + "磊", + "賂", + "雷", + "壘", + "屢", + "樓", + "淚", + "漏", + "累", + "縷", + "陋", + "勒", + "肋", + "凜", + "凌", + "稜", + "綾", + "菱", + "陵", + "讀", + "拏", + "樂", + "諾", + "丹", + "寧", + "怒", + "率", + "異", + "北", + "磻", + "便", + "復", + "不", + "泌", + "數", + "索", + "參", + "塞", + "省", + "葉", + "說", + "殺", + "辰", + "沈", + "拾", + "若", + "掠", + "略", + "亮", + "兩", + "凉", + "梁", + "糧", + "良", + "諒", + "量", + "勵", + "呂", + "女", + "廬", + "旅", + "濾", + "礪", + "閭", + "驪", + "麗", + "黎", + "力", + "曆", + "歷", + "轢", + "年", + "憐", + "戀", + "撚", + "漣", + "煉", + "璉", + "秊", + "練", + "聯", + "輦", + "蓮", + "連", + "鍊", + "列", + "劣", + "咽", + "烈", + "裂", + "說", + "廉", + "念", + "捻", + "殮", + "簾", + "獵", + "令", + "囹", + "寧", + "嶺", + "怜", + "玲", + "瑩", + "羚", + "聆", + "鈴", + "零", + "靈", + "領", + "例", + "禮", + "醴", + "隸", + "惡", + "了", + "僚", + "寮", + "尿", + "料", + "樂", + "燎", + "療", + "蓼", + "遼", + "龍", + "暈", + "阮", + "劉", + "杻", + "柳", + "流", + "溜", + "琉", + "留", + "硫", + "紐", + "類", + "六", + "戮", + "陸", + "倫", + "崙", + "淪", + "輪", + "律", + "慄", + "栗", + "率", + "隆", + "利", + "吏", + "履", + "易", + "李", + "梨", + "泥", + "理", + "痢", + "罹", + "裏", + "裡", + "里", + "離", + "匿", + "溺", + "吝", + "燐", + "璘", + "藺", + "隣", + "鱗", + "麟", + "林", + "淋", + "臨", + "立", + "笠", + "粒", + "狀", + "炙", + "識", + "什", + "茶", + "刺", + "切", + "度", + "拓", + "糖", + "宅", + "洞", + "暴", + "輻", + "行", + "降", + "見", + "廓", + "兀", + "嗀", + None, + "塚", + None, + "晴", + None, + "凞", + "猪", + "益", + "礼", + "神", + "祥", + "福", + "靖", + "精", + "羽", + None, + "蘒", + None, + "諸", + None, + "逸", + "都", + None, + "飯", + "飼", + "館", + "鶴", + "郞", + "隷", + "侮", + "僧", + "免", + "勉", + "勤", + "卑", + "喝", + "嘆", + "器", + "塀", + "墨", + "層", + "屮", + "悔", + "慨", + "憎", + "懲", + "敏", + "既", + "暑", + "梅", + "海", + "渚", + "漢", + "煮", + "爫", + "琢", + "碑", + "社", + "祉", + "祈", + "祐", + "祖", + "祝", + "禍", + "禎", + "穀", + "突", + "節", + "練", + "縉", + "繁", + "署", + "者", + "臭", + "艹", + "著", + "褐", + "視", + "謁", + "謹", + "賓", + "贈", + "辶", + "逸", + "難", + "響", + "頻", + "恵", + "𤋮", + "舘", + None, + "並", + "况", + "全", + "侀", + "充", + "冀", + "勇", + "勺", + "喝", + "啕", + "喙", + "嗢", + "塚", + "墳", + "奄", + "奔", + "婢", + "嬨", + "廒", + "廙", + "彩", + "徭", + "惘", + "慎", + "愈", + "憎", + "慠", + "懲", + "戴", + "揄", + "搜", + "摒", + "敖", + "晴", + "朗", + "望", + "杖", + "歹", + "殺", + "流", + "滛", + "滋", + "漢", + "瀞", + "煮", + "瞧", + "爵", + "犯", + "猪", + "瑱", + "甆", + "画", + "瘝", + "瘟", + "益", + "盛", + "直", + "睊", + "着", + "磌", + "窱", + "節", + "类", + "絛", + "練", + "缾", + "者", + "荒", + "華", + "蝹", + "襁", + "覆", + "視", + "調", + "諸", + "請", + "謁", + "諾", + "諭", + "謹", + "變", + "贈", + "輸", + "遲", + "醙", + "鉶", + "陼", + "難", + "靖", + "韛", + "響", + "頋", + "頻", + "鬒", + "龜", + "𢡊", + "𢡄", + "𣏕", + "㮝", + "䀘", + "䀹", + "𥉉", + "𥳐", + "𧻓", + "齃", + "龎", + None, + "ff", + "fi", + "fl", + "ffi", + "ffl", + "st", + None, + "մն", + "մե", + "մի", + "վն", + "մխ", + None, + "יִ", + None, + "ײַ", + "ע", + "א", + "ד", + "ה", + "כ", + "ל", + "ם", + "ר", + "ת", + "+", + "שׁ", + "שׂ", + "שּׁ", + "שּׂ", + "אַ", + "אָ", + "אּ", + "בּ", + "גּ", + "דּ", + "הּ", + "וּ", + "זּ", + None, + "טּ", + "יּ", + "ךּ", + "כּ", + "לּ", + None, + "מּ", + None, + "נּ", + "סּ", + None, + "ףּ", + "פּ", + None, + "צּ", + "קּ", + "רּ", + "שּ", + "תּ", + "וֹ", + "בֿ", + "כֿ", + "פֿ", + "אל", + "ٱ", + "ٻ", + "پ", + "ڀ", + "ٺ", + "ٿ", + "ٹ", + "ڤ", + "ڦ", + "ڄ", + "ڃ", + "چ", + "ڇ", + "ڍ", + "ڌ", + "ڎ", + "ڈ", + "ژ", + "ڑ", + "ک", + "گ", + "ڳ", + "ڱ", + "ں", + "ڻ", + "ۀ", + "ہ", + "ھ", + "ے", + "ۓ", + None, + "ڭ", + "ۇ", + "ۆ", + "ۈ", + "ۇٴ", + "ۋ", + "ۅ", + "ۉ", + "ې", + "ى", + "ئا", + "ئە", + "ئو", + "ئۇ", + "ئۆ", + "ئۈ", + "ئې", + "ئى", + "ی", + "ئج", + "ئح", + "ئم", + "ئى", + "ئي", + "بج", + "بح", + "بخ", + "بم", + "بى", + "بي", + "تج", + "تح", + "تخ", + "تم", + "تى", + "تي", + "ثج", + "ثم", + "ثى", + "ثي", + "جح", + "جم", + "حج", + "حم", + "خج", + "خح", + "خم", + "سج", + "سح", + "سخ", + "سم", + "صح", + "صم", + "ضج", + "ضح", + "ضخ", + "ضم", + "طح", + "طم", + "ظم", + "عج", + "عم", + "غج", + "غم", + "فج", + "فح", + "فخ", + "فم", + "فى", + "في", + "قح", + "قم", + "قى", + "قي", + "كا", + "كج", + "كح", + "كخ", + "كل", + "كم", + "كى", + "كي", + "لج", + "لح", + "لخ", + "لم", + "لى", + "لي", + "مج", + "مح", + "مخ", + "مم", + "مى", + "مي", + "نج", + "نح", + "نخ", + "نم", + "نى", + "ني", + "هج", + "هم", + "هى", + "هي", + "يج", + "يح", + "يخ", + "يم", + "يى", + "يي", + "ذٰ", + "رٰ", + "ىٰ", + " ٌّ", + " ٍّ", + " َّ", + " ُّ", + " ِّ", + " ّٰ", + "ئر", + "ئز", + "ئم", + "ئن", + "ئى", + "ئي", + "بر", + "بز", + "بم", + "بن", + "بى", + "بي", + "تر", + "تز", + "تم", + "تن", + "تى", + "تي", + "ثر", + "ثز", + "ثم", + "ثن", + "ثى", + "ثي", + "فى", + "في", + "قى", + "قي", + "كا", + "كل", + "كم", + "كى", + "كي", + "لم", + "لى", + "لي", + "ما", + "مم", + "نر", + "نز", + "نم", + "نن", + "نى", + "ني", + "ىٰ", + "ير", + "يز", + "يم", + "ين", + "يى", + "يي", + "ئج", + "ئح", + "ئخ", + "ئم", + "ئه", + "بج", + "بح", + "بخ", + "بم", + "به", + "تج", + "تح", + "تخ", + "تم", + "ته", + "ثم", + "جح", + "جم", + "حج", + "حم", + "خج", + "خم", + "سج", + "سح", + "سخ", + "سم", + "صح", + "صخ", + "صم", + "ضج", + "ضح", + "ضخ", + "ضم", + "طح", + "ظم", + "عج", + "عم", + "غج", + "غم", + "فج", + "فح", + "فخ", + "فم", + "قح", + "قم", + "كج", + "كح", + "كخ", + "كل", + "كم", + "لج", + "لح", + "لخ", + "لم", + "له", + "مج", + "مح", + "مخ", + "مم", + "نج", + "نح", + "نخ", + "نم", + "نه", + "هج", + "هم", + "هٰ", + "يج", + "يح", + "يخ", + "يم", + "يه", + "ئم", + "ئه", + "بم", + "به", + "تم", + "ته", + "ثم", + "ثه", + "سم", + "سه", + "شم", + "شه", + "كل", + "كم", + "لم", + "نم", + "نه", + "يم", + "يه", + "ـَّ", + "ـُّ", + "ـِّ", + "طى", + "طي", + "عى", + "عي", + "غى", + "غي", + "سى", + "سي", + "شى", + "شي", + "حى", + "حي", + "جى", + "جي", + "خى", + "خي", + "صى", + "صي", + "ضى", + "ضي", + "شج", + "شح", + "شخ", + "شم", + "شر", + "سر", + "صر", + "ضر", + "طى", + "طي", + "عى", + "عي", + "غى", + "غي", + "سى", + "سي", + "شى", + "شي", + "حى", + "حي", + "جى", + "جي", + "خى", + "خي", + "صى", + "صي", + "ضى", + "ضي", + "شج", + "شح", + "شخ", + "شم", + "شر", + "سر", + "صر", + "ضر", + "شج", + "شح", + "شخ", + "شم", + "سه", + "شه", + "طم", + "سج", + "سح", + "سخ", + "شج", + "شح", + "شخ", + "طم", + "ظم", + "اً", + None, + "تجم", + "تحج", + "تحم", + "تخم", + "تمج", + "تمح", + "تمخ", + "جمح", + "حمي", + "حمى", + "سحج", + "سجح", + "سجى", + "سمح", + "سمج", + "سمم", + "صحح", + "صمم", + "شحم", + "شجي", + "شمخ", + "شمم", + "ضحى", + "ضخم", + "طمح", + "طمم", + "طمي", + "عجم", + "عمم", + "عمى", + "غمم", + "غمي", + "غمى", + "فخم", + "قمح", + "قمم", + "لحم", + "لحي", + "لحى", + "لجج", + "لخم", + "لمح", + "محج", + "محم", + "محي", + "مجح", + "مجم", + "مخج", + "مخم", + None, + "مجخ", + "همج", + "همم", + "نحم", + "نحى", + "نجم", + "نجى", + "نمي", + "نمى", + "يمم", + "بخي", + "تجي", + "تجى", + "تخي", + "تخى", + "تمي", + "تمى", + "جمي", + "جحى", + "جمى", + "سخى", + "صحي", + "شحي", + "ضحي", + "لجي", + "لمي", + "يحي", + "يجي", + "يمي", + "ممي", + "قمي", + "نحي", + "قمح", + "لحم", + "عمي", + "كمي", + "نجح", + "مخي", + "لجم", + "كمم", + "لجم", + "نجح", + "جحي", + "حجي", + "مجي", + "فمي", + "بحي", + "كمم", + "عجم", + "صمم", + "سخي", + "نجي", + None, + None, + "صلے", + "قلے", + "الله", + "اكبر", + "محمد", + "صلعم", + "رسول", + "عليه", + "وسلم", + "صلى", + "صلى الله عليه وسلم", + "جل جلاله", + "ریال", + None, + None, + ",", + "、", + None, + ":", + ";", + "!", + "?", + "〖", + "〗", + None, + None, + None, + "—", + "–", + "_", + "(", + ")", + "{", + "}", + "〔", + "〕", + "【", + "】", + "《", + "》", + "〈", + "〉", + "「", + "」", + "『", + "』", + None, + "[", + "]", + " ̅", + "_", + ",", + "、", + None, + ";", + ":", + "?", + "!", + "—", + "(", + ")", + "{", + "}", + "〔", + "〕", + "#", + "&", + "*", + "+", + "-", + "<", + ">", + "=", + None, + "\\", + "$", + "%", + "@", + None, + " ً", + "ـً", + " ٌ", + None, + " ٍ", + None, + " َ", + "ـَ", + " ُ", + "ـُ", + " ِ", + "ـِ", + " ّ", + "ـّ", + " ْ", + "ـْ", + "ء", + "آ", + "أ", + "ؤ", + "إ", + "ئ", + "ا", + "ب", + "ة", + "ت", + "ث", + "ج", + "ح", + "خ", + "د", + "ذ", + "ر", + "ز", + "س", + "ش", + "ص", + "ض", + "ط", + "ظ", + "ع", + "غ", + "ف", + "ق", + "ك", + "ل", + "م", + "ن", + "ه", + "و", + "ى", + "ي", + "لآ", + "لأ", + "لإ", + "لا", + None, + None, + None, + "!", + '"', + "#", + "$", + "%", + "&", + "'", + "(", + ")", + "*", + "+", + ",", + "-", + ".", + "/", + "0", + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + ":", + ";", + "<", + "=", + ">", + "?", + "@", + "a", + "b", + "c", + "d", + "e", + "f", + "g", + "h", + "i", + "j", + "k", + "l", + "m", + "n", + "o", + "p", + "q", + "r", + "s", + "t", + "u", + "v", + "w", + "x", + "y", + "z", + "[", + "\\", + "]", + "^", + "_", + "`", + "a", + "b", + "c", + "d", + "e", + "f", + "g", + "h", + "i", + "j", + "k", + "l", + "m", + "n", + "o", + "p", + "q", + "r", + "s", + "t", + "u", + "v", + "w", + "x", + "y", + "z", + "{", + "|", + "}", + "~", + "⦅", + "⦆", + ".", + "「", + "」", + "、", + "・", + "ヲ", + "ァ", + "ィ", + "ゥ", + "ェ", + "ォ", + "ャ", + "ュ", + "ョ", + "ッ", + "ー", + "ア", + "イ", + "ウ", + "エ", + "オ", + "カ", + "キ", + "ク", + "ケ", + "コ", + "サ", + "シ", + "ス", + "セ", + "ソ", + "タ", + "チ", + "ツ", + "テ", + "ト", + "ナ", + "ニ", + "ヌ", + "ネ", + "ノ", + "ハ", + "ヒ", + "フ", + "ヘ", + "ホ", + "マ", + "ミ", + "ム", + "メ", + "モ", + "ヤ", + "ユ", + "ヨ", + "ラ", + "リ", + "ル", + "レ", + "ロ", + "ワ", + "ン", + "゙", + "゚", + None, + "ᄀ", + "ᄁ", + "ᆪ", + "ᄂ", + "ᆬ", + "ᆭ", + "ᄃ", + "ᄄ", + "ᄅ", + "ᆰ", + "ᆱ", + "ᆲ", + "ᆳ", + "ᆴ", + "ᆵ", + "ᄚ", + "ᄆ", + "ᄇ", + "ᄈ", + "ᄡ", + "ᄉ", + "ᄊ", + "ᄋ", + "ᄌ", + "ᄍ", + "ᄎ", + "ᄏ", + "ᄐ", + "ᄑ", + "ᄒ", + None, + "ᅡ", + "ᅢ", + "ᅣ", + "ᅤ", + "ᅥ", + "ᅦ", + None, + "ᅧ", + "ᅨ", + "ᅩ", + "ᅪ", + "ᅫ", + "ᅬ", + None, + "ᅭ", + "ᅮ", + "ᅯ", + "ᅰ", + "ᅱ", + "ᅲ", + None, + "ᅳ", + "ᅴ", + "ᅵ", + None, + "¢", + "£", + "¬", + " ̄", + "¦", + "¥", + "₩", + None, + "│", + "←", + "↑", + "→", + "↓", + "■", + "○", + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + "𐐨", + "𐐩", + "𐐪", + "𐐫", + "𐐬", + "𐐭", + "𐐮", + "𐐯", + "𐐰", + "𐐱", + "𐐲", + "𐐳", + "𐐴", + "𐐵", + "𐐶", + "𐐷", + "𐐸", + "𐐹", + "𐐺", + "𐐻", + "𐐼", + "𐐽", + "𐐾", + "𐐿", + "𐑀", + "𐑁", + "𐑂", + "𐑃", + "𐑄", + "𐑅", + "𐑆", + "𐑇", + "𐑈", + "𐑉", + "𐑊", + "𐑋", + "𐑌", + "𐑍", + "𐑎", + "𐑏", + None, + None, + None, + None, + "𐓘", + "𐓙", + "𐓚", + "𐓛", + "𐓜", + "𐓝", + "𐓞", + "𐓟", + "𐓠", + "𐓡", + "𐓢", + "𐓣", + "𐓤", + "𐓥", + "𐓦", + "𐓧", + "𐓨", + "𐓩", + "𐓪", + "𐓫", + "𐓬", + "𐓭", + "𐓮", + "𐓯", + "𐓰", + "𐓱", + "𐓲", + "𐓳", + "𐓴", + "𐓵", + "𐓶", + "𐓷", + "𐓸", + "𐓹", + "𐓺", + "𐓻", + None, + None, + None, + None, + None, + None, + None, + None, + "𐖗", + "𐖘", + "𐖙", + "𐖚", + "𐖛", + "𐖜", + "𐖝", + "𐖞", + "𐖟", + "𐖠", + "𐖡", + None, + "𐖣", + "𐖤", + "𐖥", + "𐖦", + "𐖧", + "𐖨", + "𐖩", + "𐖪", + "𐖫", + "𐖬", + "𐖭", + "𐖮", + "𐖯", + "𐖰", + "𐖱", + None, + "𐖳", + "𐖴", + "𐖵", + "𐖶", + "𐖷", + "𐖸", + "𐖹", + None, + "𐖻", + "𐖼", + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + "ː", + "ˑ", + "æ", + "ʙ", + "ɓ", + None, + "ʣ", + "ꭦ", + "ʥ", + "ʤ", + "ɖ", + "ɗ", + "ᶑ", + "ɘ", + "ɞ", + "ʩ", + "ɤ", + "ɢ", + "ɠ", + "ʛ", + "ħ", + "ʜ", + "ɧ", + "ʄ", + "ʪ", + "ʫ", + "ɬ", + "𝼄", + "ꞎ", + "ɮ", + "𝼅", + "ʎ", + "𝼆", + "ø", + "ɶ", + "ɷ", + "q", + "ɺ", + "𝼈", + "ɽ", + "ɾ", + "ʀ", + "ʨ", + "ʦ", + "ꭧ", + "ʧ", + "ʈ", + "ⱱ", + None, + "ʏ", + "ʡ", + "ʢ", + "ʘ", + "ǀ", + "ǁ", + "ǂ", + "𝼊", + "𝼞", + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + "𐳀", + "𐳁", + "𐳂", + "𐳃", + "𐳄", + "𐳅", + "𐳆", + "𐳇", + "𐳈", + "𐳉", + "𐳊", + "𐳋", + "𐳌", + "𐳍", + "𐳎", + "𐳏", + "𐳐", + "𐳑", + "𐳒", + "𐳓", + "𐳔", + "𐳕", + "𐳖", + "𐳗", + "𐳘", + "𐳙", + "𐳚", + "𐳛", + "𐳜", + "𐳝", + "𐳞", + "𐳟", + "𐳠", + "𐳡", + "𐳢", + "𐳣", + "𐳤", + "𐳥", + "𐳦", + "𐳧", + "𐳨", + "𐳩", + "𐳪", + "𐳫", + "𐳬", + "𐳭", + "𐳮", + "𐳯", + "𐳰", + "𐳱", + "𐳲", + None, + None, + None, + None, + None, + None, + None, + None, + "𐵰", + "𐵱", + "𐵲", + "𐵳", + "𐵴", + "𐵵", + "𐵶", + "𐵷", + "𐵸", + "𐵹", + "𐵺", + "𐵻", + "𐵼", + "𐵽", + "𐵾", + "𐵿", + "𐶀", + "𐶁", + "𐶂", + "𐶃", + "𐶄", + "𐶅", + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + "𑣀", + "𑣁", + "𑣂", + "𑣃", + "𑣄", + "𑣅", + "𑣆", + "𑣇", + "𑣈", + "𑣉", + "𑣊", + "𑣋", + "𑣌", + "𑣍", + "𑣎", + "𑣏", + "𑣐", + "𑣑", + "𑣒", + "𑣓", + "𑣔", + "𑣕", + "𑣖", + "𑣗", + "𑣘", + "𑣙", + "𑣚", + "𑣛", + "𑣜", + "𑣝", + "𑣞", + "𑣟", + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + "𖹠", + "𖹡", + "𖹢", + "𖹣", + "𖹤", + "𖹥", + "𖹦", + "𖹧", + "𖹨", + "𖹩", + "𖹪", + "𖹫", + "𖹬", + "𖹭", + "𖹮", + "𖹯", + "𖹰", + "𖹱", + "𖹲", + "𖹳", + "𖹴", + "𖹵", + "𖹶", + "𖹷", + "𖹸", + "𖹹", + "𖹺", + "𖹻", + "𖹼", + "𖹽", + "𖹾", + "𖹿", + None, + None, + "𖺻", + "𖺼", + "𖺽", + "𖺾", + "𖺿", + "𖻀", + "𖻁", + "𖻂", + "𖻃", + "𖻄", + "𖻅", + "𖻆", + "𖻇", + "𖻈", + "𖻉", + "𖻊", + "𖻋", + "𖻌", + "𖻍", + "𖻎", + "𖻏", + "𖻐", + "𖻑", + "𖻒", + "𖻓", + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + "a", + "b", + "c", + "d", + "e", + "f", + "g", + "h", + "i", + "j", + "k", + "l", + "m", + "n", + "o", + "p", + "q", + "r", + "s", + "t", + "u", + "v", + "w", + "x", + "y", + "z", + "0", + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + "𝅗𝅥", + "𝅘𝅥", + "𝅘𝅥𝅮", + "𝅘𝅥𝅯", + "𝅘𝅥𝅰", + "𝅘𝅥𝅱", + "𝅘𝅥𝅲", + None, + None, + None, + "𝆹𝅥", + "𝆺𝅥", + "𝆹𝅥𝅮", + "𝆺𝅥𝅮", + "𝆹𝅥𝅯", + "𝆺𝅥𝅯", + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + "a", + "b", + "c", + "d", + "e", + "f", + "g", + "h", + "i", + "j", + "k", + "l", + "m", + "n", + "o", + "p", + "q", + "r", + "s", + "t", + "u", + "v", + "w", + "x", + "y", + "z", + "a", + "b", + "c", + "d", + "e", + "f", + "g", + "h", + "i", + "j", + "k", + "l", + "m", + "n", + "o", + "p", + "q", + "r", + "s", + "t", + "u", + "v", + "w", + "x", + "y", + "z", + "a", + "b", + "c", + "d", + "e", + "f", + "g", + "h", + "i", + "j", + "k", + "l", + "m", + "n", + "o", + "p", + "q", + "r", + "s", + "t", + "u", + "v", + "w", + "x", + "y", + "z", + "a", + "b", + "c", + "d", + "e", + "f", + "g", + None, + "i", + "j", + "k", + "l", + "m", + "n", + "o", + "p", + "q", + "r", + "s", + "t", + "u", + "v", + "w", + "x", + "y", + "z", + "a", + "b", + "c", + "d", + "e", + "f", + "g", + "h", + "i", + "j", + "k", + "l", + "m", + "n", + "o", + "p", + "q", + "r", + "s", + "t", + "u", + "v", + "w", + "x", + "y", + "z", + "a", + "b", + "c", + "d", + "e", + "f", + "g", + "h", + "i", + "j", + "k", + "l", + "m", + "n", + "o", + "p", + "q", + "r", + "s", + "t", + "u", + "v", + "w", + "x", + "y", + "z", + "a", + None, + "c", + "d", + None, + "g", + None, + "j", + "k", + None, + "n", + "o", + "p", + "q", + None, + "s", + "t", + "u", + "v", + "w", + "x", + "y", + "z", + "a", + "b", + "c", + "d", + None, + "f", + None, + "h", + "i", + "j", + "k", + "l", + "m", + "n", + None, + "p", + "q", + "r", + "s", + "t", + "u", + "v", + "w", + "x", + "y", + "z", + "a", + "b", + "c", + "d", + "e", + "f", + "g", + "h", + "i", + "j", + "k", + "l", + "m", + "n", + "o", + "p", + "q", + "r", + "s", + "t", + "u", + "v", + "w", + "x", + "y", + "z", + "a", + "b", + "c", + "d", + "e", + "f", + "g", + "h", + "i", + "j", + "k", + "l", + "m", + "n", + "o", + "p", + "q", + "r", + "s", + "t", + "u", + "v", + "w", + "x", + "y", + "z", + "a", + "b", + None, + "d", + "e", + "f", + "g", + None, + "j", + "k", + "l", + "m", + "n", + "o", + "p", + "q", + None, + "s", + "t", + "u", + "v", + "w", + "x", + "y", + None, + "a", + "b", + "c", + "d", + "e", + "f", + "g", + "h", + "i", + "j", + "k", + "l", + "m", + "n", + "o", + "p", + "q", + "r", + "s", + "t", + "u", + "v", + "w", + "x", + "y", + "z", + "a", + "b", + None, + "d", + "e", + "f", + "g", + None, + "i", + "j", + "k", + "l", + "m", + None, + "o", + None, + "s", + "t", + "u", + "v", + "w", + "x", + "y", + None, + "a", + "b", + "c", + "d", + "e", + "f", + "g", + "h", + "i", + "j", + "k", + "l", + "m", + "n", + "o", + "p", + "q", + "r", + "s", + "t", + "u", + "v", + "w", + "x", + "y", + "z", + "a", + "b", + "c", + "d", + "e", + "f", + "g", + "h", + "i", + "j", + "k", + "l", + "m", + "n", + "o", + "p", + "q", + "r", + "s", + "t", + "u", + "v", + "w", + "x", + "y", + "z", + "a", + "b", + "c", + "d", + "e", + "f", + "g", + "h", + "i", + "j", + "k", + "l", + "m", + "n", + "o", + "p", + "q", + "r", + "s", + "t", + "u", + "v", + "w", + "x", + "y", + "z", + "a", + "b", + "c", + "d", + "e", + "f", + "g", + "h", + "i", + "j", + "k", + "l", + "m", + "n", + "o", + "p", + "q", + "r", + "s", + "t", + "u", + "v", + "w", + "x", + "y", + "z", + "a", + "b", + "c", + "d", + "e", + "f", + "g", + "h", + "i", + "j", + "k", + "l", + "m", + "n", + "o", + "p", + "q", + "r", + "s", + "t", + "u", + "v", + "w", + "x", + "y", + "z", + "a", + "b", + "c", + "d", + "e", + "f", + "g", + "h", + "i", + "j", + "k", + "l", + "m", + "n", + "o", + "p", + "q", + "r", + "s", + "t", + "u", + "v", + "w", + "x", + "y", + "z", + "a", + "b", + "c", + "d", + "e", + "f", + "g", + "h", + "i", + "j", + "k", + "l", + "m", + "n", + "o", + "p", + "q", + "r", + "s", + "t", + "u", + "v", + "w", + "x", + "y", + "z", + "a", + "b", + "c", + "d", + "e", + "f", + "g", + "h", + "i", + "j", + "k", + "l", + "m", + "n", + "o", + "p", + "q", + "r", + "s", + "t", + "u", + "v", + "w", + "x", + "y", + "z", + "a", + "b", + "c", + "d", + "e", + "f", + "g", + "h", + "i", + "j", + "k", + "l", + "m", + "n", + "o", + "p", + "q", + "r", + "s", + "t", + "u", + "v", + "w", + "x", + "y", + "z", + "a", + "b", + "c", + "d", + "e", + "f", + "g", + "h", + "i", + "j", + "k", + "l", + "m", + "n", + "o", + "p", + "q", + "r", + "s", + "t", + "u", + "v", + "w", + "x", + "y", + "z", + "a", + "b", + "c", + "d", + "e", + "f", + "g", + "h", + "i", + "j", + "k", + "l", + "m", + "n", + "o", + "p", + "q", + "r", + "s", + "t", + "u", + "v", + "w", + "x", + "y", + "z", + "a", + "b", + "c", + "d", + "e", + "f", + "g", + "h", + "i", + "j", + "k", + "l", + "m", + "n", + "o", + "p", + "q", + "r", + "s", + "t", + "u", + "v", + "w", + "x", + "y", + "z", + "a", + "b", + "c", + "d", + "e", + "f", + "g", + "h", + "i", + "j", + "k", + "l", + "m", + "n", + "o", + "p", + "q", + "r", + "s", + "t", + "u", + "v", + "w", + "x", + "y", + "z", + "ı", + "ȷ", + None, + "α", + "β", + "γ", + "δ", + "ε", + "ζ", + "η", + "θ", + "ι", + "κ", + "λ", + "μ", + "ν", + "ξ", + "ο", + "π", + "ρ", + "θ", + "σ", + "τ", + "υ", + "φ", + "χ", + "ψ", + "ω", + "∇", + "α", + "β", + "γ", + "δ", + "ε", + "ζ", + "η", + "θ", + "ι", + "κ", + "λ", + "μ", + "ν", + "ξ", + "ο", + "π", + "ρ", + "σ", + "τ", + "υ", + "φ", + "χ", + "ψ", + "ω", + "∂", + "ε", + "θ", + "κ", + "φ", + "ρ", + "π", + "α", + "β", + "γ", + "δ", + "ε", + "ζ", + "η", + "θ", + "ι", + "κ", + "λ", + "μ", + "ν", + "ξ", + "ο", + "π", + "ρ", + "θ", + "σ", + "τ", + "υ", + "φ", + "χ", + "ψ", + "ω", + "∇", + "α", + "β", + "γ", + "δ", + "ε", + "ζ", + "η", + "θ", + "ι", + "κ", + "λ", + "μ", + "ν", + "ξ", + "ο", + "π", + "ρ", + "σ", + "τ", + "υ", + "φ", + "χ", + "ψ", + "ω", + "∂", + "ε", + "θ", + "κ", + "φ", + "ρ", + "π", + "α", + "β", + "γ", + "δ", + "ε", + "ζ", + "η", + "θ", + "ι", + "κ", + "λ", + "μ", + "ν", + "ξ", + "ο", + "π", + "ρ", + "θ", + "σ", + "τ", + "υ", + "φ", + "χ", + "ψ", + "ω", + "∇", + "α", + "β", + "γ", + "δ", + "ε", + "ζ", + "η", + "θ", + "ι", + "κ", + "λ", + "μ", + "ν", + "ξ", + "ο", + "π", + "ρ", + "σ", + "τ", + "υ", + "φ", + "χ", + "ψ", + "ω", + "∂", + "ε", + "θ", + "κ", + "φ", + "ρ", + "π", + "α", + "β", + "γ", + "δ", + "ε", + "ζ", + "η", + "θ", + "ι", + "κ", + "λ", + "μ", + "ν", + "ξ", + "ο", + "π", + "ρ", + "θ", + "σ", + "τ", + "υ", + "φ", + "χ", + "ψ", + "ω", + "∇", + "α", + "β", + "γ", + "δ", + "ε", + "ζ", + "η", + "θ", + "ι", + "κ", + "λ", + "μ", + "ν", + "ξ", + "ο", + "π", + "ρ", + "σ", + "τ", + "υ", + "φ", + "χ", + "ψ", + "ω", + "∂", + "ε", + "θ", + "κ", + "φ", + "ρ", + "π", + "α", + "β", + "γ", + "δ", + "ε", + "ζ", + "η", + "θ", + "ι", + "κ", + "λ", + "μ", + "ν", + "ξ", + "ο", + "π", + "ρ", + "θ", + "σ", + "τ", + "υ", + "φ", + "χ", + "ψ", + "ω", + "∇", + "α", + "β", + "γ", + "δ", + "ε", + "ζ", + "η", + "θ", + "ι", + "κ", + "λ", + "μ", + "ν", + "ξ", + "ο", + "π", + "ρ", + "σ", + "τ", + "υ", + "φ", + "χ", + "ψ", + "ω", + "∂", + "ε", + "θ", + "κ", + "φ", + "ρ", + "π", + "ϝ", + None, + "0", + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "0", + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "0", + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "0", + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "0", + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + "а", + "б", + "в", + "г", + "д", + "е", + "ж", + "з", + "и", + "к", + "л", + "м", + "о", + "п", + "р", + "с", + "т", + "у", + "ф", + "х", + "ц", + "ч", + "ш", + "ы", + "э", + "ю", + "ꚉ", + "ә", + "і", + "ј", + "ө", + "ү", + "ӏ", + "а", + "б", + "в", + "г", + "д", + "е", + "ж", + "з", + "и", + "к", + "л", + "о", + "п", + "с", + "у", + "ф", + "х", + "ц", + "ч", + "ш", + "ъ", + "ы", + "ґ", + "і", + "ѕ", + "џ", + "ҫ", + "ꙑ", + "ұ", + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + "𞤢", + "𞤣", + "𞤤", + "𞤥", + "𞤦", + "𞤧", + "𞤨", + "𞤩", + "𞤪", + "𞤫", + "𞤬", + "𞤭", + "𞤮", + "𞤯", + "𞤰", + "𞤱", + "𞤲", + "𞤳", + "𞤴", + "𞤵", + "𞤶", + "𞤷", + "𞤸", + "𞤹", + "𞤺", + "𞤻", + "𞤼", + "𞤽", + "𞤾", + "𞤿", + "𞥀", + "𞥁", + "𞥂", + "𞥃", + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + "ا", + "ب", + "ج", + "د", + None, + "و", + "ز", + "ح", + "ط", + "ي", + "ك", + "ل", + "م", + "ن", + "س", + "ع", + "ف", + "ص", + "ق", + "ر", + "ش", + "ت", + "ث", + "خ", + "ذ", + "ض", + "ظ", + "غ", + "ٮ", + "ں", + "ڡ", + "ٯ", + None, + "ب", + "ج", + None, + "ه", + None, + "ح", + None, + "ي", + "ك", + "ل", + "م", + "ن", + "س", + "ع", + "ف", + "ص", + "ق", + None, + "ش", + "ت", + "ث", + "خ", + None, + "ض", + None, + "غ", + None, + "ج", + None, + "ح", + None, + "ي", + None, + "ل", + None, + "ن", + "س", + "ع", + None, + "ص", + "ق", + None, + "ش", + None, + "خ", + None, + "ض", + None, + "غ", + None, + "ں", + None, + "ٯ", + None, + "ب", + "ج", + None, + "ه", + None, + "ح", + "ط", + "ي", + "ك", + None, + "م", + "ن", + "س", + "ع", + "ف", + "ص", + "ق", + None, + "ش", + "ت", + "ث", + "خ", + None, + "ض", + "ظ", + "غ", + "ٮ", + None, + "ڡ", + None, + "ا", + "ب", + "ج", + "د", + "ه", + "و", + "ز", + "ح", + "ط", + "ي", + None, + "ل", + "م", + "ن", + "س", + "ع", + "ف", + "ص", + "ق", + "ر", + "ش", + "ت", + "ث", + "خ", + "ذ", + "ض", + "ظ", + "غ", + None, + "ب", + "ج", + "د", + None, + "و", + "ز", + "ح", + "ط", + "ي", + None, + "ل", + "م", + "ن", + "س", + "ع", + "ف", + "ص", + "ق", + "ر", + "ش", + "ت", + "ث", + "خ", + "ذ", + "ض", + "ظ", + "غ", + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + "0,", + "1,", + "2,", + "3,", + "4,", + "5,", + "6,", + "7,", + "8,", + "9,", + None, + "(a)", + "(b)", + "(c)", + "(d)", + "(e)", + "(f)", + "(g)", + "(h)", + "(i)", + "(j)", + "(k)", + "(l)", + "(m)", + "(n)", + "(o)", + "(p)", + "(q)", + "(r)", + "(s)", + "(t)", + "(u)", + "(v)", + "(w)", + "(x)", + "(y)", + "(z)", + "〔s〕", + "c", + "r", + "cd", + "wz", + None, + "a", + "b", + "c", + "d", + "e", + "f", + "g", + "h", + "i", + "j", + "k", + "l", + "m", + "n", + "o", + "p", + "q", + "r", + "s", + "t", + "u", + "v", + "w", + "x", + "y", + "z", + "hv", + "mv", + "sd", + "ss", + "ppv", + "wc", + None, + "mc", + "md", + "mr", + None, + "dj", + None, + None, + None, + "ほか", + "ココ", + "サ", + None, + "手", + "字", + "双", + "デ", + "二", + "多", + "解", + "天", + "交", + "映", + "無", + "料", + "前", + "後", + "再", + "新", + "初", + "終", + "生", + "販", + "声", + "吹", + "演", + "投", + "捕", + "一", + "三", + "遊", + "左", + "中", + "右", + "指", + "走", + "打", + "禁", + "空", + "合", + "満", + "有", + "月", + "申", + "割", + "営", + "配", + None, + "〔本〕", + "〔三〕", + "〔二〕", + "〔安〕", + "〔点〕", + "〔打〕", + "〔盗〕", + "〔勝〕", + "〔敗〕", + None, + "得", + "可", + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + "0", + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + "丽", + "丸", + "乁", + "𠄢", + "你", + "侮", + "侻", + "倂", + "偺", + "備", + "僧", + "像", + "㒞", + "𠘺", + "免", + "兔", + "兤", + "具", + "𠔜", + "㒹", + "內", + "再", + "𠕋", + "冗", + "冤", + "仌", + "冬", + "况", + "𩇟", + "凵", + "刃", + "㓟", + "刻", + "剆", + "割", + "剷", + "㔕", + "勇", + "勉", + "勤", + "勺", + "包", + "匆", + "北", + "卉", + "卑", + "博", + "即", + "卽", + "卿", + "𠨬", + "灰", + "及", + "叟", + "𠭣", + "叫", + "叱", + "吆", + "咞", + "吸", + "呈", + "周", + "咢", + "哶", + "唐", + "啓", + "啣", + "善", + "喙", + "喫", + "喳", + "嗂", + "圖", + "嘆", + "圗", + "噑", + "噴", + "切", + "壮", + "城", + "埴", + "堍", + "型", + "堲", + "報", + "墬", + "𡓤", + "売", + "壷", + "夆", + "多", + "夢", + "奢", + "𡚨", + "𡛪", + "姬", + "娛", + "娧", + "姘", + "婦", + "㛮", + "㛼", + "嬈", + "嬾", + "𡧈", + "寃", + "寘", + "寧", + "寳", + "𡬘", + "寿", + "将", + "当", + "尢", + "㞁", + "屠", + "屮", + "峀", + "岍", + "𡷤", + "嵃", + "𡷦", + "嵮", + "嵫", + "嵼", + "巡", + "巢", + "㠯", + "巽", + "帨", + "帽", + "幩", + "㡢", + "𢆃", + "㡼", + "庰", + "庳", + "庶", + "廊", + "𪎒", + "廾", + "𢌱", + "舁", + "弢", + "㣇", + "𣊸", + "𦇚", + "形", + "彫", + "㣣", + "徚", + "忍", + "志", + "忹", + "悁", + "㤺", + "㤜", + "悔", + "𢛔", + "惇", + "慈", + "慌", + "慎", + "慌", + "慺", + "憎", + "憲", + "憤", + "憯", + "懞", + "懲", + "懶", + "成", + "戛", + "扝", + "抱", + "拔", + "捐", + "𢬌", + "挽", + "拼", + "捨", + "掃", + "揤", + "𢯱", + "搢", + "揅", + "掩", + "㨮", + "摩", + "摾", + "撝", + "摷", + "㩬", + "敏", + "敬", + "𣀊", + "旣", + "書", + "晉", + "㬙", + "暑", + "㬈", + "㫤", + "冒", + "冕", + "最", + "暜", + "肭", + "䏙", + "朗", + "望", + "朡", + "杞", + "杓", + "𣏃", + "㭉", + "柺", + "枅", + "桒", + "梅", + "𣑭", + "梎", + "栟", + "椔", + "㮝", + "楂", + "榣", + "槪", + "檨", + "𣚣", + "櫛", + "㰘", + "次", + "𣢧", + "歔", + "㱎", + "歲", + "殟", + "殺", + "殻", + "𣪍", + "𡴋", + "𣫺", + "汎", + "𣲼", + "沿", + "泍", + "汧", + "洖", + "派", + "海", + "流", + "浩", + "浸", + "涅", + "𣴞", + "洴", + "港", + "湮", + "㴳", + "滋", + "滇", + "𣻑", + "淹", + "潮", + "𣽞", + "𣾎", + "濆", + "瀹", + "瀞", + "瀛", + "㶖", + "灊", + "災", + "灷", + "炭", + "𠔥", + "煅", + "𤉣", + "熜", + "𤎫", + "爨", + "爵", + "牐", + "𤘈", + "犀", + "犕", + "𤜵", + "𤠔", + "獺", + "王", + "㺬", + "玥", + "㺸", + "瑇", + "瑜", + "瑱", + "璅", + "瓊", + "㼛", + "甤", + "𤰶", + "甾", + "𤲒", + "異", + "𢆟", + "瘐", + "𤾡", + "𤾸", + "𥁄", + "㿼", + "䀈", + "直", + "𥃳", + "𥃲", + "𥄙", + "𥄳", + "眞", + "真", + "睊", + "䀹", + "瞋", + "䁆", + "䂖", + "𥐝", + "硎", + "碌", + "磌", + "䃣", + "𥘦", + "祖", + "𥚚", + "𥛅", + "福", + "秫", + "䄯", + "穀", + "穊", + "穏", + "𥥼", + "𥪧", + "竮", + "䈂", + "𥮫", + "篆", + "築", + "䈧", + "𥲀", + "糒", + "䊠", + "糨", + "糣", + "紀", + "𥾆", + "絣", + "䌁", + "緇", + "縂", + "繅", + "䌴", + "𦈨", + "𦉇", + "䍙", + "𦋙", + "罺", + "𦌾", + "羕", + "翺", + "者", + "𦓚", + "𦔣", + "聠", + "𦖨", + "聰", + "𣍟", + "䏕", + "育", + "脃", + "䐋", + "脾", + "媵", + "𦞧", + "𦞵", + "𣎓", + "𣎜", + "舁", + "舄", + "辞", + "䑫", + "芑", + "芋", + "芝", + "劳", + "花", + "芳", + "芽", + "苦", + "𦬼", + "若", + "茝", + "荣", + "莭", + "茣", + "莽", + "菧", + "著", + "荓", + "菊", + "菌", + "菜", + "𦰶", + "𦵫", + "𦳕", + "䔫", + "蓱", + "蓳", + "蔖", + "𧏊", + "蕤", + "𦼬", + "䕝", + "䕡", + "𦾱", + "𧃒", + "䕫", + "虐", + "虜", + "虧", + "虩", + "蚩", + "蚈", + "蜎", + "蛢", + "蝹", + "蜨", + "蝫", + "螆", + "䗗", + "蟡", + "蠁", + "䗹", + "衠", + "衣", + "𧙧", + "裗", + "裞", + "䘵", + "裺", + "㒻", + "𧢮", + "𧥦", + "䚾", + "䛇", + "誠", + "諭", + "變", + "豕", + "𧲨", + "貫", + "賁", + "贛", + "起", + "𧼯", + "𠠄", + "跋", + "趼", + "跰", + "𠣞", + "軔", + "輸", + "𨗒", + "𨗭", + "邔", + "郱", + "鄑", + "𨜮", + "鄛", + "鈸", + "鋗", + "鋘", + "鉼", + "鏹", + "鐕", + "𨯺", + "開", + "䦕", + "閷", + "𨵷", + "䧦", + "雃", + "嶲", + "霣", + "𩅅", + "𩈚", + "䩮", + "䩶", + "韠", + "𩐊", + "䪲", + "𩒖", + "頋", + "頩", + "𩖶", + "飢", + "䬳", + "餩", + "馧", + "駂", + "駾", + "䯎", + "𩬰", + "鬒", + "鱀", + "鳽", + "䳎", + "䳭", + "鵧", + "𪃎", + "䳸", + "𪄅", + "𪈎", + "𪊑", + "麻", + "䵖", + "黹", + "黾", + "鼅", + "鼏", + "鼖", + "鼻", + "𪘀", + None, + None, + None, + None, + None, + None, + None, +) diff --git a/py_modules/multidict/__init__.py b/py_modules/multidict/__init__.py new file mode 100644 index 0000000..e3db9e1 --- /dev/null +++ b/py_modules/multidict/__init__.py @@ -0,0 +1,60 @@ +""" +Multidict implementation. + +HTTP Headers and URL query string require specific data structure: +multidict. It behaves mostly like a dict but it can have +several values for the same key. +""" + +from typing import TYPE_CHECKING + +from ._abc import MultiMapping, MutableMultiMapping +from ._compat import USE_EXTENSIONS + +__all__ = ( + "CIMultiDict", + "CIMultiDictProxy", + "MultiDict", + "MultiDictProxy", + "MultiMapping", + "MutableMultiMapping", + "getversion", + "istr", + "upstr", +) + +__version__ = "6.7.1" + + +if TYPE_CHECKING or not USE_EXTENSIONS: + from ._multidict_py import ( + CIMultiDict, + CIMultiDictProxy, + MultiDict, + MultiDictProxy, + getversion, + istr, + ) +else: + from collections.abc import ItemsView, KeysView, ValuesView + + from ._multidict import ( + CIMultiDict, + CIMultiDictProxy, + MultiDict, + MultiDictProxy, + _ItemsView, + _KeysView, + _ValuesView, + getversion, + istr, + ) + + MultiMapping.register(MultiDictProxy) + MutableMultiMapping.register(MultiDict) + KeysView.register(_KeysView) + ItemsView.register(_ItemsView) + ValuesView.register(_ValuesView) + + +upstr = istr diff --git a/py_modules/multidict/_abc.py b/py_modules/multidict/_abc.py new file mode 100644 index 0000000..54253e9 --- /dev/null +++ b/py_modules/multidict/_abc.py @@ -0,0 +1,73 @@ +import abc +from collections.abc import Iterable, Mapping, MutableMapping +from typing import TYPE_CHECKING, Protocol, TypeVar, Union, overload + +if TYPE_CHECKING: + from ._multidict_py import istr +else: + istr = str + +_V = TypeVar("_V") +_V_co = TypeVar("_V_co", covariant=True) +_T = TypeVar("_T") + + +class SupportsKeys(Protocol[_V_co]): + def keys(self) -> Iterable[str]: ... + def __getitem__(self, key: str, /) -> _V_co: ... + + +class SupportsIKeys(Protocol[_V_co]): + def keys(self) -> Iterable[istr]: ... + def __getitem__(self, key: istr, /) -> _V_co: ... + + +MDArg = Union[SupportsKeys[_V], SupportsIKeys[_V], Iterable[tuple[str, _V]], None] + + +class MultiMapping(Mapping[str, _V_co]): + @overload + def getall(self, key: str) -> list[_V_co]: ... + @overload + def getall(self, key: str, default: _T) -> Union[list[_V_co], _T]: ... + @abc.abstractmethod + def getall(self, key: str, default: _T = ...) -> Union[list[_V_co], _T]: + """Return all values for key.""" + + @overload + def getone(self, key: str) -> _V_co: ... + @overload + def getone(self, key: str, default: _T) -> Union[_V_co, _T]: ... + @abc.abstractmethod + def getone(self, key: str, default: _T = ...) -> Union[_V_co, _T]: + """Return first value for key.""" + + +class MutableMultiMapping(MultiMapping[_V], MutableMapping[str, _V]): + @abc.abstractmethod + def add(self, key: str, value: _V) -> None: + """Add value to list.""" + + @abc.abstractmethod + def extend(self, arg: MDArg[_V] = None, /, **kwargs: _V) -> None: + """Add everything from arg and kwargs to the mapping.""" + + @abc.abstractmethod + def merge(self, arg: MDArg[_V] = None, /, **kwargs: _V) -> None: + """Merge into the mapping, adding non-existing keys.""" + + @overload + def popone(self, key: str) -> _V: ... + @overload + def popone(self, key: str, default: _T) -> Union[_V, _T]: ... + @abc.abstractmethod + def popone(self, key: str, default: _T = ...) -> Union[_V, _T]: + """Remove specified key and return the corresponding value.""" + + @overload + def popall(self, key: str) -> list[_V]: ... + @overload + def popall(self, key: str, default: _T) -> Union[list[_V], _T]: ... + @abc.abstractmethod + def popall(self, key: str, default: _T = ...) -> Union[list[_V], _T]: + """Remove all occurrences of key and return the list of corresponding values.""" diff --git a/py_modules/multidict/_compat.py b/py_modules/multidict/_compat.py new file mode 100644 index 0000000..264d327 --- /dev/null +++ b/py_modules/multidict/_compat.py @@ -0,0 +1,15 @@ +import os +import platform + +NO_EXTENSIONS = bool(os.environ.get("MULTIDICT_NO_EXTENSIONS")) + +PYPY = platform.python_implementation() == "PyPy" + +USE_EXTENSIONS = not NO_EXTENSIONS and not PYPY + +if USE_EXTENSIONS: + try: + from . import _multidict # type: ignore[attr-defined] # noqa: F401 + except ImportError: # pragma: no cover + # FIXME: Refactor for coverage. See #837. + USE_EXTENSIONS = False diff --git a/py_modules/multidict/_multidict.cpython-311-x86_64-linux-gnu.so b/py_modules/multidict/_multidict.cpython-311-x86_64-linux-gnu.so new file mode 100755 index 0000000..95df7e1 Binary files /dev/null and b/py_modules/multidict/_multidict.cpython-311-x86_64-linux-gnu.so differ diff --git a/py_modules/multidict/_multidict_py.py b/py_modules/multidict/_multidict_py.py new file mode 100644 index 0000000..6b68d52 --- /dev/null +++ b/py_modules/multidict/_multidict_py.py @@ -0,0 +1,1242 @@ +import enum +import functools +import reprlib +import sys +from array import array +from collections.abc import ( + ItemsView, + Iterable, + Iterator, + KeysView, + Mapping, + ValuesView, +) +from dataclasses import dataclass +from typing import ( + TYPE_CHECKING, + Any, + ClassVar, + Generic, + NoReturn, + Optional, + TypeVar, + Union, + cast, + overload, +) + +from ._abc import MDArg, MultiMapping, MutableMultiMapping, SupportsKeys + +if sys.version_info >= (3, 11): + from typing import Self +else: + from typing_extensions import Self + + +class istr(str): + """Case insensitive str.""" + + __is_istr__ = True + __istr_identity__: Optional[str] = None + + +_V = TypeVar("_V") +_T = TypeVar("_T") + +_SENTINEL = enum.Enum("_SENTINEL", "sentinel") +sentinel = _SENTINEL.sentinel + +_version = array("Q", [0]) + + +class _Iter(Generic[_T]): + __slots__ = ("_size", "_iter") + + def __init__(self, size: int, iterator: Iterator[_T]): + self._size = size + self._iter = iterator + + def __iter__(self) -> Self: + return self + + def __next__(self) -> _T: + return next(self._iter) + + def __length_hint__(self) -> int: + return self._size + + +class _ViewBase(Generic[_V]): + def __init__( + self, + md: "MultiDict[_V]", + ): + self._md = md + + def __len__(self) -> int: + return len(self._md) + + +class _ItemsView(_ViewBase[_V], ItemsView[str, _V]): + def __contains__(self, item: object) -> bool: + if not isinstance(item, (tuple, list)) or len(item) != 2: + return False + key, value = item + try: + identity = self._md._identity(key) + except TypeError: + return False + hash_ = hash(identity) + for slot, idx, e in self._md._keys.iter_hash(hash_): + if e.identity == identity and value == e.value: + return True + return False + + def __iter__(self) -> _Iter[tuple[str, _V]]: + return _Iter(len(self), self._iter(self._md._version)) + + def _iter(self, version: int) -> Iterator[tuple[str, _V]]: + for e in self._md._keys.iter_entries(): + if version != self._md._version: + raise RuntimeError("Dictionary changed during iteration") + yield self._md._key(e.key), e.value + + @reprlib.recursive_repr() + def __repr__(self) -> str: + lst = [] + for e in self._md._keys.iter_entries(): + lst.append(f"'{e.key}': {e.value!r}") + body = ", ".join(lst) + return f"<{self.__class__.__name__}({body})>" + + def _parse_item( + self, arg: Union[tuple[str, _V], _T] + ) -> Optional[tuple[int, str, str, _V]]: + if not isinstance(arg, tuple): + return None + if len(arg) != 2: + return None + try: + identity = self._md._identity(arg[0]) + return (hash(identity), identity, arg[0], arg[1]) + except TypeError: + return None + + def _tmp_set(self, it: Iterable[_T]) -> set[tuple[str, _V]]: + tmp = set() + for arg in it: + item = self._parse_item(arg) + if item is None: + continue + else: + tmp.add((item[1], item[3])) + return tmp + + def __and__(self, other: Iterable[Any]) -> set[tuple[str, _V]]: + ret = set() + try: + it = iter(other) + except TypeError: + return NotImplemented + for arg in it: + item = self._parse_item(arg) + if item is None: + continue + hash_, identity, key, value = item + for slot, idx, e in self._md._keys.iter_hash(hash_): + e.hash = -1 + if e.identity == identity and e.value == value: + ret.add((e.key, e.value)) + self._md._keys.restore_hash(hash_) + return ret + + def __rand__(self, other: Iterable[_T]) -> set[_T]: + ret = set() + try: + it = iter(other) + except TypeError: + return NotImplemented + for arg in it: + item = self._parse_item(arg) + if item is None: + continue + hash_, identity, key, value = item + for slot, idx, e in self._md._keys.iter_hash(hash_): + if e.identity == identity and e.value == value: + ret.add(arg) + break + return ret + + def __or__(self, other: Iterable[_T]) -> set[Union[tuple[str, _V], _T]]: + ret: set[Union[tuple[str, _V], _T]] = set(self) + try: + it = iter(other) + except TypeError: + return NotImplemented + for arg in it: + item: Optional[tuple[int, str, str, _V]] = self._parse_item(arg) + if item is None: + ret.add(arg) + continue + hash_, identity, key, value = item + for slot, idx, e in self._md._keys.iter_hash(hash_): + if e.identity == identity and e.value == value: # pragma: no branch + break + else: + ret.add(arg) + return ret + + def __ror__(self, other: Iterable[_T]) -> set[Union[tuple[str, _V], _T]]: + try: + ret: set[Union[tuple[str, _V], _T]] = set(other) + except TypeError: + return NotImplemented + tmp = self._tmp_set(ret) + + for e in self._md._keys.iter_entries(): + if (e.identity, e.value) not in tmp: + ret.add((e.key, e.value)) + return ret + + def __sub__(self, other: Iterable[_T]) -> set[Union[tuple[str, _V], _T]]: + ret: set[Union[tuple[str, _V], _T]] = set() + try: + it = iter(other) + except TypeError: + return NotImplemented + tmp = self._tmp_set(it) + + for e in self._md._keys.iter_entries(): + if (e.identity, e.value) not in tmp: + ret.add((e.key, e.value)) + + return ret + + def __rsub__(self, other: Iterable[_T]) -> set[_T]: + ret: set[_T] = set() + try: + it = iter(other) + except TypeError: + return NotImplemented + for arg in it: + item = self._parse_item(arg) + if item is None: + ret.add(arg) + continue + + hash_, identity, key, value = item + for slot, idx, e in self._md._keys.iter_hash(hash_): + if e.identity == identity and e.value == value: # pragma: no branch + break + else: + ret.add(arg) + return ret + + def __xor__(self, other: Iterable[_T]) -> set[Union[tuple[str, _V], _T]]: + try: + rgt = set(other) + except TypeError: + return NotImplemented + ret: set[Union[tuple[str, _V], _T]] = self - rgt + ret |= rgt - self + return ret + + __rxor__ = __xor__ + + def isdisjoint(self, other: Iterable[tuple[str, _V]]) -> bool: + for arg in other: + item = self._parse_item(arg) + if item is None: + continue + + hash_, identity, key, value = item + for slot, idx, e in self._md._keys.iter_hash(hash_): + if e.identity == identity and e.value == value: # pragma: no branch + return False + return True + + +class _ValuesView(_ViewBase[_V], ValuesView[_V]): + def __contains__(self, value: object) -> bool: + for e in self._md._keys.iter_entries(): + if e.value == value: + return True + return False + + def __iter__(self) -> _Iter[_V]: + return _Iter(len(self), self._iter(self._md._version)) + + def _iter(self, version: int) -> Iterator[_V]: + for e in self._md._keys.iter_entries(): + if version != self._md._version: + raise RuntimeError("Dictionary changed during iteration") + yield e.value + + @reprlib.recursive_repr() + def __repr__(self) -> str: + lst = [] + for e in self._md._keys.iter_entries(): + lst.append(repr(e.value)) + body = ", ".join(lst) + return f"<{self.__class__.__name__}({body})>" + + +class _KeysView(_ViewBase[_V], KeysView[str]): + def __contains__(self, key: object) -> bool: + if not isinstance(key, str): + return False + identity = self._md._identity(key) + hash_ = hash(identity) + for slot, idx, e in self._md._keys.iter_hash(hash_): + if e.identity == identity: # pragma: no branch + return True + return False + + def __iter__(self) -> _Iter[str]: + return _Iter(len(self), self._iter(self._md._version)) + + def _iter(self, version: int) -> Iterator[str]: + for e in self._md._keys.iter_entries(): + if version != self._md._version: + raise RuntimeError("Dictionary changed during iteration") + yield self._md._key(e.key) + + def __repr__(self) -> str: + lst = [] + for e in self._md._keys.iter_entries(): + lst.append(f"'{e.key}'") + body = ", ".join(lst) + return f"<{self.__class__.__name__}({body})>" + + def __and__(self, other: Iterable[object]) -> set[str]: + ret = set() + try: + it = iter(other) + except TypeError: + return NotImplemented + for key in it: + if not isinstance(key, str): + continue + identity = self._md._identity(key) + hash_ = hash(identity) + for slot, idx, e in self._md._keys.iter_hash(hash_): + if e.identity == identity: # pragma: no branch + ret.add(e.key) + break + return ret + + def __rand__(self, other: Iterable[_T]) -> set[_T]: + ret = set() + try: + it = iter(other) + except TypeError: + return NotImplemented + for key in it: + if not isinstance(key, str): + continue + if key in self._md: + ret.add(key) + return cast(set[_T], ret) + + def __or__(self, other: Iterable[_T]) -> set[Union[str, _T]]: + ret: set[Union[str, _T]] = set(self) + try: + it = iter(other) + except TypeError: + return NotImplemented + for key in it: + if not isinstance(key, str): + ret.add(key) + continue + if key not in self._md: + ret.add(key) + return ret + + def __ror__(self, other: Iterable[_T]) -> set[Union[str, _T]]: + try: + ret: set[Union[str, _T]] = set(other) + except TypeError: + return NotImplemented + + tmp = set() + for key in ret: + if not isinstance(key, str): + continue + identity = self._md._identity(key) + tmp.add(identity) + + for e in self._md._keys.iter_entries(): + if e.identity not in tmp: + ret.add(e.key) + return ret + + def __sub__(self, other: Iterable[object]) -> set[str]: + ret = set(self) + try: + it = iter(other) + except TypeError: + return NotImplemented + for key in it: + if not isinstance(key, str): + continue + identity = self._md._identity(key) + hash_ = hash(identity) + for slot, idx, e in self._md._keys.iter_hash(hash_): + if e.identity == identity: # pragma: no branch + ret.discard(e.key) + break + return ret + + def __rsub__(self, other: Iterable[_T]) -> set[_T]: + try: + ret: set[_T] = set(other) + except TypeError: + return NotImplemented + for key in other: + if not isinstance(key, str): + continue + if key in self._md: + ret.discard(key) # type: ignore[arg-type] + return ret + + def __xor__(self, other: Iterable[_T]) -> set[Union[str, _T]]: + try: + rgt = set(other) + except TypeError: + return NotImplemented + ret: set[Union[str, _T]] = self - rgt # type: ignore[assignment] + ret |= rgt - self + return ret + + __rxor__ = __xor__ + + def isdisjoint(self, other: Iterable[object]) -> bool: + for key in other: + if not isinstance(key, str): + continue + if key in self._md: + return False + return True + + +class _CSMixin: + _ci: ClassVar[bool] = False + + def _key(self, key: str) -> str: + return key + + def _identity(self, key: str) -> str: + if isinstance(key, str): + return key + else: + raise TypeError("MultiDict keys should be either str or subclasses of str") + + +class _CIMixin: + _ci: ClassVar[bool] = True + + def _key(self, key: str) -> str: + if type(key) is istr: + return key + else: + return istr(key) + + def _identity(self, key: str) -> str: + if isinstance(key, istr): + ret = key.__istr_identity__ + if ret is None: + ret = key.lower() + key.__istr_identity__ = ret + return ret + if isinstance(key, str): + return key.lower() + else: + raise TypeError("MultiDict keys should be either str or subclasses of str") + + +def estimate_log2_keysize(n: int) -> int: + # 7 == HT_MINSIZE - 1 + return (((n * 3 + 1) // 2) | 7).bit_length() + + +@dataclass +class _Entry(Generic[_V]): + hash: int + identity: str + key: str + value: _V + + +@dataclass +class _HtKeys(Generic[_V]): # type: ignore[misc] + LOG_MINSIZE: ClassVar[int] = 3 + MINSIZE: ClassVar[int] = 8 + PREALLOCATED_INDICES: ClassVar[dict[int, array]] = { # type: ignore[type-arg] + log2_size: array( + "b" if log2_size < 8 else "h", (-1 for i in range(1 << log2_size)) + ) + for log2_size in range(3, 10) + } + + log2_size: int + usable: int + + indices: array # type: ignore[type-arg] # in py3.9 array is not generic + entries: list[Optional[_Entry[_V]]] + + @functools.cached_property + def nslots(self) -> int: + return 1 << self.log2_size + + @functools.cached_property + def mask(self) -> int: + return self.nslots - 1 + + if sys.implementation.name != "pypy": + + def __sizeof__(self) -> int: + return ( + object.__sizeof__(self) + + sys.getsizeof(self.indices) + + sys.getsizeof(self.entries) + ) + + @classmethod + def new(cls, log2_size: int, entries: list[Optional[_Entry[_V]]]) -> Self: + size = 1 << log2_size + usable = (size << 1) // 3 + if log2_size < 10: + indices = cls.PREALLOCATED_INDICES[log2_size].__copy__() + elif log2_size < 16: + indices = array("h", (-1 for i in range(size))) + elif log2_size < 32: + indices = array("l", (-1 for i in range(size))) + else: # pragma: no cover # don't test huge multidicts + indices = array("q", (-1 for i in range(size))) + ret = cls( + log2_size=log2_size, + usable=usable, + indices=indices, + entries=entries, + ) + return ret + + def clone(self) -> "_HtKeys[_V]": + entries = [ + _Entry(e.hash, e.identity, e.key, e.value) if e is not None else None + for e in self.entries + ] + + return _HtKeys( + log2_size=self.log2_size, + usable=self.usable, + indices=self.indices.__copy__(), + entries=entries, + ) + + def build_indices(self, update: bool) -> None: + mask = self.mask + indices = self.indices + for idx, e in enumerate(self.entries): + assert e is not None + hash_ = e.hash + if update: + if hash_ == -1: + hash_ = hash(e.identity) + else: + assert hash_ != -1 + i = hash_ & mask + perturb = hash_ & sys.maxsize + while indices[i] != -1: + perturb >>= 5 + i = mask & (i * 5 + perturb + 1) + indices[i] = idx + + def find_empty_slot(self, hash_: int) -> int: + mask = self.mask + indices = self.indices + i = hash_ & mask + perturb = hash_ & sys.maxsize + ix = indices[i] + while ix != -1: + perturb >>= 5 + i = (i * 5 + perturb + 1) & mask + ix = indices[i] + return i + + def iter_hash(self, hash_: int) -> Iterator[tuple[int, int, _Entry[_V]]]: + mask = self.mask + indices = self.indices + entries = self.entries + i = hash_ & mask + perturb = hash_ & sys.maxsize + ix = indices[i] + while ix != -1: + if ix != -2: + e = entries[ix] + if e.hash == hash_: + yield i, ix, e + perturb >>= 5 + i = (i * 5 + perturb + 1) & mask + ix = indices[i] + + def del_idx(self, hash_: int, idx: int) -> None: + mask = self.mask + indices = self.indices + i = hash_ & mask + perturb = hash_ & sys.maxsize + ix = indices[i] + while ix != idx: + perturb >>= 5 + i = (i * 5 + perturb + 1) & mask + ix = indices[i] + indices[i] = -2 + + def iter_entries(self) -> Iterator[_Entry[_V]]: + return filter(None, self.entries) + + def restore_hash(self, hash_: int) -> None: + mask = self.mask + indices = self.indices + entries = self.entries + i = hash_ & mask + perturb = hash_ & sys.maxsize + ix = indices[i] + while ix != -1: + if ix != -2: + entry = entries[ix] + if entry.hash == -1: + entry.hash = hash_ + perturb >>= 5 + i = (i * 5 + perturb + 1) & mask + ix = indices[i] + + +class MultiDict(_CSMixin, MutableMultiMapping[_V]): + """Dictionary with the support for duplicate keys.""" + + __slots__ = ("_keys", "_used", "_version") + + def __init__(self, arg: MDArg[_V] = None, /, **kwargs: _V): + self._used = 0 + v = _version + v[0] += 1 + self._version = v[0] + if not kwargs: + md = None + if isinstance(arg, MultiDictProxy): + md = arg._md + elif isinstance(arg, MultiDict): + md = arg + if md is not None and md._ci is self._ci: + self._from_md(md) + return + + it = self._parse_args(arg, kwargs) + log2_size = estimate_log2_keysize(cast(int, next(it))) + if log2_size > 17: # pragma: no cover + # Don't overallocate really huge keys space in init + log2_size = 17 + self._keys: _HtKeys[_V] = _HtKeys.new(log2_size, []) + self._extend_items(cast(Iterator[_Entry[_V]], it)) + + def _from_md(self, md: "MultiDict[_V]") -> None: + # Copy everything as-is without compacting the new multidict, + # otherwise it requires reindexing + self._keys = md._keys.clone() + self._used = md._used + + @overload + def getall(self, key: str) -> list[_V]: ... + @overload + def getall(self, key: str, default: _T) -> Union[list[_V], _T]: ... + def getall( + self, key: str, default: Union[_T, _SENTINEL] = sentinel + ) -> Union[list[_V], _T]: + """Return a list of all values matching the key.""" + identity = self._identity(key) + hash_ = hash(identity) + res = [] + restore = [] + for slot, idx, e in self._keys.iter_hash(hash_): + if e.identity == identity: # pragma: no branch + res.append(e.value) + e.hash = -1 + restore.append(idx) + + if res: + entries = self._keys.entries + for idx in restore: + entries[idx].hash = hash_ # type: ignore[union-attr] + return res + if not res and default is not sentinel: + return default + raise KeyError("Key not found: %r" % key) + + @overload + def getone(self, key: str) -> _V: ... + @overload + def getone(self, key: str, default: _T) -> Union[_V, _T]: ... + def getone( + self, key: str, default: Union[_T, _SENTINEL] = sentinel + ) -> Union[_V, _T]: + """Get first value matching the key. + + Raises KeyError if the key is not found and no default is provided. + """ + identity = self._identity(key) + hash_ = hash(identity) + for slot, idx, e in self._keys.iter_hash(hash_): + if e.identity == identity: # pragma: no branch + return e.value + if default is not sentinel: + return default + raise KeyError("Key not found: %r" % key) + + # Mapping interface # + + def __getitem__(self, key: str) -> _V: + return self.getone(key) + + @overload + def get(self, key: str, /) -> Union[_V, None]: ... + @overload + def get(self, key: str, /, default: _T) -> Union[_V, _T]: ... + def get(self, key: str, default: Union[_T, None] = None) -> Union[_V, _T, None]: + """Get first value matching the key. + + If the key is not found, returns the default (or None if no default is provided) + """ + return self.getone(key, default) + + def __iter__(self) -> Iterator[str]: + return iter(self.keys()) + + def __len__(self) -> int: + return self._used + + def keys(self) -> KeysView[str]: + """Return a new view of the dictionary's keys.""" + return _KeysView(self) + + def items(self) -> ItemsView[str, _V]: + """Return a new view of the dictionary's items *(key, value) pairs).""" + return _ItemsView(self) + + def values(self) -> _ValuesView[_V]: + """Return a new view of the dictionary's values.""" + return _ValuesView(self) + + def __eq__(self, other: object) -> bool: + if not isinstance(other, Mapping): + return NotImplemented + if isinstance(other, MultiDictProxy): + return self == other._md + if isinstance(other, MultiDict): + lft = self._keys + rht = other._keys + if self._used != other._used: + return False + for e1, e2 in zip(lft.iter_entries(), rht.iter_entries()): + if e1.identity != e2.identity or e1.value != e2.value: + return False + return True + if self._used != len(other): + return False + for k, v in self.items(): + nv = other.get(k, sentinel) + if v != nv: + return False + return True + + def __contains__(self, key: object) -> bool: + if not isinstance(key, str): + return False + identity = self._identity(key) + hash_ = hash(identity) + for slot, idx, e in self._keys.iter_hash(hash_): + if e.identity == identity: # pragma: no branch + return True + return False + + @reprlib.recursive_repr() + def __repr__(self) -> str: + body = ", ".join(f"'{e.key}': {e.value!r}" for e in self._keys.iter_entries()) + return f"<{self.__class__.__name__}({body})>" + + if sys.implementation.name != "pypy": + + def __sizeof__(self) -> int: + return object.__sizeof__(self) + sys.getsizeof(self._keys) + + def __reduce__(self) -> tuple[type[Self], tuple[list[tuple[str, _V]]]]: + return (self.__class__, (list(self.items()),)) + + def add(self, key: str, value: _V) -> None: + identity = self._identity(key) + hash_ = hash(identity) + self._add_with_hash(_Entry(hash_, identity, key, value)) + self._incr_version() + + def copy(self) -> Self: + """Return a copy of itself.""" + cls = self.__class__ + return cls(self) + + __copy__ = copy + + def extend(self, arg: MDArg[_V] = None, /, **kwargs: _V) -> None: + """Extend current MultiDict with more values. + + This method must be used instead of update. + """ + it = self._parse_args(arg, kwargs) + newsize = self._used + cast(int, next(it)) + self._resize(estimate_log2_keysize(newsize), False) + self._extend_items(cast(Iterator[_Entry[_V]], it)) + + def _parse_args( + self, + arg: MDArg[_V], + kwargs: Mapping[str, _V], + ) -> Iterator[Union[int, _Entry[_V]]]: + identity_func = self._identity + if arg: + if isinstance(arg, MultiDictProxy): + arg = arg._md + if isinstance(arg, MultiDict): + yield len(arg) + len(kwargs) + if self._ci is not arg._ci: + for e in arg._keys.iter_entries(): + identity = identity_func(e.key) + yield _Entry(hash(identity), identity, e.key, e.value) + else: + for e in arg._keys.iter_entries(): + yield _Entry(e.hash, e.identity, e.key, e.value) + if kwargs: + for key, value in kwargs.items(): + identity = identity_func(key) + yield _Entry(hash(identity), identity, key, value) + else: + if hasattr(arg, "keys"): + arg = cast(SupportsKeys[_V], arg) + arg = [(k, arg[k]) for k in arg.keys()] + if kwargs: + arg = list(arg) + arg.extend(list(kwargs.items())) + try: + yield len(arg) + len(kwargs) # type: ignore[arg-type] + except TypeError: + yield 0 + for pos, item in enumerate(arg): + if not len(item) == 2: + raise ValueError( + f"multidict update sequence element #{pos}" + f"has length {len(item)}; 2 is required" + ) + identity = identity_func(item[0]) + yield _Entry(hash(identity), identity, item[0], item[1]) + else: + yield len(kwargs) + for key, value in kwargs.items(): + identity = identity_func(key) + yield _Entry(hash(identity), identity, key, value) + + def _extend_items(self, items: Iterable[_Entry[_V]]) -> None: + for e in items: + self._add_with_hash(e) + self._incr_version() + + def clear(self) -> None: + """Remove all items from MultiDict.""" + self._used = 0 + self._keys = _HtKeys.new(_HtKeys.LOG_MINSIZE, []) + self._incr_version() + + # Mapping interface # + + def __setitem__(self, key: str, value: _V) -> None: + identity = self._identity(key) + hash_ = hash(identity) + found = False + + for slot, idx, e in self._keys.iter_hash(hash_): + if e.identity == identity: # pragma: no branch + if not found: + e.key = key + e.value = value + e.hash = -1 + found = True + self._incr_version() + elif e.hash != -1: # pragma: no branch + self._del_at(slot, idx) + + if not found: + self._add_with_hash(_Entry(hash_, identity, key, value)) + else: + self._keys.restore_hash(hash_) + + def __delitem__(self, key: str) -> None: + found = False + identity = self._identity(key) + hash_ = hash(identity) + for slot, idx, e in self._keys.iter_hash(hash_): + if e.identity == identity: # pragma: no branch + self._del_at(slot, idx) + found = True + if not found: + raise KeyError(key) + else: + self._incr_version() + + @overload + def setdefault( + self: "MultiDict[Union[_T, None]]", key: str, default: None = None + ) -> Union[_T, None]: ... + @overload + def setdefault(self, key: str, default: _V) -> _V: ... + def setdefault(self, key: str, default: Union[_V, None] = None) -> Union[_V, None]: # type: ignore[misc] + """Return value for key, set value to default if key is not present.""" + identity = self._identity(key) + hash_ = hash(identity) + for slot, idx, e in self._keys.iter_hash(hash_): + if e.identity == identity: # pragma: no branch + return e.value + self.add(key, default) # type: ignore[arg-type] + return default + + @overload + def popone(self, key: str) -> _V: ... + @overload + def popone(self, key: str, default: _T) -> Union[_V, _T]: ... + def popone( + self, key: str, default: Union[_T, _SENTINEL] = sentinel + ) -> Union[_V, _T]: + """Remove specified key and return the corresponding value. + + If key is not found, d is returned if given, otherwise + KeyError is raised. + + """ + identity = self._identity(key) + hash_ = hash(identity) + for slot, idx, e in self._keys.iter_hash(hash_): + if e.identity == identity: # pragma: no branch + value = e.value + self._del_at(slot, idx) + self._incr_version() + return value + if default is sentinel: + raise KeyError(key) + else: + return default + + # Type checking will inherit signature for pop() if we don't confuse it here. + if not TYPE_CHECKING: + pop = popone + + @overload + def popall(self, key: str) -> list[_V]: ... + @overload + def popall(self, key: str, default: _T) -> Union[list[_V], _T]: ... + def popall( + self, key: str, default: Union[_T, _SENTINEL] = sentinel + ) -> Union[list[_V], _T]: + """Remove all occurrences of key and return the list of corresponding + values. + + If key is not found, default is returned if given, otherwise + KeyError is raised. + + """ + found = False + identity = self._identity(key) + hash_ = hash(identity) + ret = [] + for slot, idx, e in self._keys.iter_hash(hash_): + if e.identity == identity: # pragma: no branch + found = True + ret.append(e.value) + self._del_at(slot, idx) + self._incr_version() + + if not found: + if default is sentinel: + raise KeyError(key) + else: + return default + else: + return ret + + def popitem(self) -> tuple[str, _V]: + """Remove and return an arbitrary (key, value) pair.""" + if self._used <= 0: + raise KeyError("empty multidict") + + pos = len(self._keys.entries) - 1 + entry = self._keys.entries.pop() + + while entry is None: + pos -= 1 + entry = self._keys.entries.pop() + + ret = self._key(entry.key), entry.value + self._keys.del_idx(entry.hash, pos) + self._used -= 1 + self._incr_version() + return ret + + def update(self, arg: MDArg[_V] = None, /, **kwargs: _V) -> None: + """Update the dictionary, overwriting existing keys.""" + it = self._parse_args(arg, kwargs) + newsize = self._used + cast(int, next(it)) + log2_size = estimate_log2_keysize(newsize) + if log2_size > 17: # pragma: no cover + # Don't overallocate really huge keys space in update, + # duplicate keys could reduce the resulting anount of entries + log2_size = 17 + if log2_size > self._keys.log2_size: + self._resize(log2_size, False) + try: + self._update_items(cast(Iterator[_Entry[_V]], it)) + finally: + self._post_update() + + def _update_items(self, items: Iterator[_Entry[_V]]) -> None: + for entry in items: + found = False + hash_ = entry.hash + identity = entry.identity + for slot, idx, e in self._keys.iter_hash(hash_): + if e.identity == identity: # pragma: no branch + if not found: + found = True + e.key = entry.key + e.value = entry.value + e.hash = -1 + else: + self._del_at_for_upd(e) + if not found: + self._add_with_hash_for_upd(entry) + + def _post_update(self) -> None: + keys = self._keys + indices = keys.indices + entries = keys.entries + for slot in range(keys.nslots): + idx = indices[slot] + if idx >= 0: + e2 = entries[idx] + assert e2 is not None + if e2.key is None: + entries[idx] = None + indices[slot] = -2 + self._used -= 1 + if e2.hash == -1: + e2.hash = hash(e2.identity) + + self._incr_version() + + def merge(self, arg: MDArg[_V] = None, /, **kwargs: _V) -> None: + """Merge into the dictionary, adding non-existing keys.""" + it = self._parse_args(arg, kwargs) + newsize = self._used + cast(int, next(it)) + log2_size = estimate_log2_keysize(newsize) + if log2_size > 17: # pragma: no cover + # Don't overallocate really huge keys space in update, + # duplicate keys could reduce the resulting anount of entries + log2_size = 17 + if log2_size > self._keys.log2_size: + self._resize(log2_size, False) + try: + self._merge_items(cast(Iterator[_Entry[_V]], it)) + finally: + self._post_update() + + def _merge_items(self, items: Iterator[_Entry[_V]]) -> None: + for entry in items: + hash_ = entry.hash + identity = entry.identity + for slot, idx, e in self._keys.iter_hash(hash_): + if e.identity == identity: # pragma: no branch + break + else: + self._add_with_hash_for_upd(entry) + + def _incr_version(self) -> None: + v = _version + v[0] += 1 + self._version = v[0] + + def _resize(self, log2_newsize: int, update: bool) -> None: + oldkeys = self._keys + newentries = self._used + + if len(oldkeys.entries) == newentries: + entries = oldkeys.entries + else: + entries = [e for e in oldkeys.entries if e is not None] + newkeys: _HtKeys[_V] = _HtKeys.new(log2_newsize, entries) + newkeys.usable -= newentries + newkeys.build_indices(update) + self._keys = newkeys + + def _add_with_hash(self, entry: _Entry[_V]) -> None: + if self._keys.usable <= 0: + self._resize((self._used * 3 | _HtKeys.MINSIZE - 1).bit_length(), False) + keys = self._keys + slot = keys.find_empty_slot(entry.hash) + keys.indices[slot] = len(keys.entries) + keys.entries.append(entry) + self._incr_version() + self._used += 1 + keys.usable -= 1 + + def _add_with_hash_for_upd(self, entry: _Entry[_V]) -> None: + if self._keys.usable <= 0: + self._resize((self._used * 3 | _HtKeys.MINSIZE - 1).bit_length(), True) + keys = self._keys + slot = keys.find_empty_slot(entry.hash) + keys.indices[slot] = len(keys.entries) + entry.hash = -1 + keys.entries.append(entry) + self._incr_version() + self._used += 1 + keys.usable -= 1 + + def _del_at(self, slot: int, idx: int) -> None: + self._keys.entries[idx] = None + self._keys.indices[slot] = -2 + self._used -= 1 + + def _del_at_for_upd(self, entry: _Entry[_V]) -> None: + entry.key = None # type: ignore[assignment] + entry.value = None # type: ignore[assignment] + + +class CIMultiDict(_CIMixin, MultiDict[_V]): + """Dictionary with the support for duplicate case-insensitive keys.""" + + +class MultiDictProxy(_CSMixin, MultiMapping[_V]): + """Read-only proxy for MultiDict instance.""" + + __slots__ = ("_md",) + + _md: MultiDict[_V] + + def __init__(self, arg: Union[MultiDict[_V], "MultiDictProxy[_V]"]): + if not isinstance(arg, (MultiDict, MultiDictProxy)): + raise TypeError( + f"ctor requires MultiDict or MultiDictProxy instance, not {type(arg)}" + ) + if isinstance(arg, MultiDictProxy): + self._md = arg._md + else: + self._md = arg + + def __reduce__(self) -> NoReturn: + raise TypeError(f"can't pickle {self.__class__.__name__} objects") + + @overload + def getall(self, key: str) -> list[_V]: ... + @overload + def getall(self, key: str, default: _T) -> Union[list[_V], _T]: ... + def getall( + self, key: str, default: Union[_T, _SENTINEL] = sentinel + ) -> Union[list[_V], _T]: + """Return a list of all values matching the key.""" + if default is not sentinel: + return self._md.getall(key, default) + else: + return self._md.getall(key) + + @overload + def getone(self, key: str) -> _V: ... + @overload + def getone(self, key: str, default: _T) -> Union[_V, _T]: ... + def getone( + self, key: str, default: Union[_T, _SENTINEL] = sentinel + ) -> Union[_V, _T]: + """Get first value matching the key. + + Raises KeyError if the key is not found and no default is provided. + """ + if default is not sentinel: + return self._md.getone(key, default) + else: + return self._md.getone(key) + + # Mapping interface # + + def __getitem__(self, key: str) -> _V: + return self.getone(key) + + @overload + def get(self, key: str, /) -> Union[_V, None]: ... + @overload + def get(self, key: str, /, default: _T) -> Union[_V, _T]: ... + def get(self, key: str, default: Union[_T, None] = None) -> Union[_V, _T, None]: + """Get first value matching the key. + + If the key is not found, returns the default (or None if no default is provided) + """ + return self._md.getone(key, default) + + def __iter__(self) -> Iterator[str]: + return iter(self._md.keys()) + + def __len__(self) -> int: + return len(self._md) + + def keys(self) -> KeysView[str]: + """Return a new view of the dictionary's keys.""" + return self._md.keys() + + def items(self) -> ItemsView[str, _V]: + """Return a new view of the dictionary's items *(key, value) pairs).""" + return self._md.items() + + def values(self) -> _ValuesView[_V]: + """Return a new view of the dictionary's values.""" + return self._md.values() + + def __eq__(self, other: object) -> bool: + return self._md == other + + def __contains__(self, key: object) -> bool: + return key in self._md + + @reprlib.recursive_repr() + def __repr__(self) -> str: + body = ", ".join(f"'{k}': {v!r}" for k, v in self.items()) + return f"<{self.__class__.__name__}({body})>" + + def copy(self) -> MultiDict[_V]: + """Return a copy of itself.""" + return MultiDict(self._md) + + +class CIMultiDictProxy(_CIMixin, MultiDictProxy[_V]): + """Read-only proxy for CIMultiDict instance.""" + + def __init__(self, arg: Union[MultiDict[_V], MultiDictProxy[_V]]): + if not isinstance(arg, (CIMultiDict, CIMultiDictProxy)): + raise TypeError( + "ctor requires CIMultiDict or CIMultiDictProxy instance" + f", not {type(arg)}" + ) + + super().__init__(arg) + + def copy(self) -> CIMultiDict[_V]: + """Return a copy of itself.""" + return CIMultiDict(self._md) + + +def getversion(md: Union[MultiDict[object], MultiDictProxy[object]]) -> int: + if isinstance(md, MultiDictProxy): + md = md._md + elif not isinstance(md, MultiDict): + raise TypeError("Parameter should be multidict or proxy") + return md._version diff --git a/py_modules/multidict/py.typed b/py_modules/multidict/py.typed new file mode 100644 index 0000000..dfe8cc0 --- /dev/null +++ b/py_modules/multidict/py.typed @@ -0,0 +1 @@ +PEP-561 marker. \ No newline at end of file diff --git a/py_modules/propcache/__init__.py b/py_modules/propcache/__init__.py new file mode 100644 index 0000000..bbfd78a --- /dev/null +++ b/py_modules/propcache/__init__.py @@ -0,0 +1,32 @@ +"""propcache: An accelerated property cache for Python classes.""" + +from typing import TYPE_CHECKING + +_PUBLIC_API = ("cached_property", "under_cached_property") + +__version__ = "0.5.2" +__all__ = () + +# Imports have moved to `propcache.api` in 0.2.0+. +# This module is now a facade for the API. +if TYPE_CHECKING: + from .api import cached_property as cached_property # noqa: F401 + from .api import under_cached_property as under_cached_property # noqa: F401 + + +def _import_facade(attr: str) -> object: + """Import the public API from the `api` module.""" + if attr in _PUBLIC_API: + from . import api # pylint: disable=import-outside-toplevel + + return getattr(api, attr) + raise AttributeError(f"module '{__package__}' has no attribute '{attr}'") + + +def _dir_facade() -> list[str]: + """Include the public API in the module's dir() output.""" + return [*_PUBLIC_API, *globals().keys()] + + +__getattr__ = _import_facade +__dir__ = _dir_facade diff --git a/py_modules/propcache/_helpers.py b/py_modules/propcache/_helpers.py new file mode 100644 index 0000000..1e52895 --- /dev/null +++ b/py_modules/propcache/_helpers.py @@ -0,0 +1,39 @@ +import os +import sys +from typing import TYPE_CHECKING + +__all__ = ("cached_property", "under_cached_property") + + +NO_EXTENSIONS = bool(os.environ.get("PROPCACHE_NO_EXTENSIONS")) # type: bool +if sys.implementation.name != "cpython": + NO_EXTENSIONS = True + + +# isort: off +if TYPE_CHECKING: + from ._helpers_py import cached_property as cached_property_py + from ._helpers_py import under_cached_property as under_cached_property_py + + cached_property = cached_property_py + under_cached_property = under_cached_property_py +elif not NO_EXTENSIONS: # pragma: no branch + try: + from ._helpers_c import cached_property as cached_property_c # type: ignore[attr-defined, unused-ignore] + from ._helpers_c import under_cached_property as under_cached_property_c # type: ignore[attr-defined, unused-ignore] + + cached_property = cached_property_c + under_cached_property = under_cached_property_c + except ImportError: # pragma: no cover + from ._helpers_py import cached_property as cached_property_py + from ._helpers_py import under_cached_property as under_cached_property_py + + cached_property = cached_property_py # type: ignore[assignment, misc] + under_cached_property = under_cached_property_py +else: + from ._helpers_py import cached_property as cached_property_py + from ._helpers_py import under_cached_property as under_cached_property_py + + cached_property = cached_property_py # type: ignore[assignment, misc] + under_cached_property = under_cached_property_py +# isort: on diff --git a/py_modules/propcache/_helpers_c.cpython-311-x86_64-linux-gnu.so b/py_modules/propcache/_helpers_c.cpython-311-x86_64-linux-gnu.so new file mode 100755 index 0000000..75d7c20 Binary files /dev/null and b/py_modules/propcache/_helpers_c.cpython-311-x86_64-linux-gnu.so differ diff --git a/py_modules/propcache/_helpers_c.pyx b/py_modules/propcache/_helpers_c.pyx new file mode 100644 index 0000000..9e9e558 --- /dev/null +++ b/py_modules/propcache/_helpers_c.pyx @@ -0,0 +1,103 @@ +# cython: language_level=3, freethreading_compatible=True +from types import GenericAlias + +from cpython.dict cimport PyDict_GetItem +from cpython.object cimport PyObject + + +cdef extern from "Python.h": + # Call a callable Python object callable with exactly + # 1 positional argument arg and no keyword arguments. + # Return the result of the call on success, or raise + # an exception and return NULL on failure. + PyObject* PyObject_CallOneArg( + object callable, object arg + ) except NULL + int PyDict_SetItem( + object dict, object key, PyObject* value + ) except -1 + void Py_DECREF(PyObject*) + + +cdef class under_cached_property: + """Use as a class method decorator. It operates almost exactly like + the Python `@property` decorator, but it puts the result of the + method it decorates into the instance dict after the first call, + effectively replacing the function it decorates with an instance + variable. It is, in Python parlance, a data descriptor. + + """ + + cdef readonly object wrapped + cdef object name + + def __init__(self, object wrapped): + self.wrapped = wrapped + self.name = wrapped.__name__ + + @property + def __doc__(self): + return self.wrapped.__doc__ + + def __get__(self, object inst, owner): + if inst is None: + return self + cdef dict cache = inst._cache + cdef PyObject* val = PyDict_GetItem(cache, self.name) + if val == NULL: + val = PyObject_CallOneArg(self.wrapped, inst) + PyDict_SetItem(cache, self.name, val) + Py_DECREF(val) + return val + + def __set__(self, inst, value): + raise AttributeError("cached property is read-only") + + __class_getitem__ = classmethod(GenericAlias) + + +cdef class cached_property: + """Use as a class method decorator. It operates almost exactly like + the Python `@property` decorator, but it puts the result of the + method it decorates into the instance dict after the first call, + effectively replacing the function it decorates with an instance + variable. It is, in Python parlance, a data descriptor. + + """ + + cdef readonly object func + cdef object name + + def __init__(self, func): + self.func = func + self.name = None + + @property + def __doc__(self): + return self.func.__doc__ + + def __set_name__(self, owner, object name): + if self.name is None: + self.name = name + elif name != self.name: + raise TypeError( + "Cannot assign the same cached_property to two different names " + f"({self.name!r} and {name!r})." + ) + + def __get__(self, inst, owner): + if inst is None: + return self + if self.name is None: + raise TypeError( + "Cannot use cached_property instance" + " without calling __set_name__ on it.") + cdef dict cache = inst.__dict__ + cdef PyObject* val = PyDict_GetItem(cache, self.name) + if val is NULL: + val = PyObject_CallOneArg(self.func, inst) + PyDict_SetItem(cache, self.name, val) + Py_DECREF(val) + return val + + __class_getitem__ = classmethod(GenericAlias) diff --git a/py_modules/propcache/_helpers_py.py b/py_modules/propcache/_helpers_py.py new file mode 100644 index 0000000..db5b8d6 --- /dev/null +++ b/py_modules/propcache/_helpers_py.py @@ -0,0 +1,64 @@ +"""Various helper functions.""" + +from __future__ import annotations + +import sys +from collections.abc import Callable, Mapping +from functools import cached_property +from typing import Any, Generic, Protocol, TypeVar, overload + +__all__ = ("under_cached_property", "cached_property") + + +if sys.version_info >= (3, 11): + from typing import Self +else: + Self = Any + +_T = TypeVar("_T") +# We use Mapping to make it possible to use TypedDict, but this isn't +# technically type safe as we need to assign into the dict. +_Cache = TypeVar("_Cache", bound=Mapping[str, Any]) + + +class _CacheImpl(Protocol[_Cache]): + _cache: _Cache + + +class under_cached_property(Generic[_T]): + """Use as a class method decorator. + + It operates almost exactly like + the Python `@property` decorator, but it puts the result of the + method it decorates into the instance dict after the first call, + effectively replacing the function it decorates with an instance + variable. It is, in Python parlance, a data descriptor. + """ + + def __init__(self, wrapped: Callable[[Any], _T]) -> None: + self.wrapped = wrapped + self.__doc__ = wrapped.__doc__ + self.name = wrapped.__name__ + + @overload + def __get__(self, inst: None, owner: type[object] | None = None) -> Self: ... + + @overload + def __get__( + self, inst: _CacheImpl[Any], owner: type[object] | None = None + ) -> _T: ... + + def __get__( + self, inst: _CacheImpl[Any] | None, owner: type[object] | None = None + ) -> _T | Self: + if inst is None: + return self + try: + return inst._cache[self.name] # type: ignore[no-any-return] + except KeyError: + val = self.wrapped(inst) + inst._cache[self.name] = val + return val + + def __set__(self, inst: _CacheImpl[Any], value: _T) -> None: + raise AttributeError("cached property is read-only") diff --git a/py_modules/propcache/api.py b/py_modules/propcache/api.py new file mode 100644 index 0000000..22389e6 --- /dev/null +++ b/py_modules/propcache/api.py @@ -0,0 +1,8 @@ +"""Public API of the property caching library.""" + +from ._helpers import cached_property, under_cached_property + +__all__ = ( + "cached_property", + "under_cached_property", +) diff --git a/py_modules/propcache/py.typed b/py_modules/propcache/py.typed new file mode 100644 index 0000000..dcf2c80 --- /dev/null +++ b/py_modules/propcache/py.typed @@ -0,0 +1 @@ +# Placeholder diff --git a/py_modules/pycparser/__init__.py b/py_modules/pycparser/__init__.py new file mode 100644 index 0000000..0606c03 --- /dev/null +++ b/py_modules/pycparser/__init__.py @@ -0,0 +1,99 @@ +# ----------------------------------------------------------------- +# pycparser: __init__.py +# +# This package file exports some convenience functions for +# interacting with pycparser +# +# Eli Bendersky [https://eli.thegreenplace.net/] +# License: BSD +# ----------------------------------------------------------------- +__all__ = ["c_lexer", "c_parser", "c_ast"] +__version__ = "3.00" + +import io +from subprocess import check_output + +from . import c_parser + +CParser = c_parser.CParser + + +def preprocess_file(filename, cpp_path="cpp", cpp_args=""): + """Preprocess a file using cpp. + + filename: + Name of the file you want to preprocess. + + cpp_path: + cpp_args: + Refer to the documentation of parse_file for the meaning of these + arguments. + + When successful, returns the preprocessed file's contents. + Errors from cpp will be printed out. + """ + path_list = [cpp_path] + if isinstance(cpp_args, list): + path_list += cpp_args + elif cpp_args != "": + path_list += [cpp_args] + path_list += [filename] + + try: + # Note the use of universal_newlines to treat all newlines + # as \n for Python's purpose + text = check_output(path_list, universal_newlines=True) + except OSError as e: + raise RuntimeError( + "Unable to invoke 'cpp'. " + + "Make sure its path was passed correctly\n" + + f"Original error: {e}" + ) + + return text + + +def parse_file( + filename, use_cpp=False, cpp_path="cpp", cpp_args="", parser=None, encoding=None +): + """Parse a C file using pycparser. + + filename: + Name of the file you want to parse. + + use_cpp: + Set to True if you want to execute the C pre-processor + on the file prior to parsing it. + + cpp_path: + If use_cpp is True, this is the path to 'cpp' on your + system. If no path is provided, it attempts to just + execute 'cpp', so it must be in your PATH. + + cpp_args: + If use_cpp is True, set this to the command line arguments strings + to cpp. Be careful with quotes - it's best to pass a raw string + (r'') here. For example: + r'-I../utils/fake_libc_include' + If several arguments are required, pass a list of strings. + + encoding: + Encoding to use for the file to parse + + parser: + Optional parser object to be used instead of the default CParser + + When successful, an AST is returned. ParseError can be + thrown if the file doesn't parse successfully. + + Errors from cpp will be printed out. + """ + if use_cpp: + text = preprocess_file(filename, cpp_path, cpp_args) + else: + with io.open(filename, encoding=encoding) as f: + text = f.read() + + if parser is None: + parser = CParser() + return parser.parse(text, filename) diff --git a/py_modules/pycparser/_ast_gen.py b/py_modules/pycparser/_ast_gen.py new file mode 100644 index 0000000..6df0b6a --- /dev/null +++ b/py_modules/pycparser/_ast_gen.py @@ -0,0 +1,355 @@ +# ----------------------------------------------------------------- +# _ast_gen.py +# +# Generates the AST Node classes from a specification given in +# a configuration file. This module can also be run as a script to +# regenerate c_ast.py from _c_ast.cfg (from the repo root or the +# pycparser/ directory). Use 'make check' to reformat the generated +# file after running this script. +# +# The design of this module was inspired by astgen.py from the +# Python 2.5 code-base. +# +# Eli Bendersky [https://eli.thegreenplace.net/] +# License: BSD +# ----------------------------------------------------------------- +from string import Template +import os +from typing import IO + + +class ASTCodeGenerator: + def __init__(self, cfg_filename="_c_ast.cfg"): + """Initialize the code generator from a configuration + file. + """ + self.cfg_filename = cfg_filename + self.node_cfg = [ + NodeCfg(name, contents) + for (name, contents) in self.parse_cfgfile(cfg_filename) + ] + + def generate(self, file: IO[str]) -> None: + """Generates the code into file, an open file buffer.""" + src = Template(_PROLOGUE_COMMENT).substitute(cfg_filename=self.cfg_filename) + + src += _PROLOGUE_CODE + for node_cfg in self.node_cfg: + src += node_cfg.generate_source() + "\n\n" + + file.write(src) + + def parse_cfgfile(self, filename): + """Parse the configuration file and yield pairs of + (name, contents) for each node. + """ + with open(filename, "r") as f: + for line in f: + line = line.strip() + if not line or line.startswith("#"): + continue + colon_i = line.find(":") + lbracket_i = line.find("[") + rbracket_i = line.find("]") + if colon_i < 1 or lbracket_i <= colon_i or rbracket_i <= lbracket_i: + raise RuntimeError(f"Invalid line in {filename}:\n{line}\n") + + name = line[:colon_i] + val = line[lbracket_i + 1 : rbracket_i] + vallist = [v.strip() for v in val.split(",")] if val else [] + yield name, vallist + + +class NodeCfg: + """Node configuration. + + name: node name + contents: a list of contents - attributes and child nodes + See comment at the top of the configuration file for details. + """ + + def __init__(self, name, contents): + self.name = name + self.all_entries = [] + self.attr = [] + self.child = [] + self.seq_child = [] + + for entry in contents: + clean_entry = entry.rstrip("*") + self.all_entries.append(clean_entry) + + if entry.endswith("**"): + self.seq_child.append(clean_entry) + elif entry.endswith("*"): + self.child.append(clean_entry) + else: + self.attr.append(entry) + + def generate_source(self): + src = self._gen_init() + src += "\n" + self._gen_children() + src += "\n" + self._gen_iter() + src += "\n" + self._gen_attr_names() + return src + + def _gen_init(self): + src = f"class {self.name}(Node):\n" + + if self.all_entries: + args = ", ".join(self.all_entries) + slots = ", ".join(f"'{e}'" for e in self.all_entries) + slots += ", 'coord', '__weakref__'" + arglist = f"(self, {args}, coord=None)" + else: + slots = "'coord', '__weakref__'" + arglist = "(self, coord=None)" + + src += f" __slots__ = ({slots})\n" + src += f" def __init__{arglist}:\n" + + for name in self.all_entries + ["coord"]: + src += f" self.{name} = {name}\n" + + return src + + def _gen_children(self): + src = " def children(self):\n" + + if self.all_entries: + src += " nodelist = []\n" + + for child in self.child: + src += f" if self.{child} is not None:\n" + src += f' nodelist.append(("{child}", self.{child}))\n' + + for seq_child in self.seq_child: + src += f" for i, child in enumerate(self.{seq_child} or []):\n" + src += f' nodelist.append((f"{seq_child}[{{i}}]", child))\n' + + src += " return tuple(nodelist)\n" + else: + src += " return ()\n" + + return src + + def _gen_iter(self): + src = " def __iter__(self):\n" + + if self.all_entries: + for child in self.child: + src += f" if self.{child} is not None:\n" + src += f" yield self.{child}\n" + + for seq_child in self.seq_child: + src += f" for child in (self.{seq_child} or []):\n" + src += " yield child\n" + + if not (self.child or self.seq_child): + # Empty generator + src += " return\n" + " yield\n" + else: + # Empty generator + src += " return\n" + " yield\n" + + return src + + def _gen_attr_names(self): + src = " attr_names = (" + "".join(f"{nm!r}, " for nm in self.attr) + ")" + return src + + +_PROLOGUE_COMMENT = r"""#----------------------------------------------------------------- +# ** ATTENTION ** +# This code was automatically generated from _c_ast.cfg +# +# Do not modify it directly. Modify the configuration file and +# run the generator again. +# ** ** *** ** ** +# +# pycparser: c_ast.py +# +# AST Node classes. +# +# Eli Bendersky [https://eli.thegreenplace.net/] +# License: BSD +#----------------------------------------------------------------- + +""" +_PROLOGUE_CODE = r''' +import sys +from typing import Any, ClassVar, IO, Optional + +def _repr(obj): + """ + Get the representation of an object, with dedicated pprint-like format for lists. + """ + if isinstance(obj, list): + return '[' + (',\n '.join((_repr(e).replace('\n', '\n ') for e in obj))) + '\n]' + else: + return repr(obj) + +class Node: + __slots__ = () + """ Abstract base class for AST nodes. + """ + attr_names: ClassVar[tuple[str, ...]] = () + coord: Optional[Any] + def __repr__(self): + """ Generates a python representation of the current node + """ + result = self.__class__.__name__ + '(' + + indent = '' + separator = '' + for name in self.__slots__[:-2]: + result += separator + result += indent + result += name + '=' + (_repr(getattr(self, name)).replace('\n', '\n ' + (' ' * (len(name) + len(self.__class__.__name__))))) + + separator = ',' + indent = '\n ' + (' ' * len(self.__class__.__name__)) + + result += indent + ')' + + return result + + def children(self): + """ A sequence of all children that are Nodes + """ + pass + + def show( + self, + buf: IO[str] = sys.stdout, + offset: int = 0, + attrnames: bool = False, + showemptyattrs: bool = True, + nodenames: bool = False, + showcoord: bool = False, + _my_node_name: Optional[str] = None, + ): + """ Pretty print the Node and all its attributes and + children (recursively) to a buffer. + + buf: + Open IO buffer into which the Node is printed. + + offset: + Initial offset (amount of leading spaces) + + attrnames: + True if you want to see the attribute names in + name=value pairs. False to only see the values. + + showemptyattrs: + False if you want to suppress printing empty attributes. + + nodenames: + True if you want to see the actual node names + within their parents. + + showcoord: + Do you want the coordinates of each Node to be + displayed. + """ + lead = ' ' * offset + if nodenames and _my_node_name is not None: + buf.write(lead + self.__class__.__name__+ ' <' + _my_node_name + '>: ') + else: + buf.write(lead + self.__class__.__name__+ ': ') + + if self.attr_names: + def is_empty(v): + v is None or (hasattr(v, '__len__') and len(v) == 0) + nvlist = [(n, getattr(self,n)) for n in self.attr_names \ + if showemptyattrs or not is_empty(getattr(self,n))] + if attrnames: + attrstr = ', '.join(f'{name}={value}' for name, value in nvlist) + else: + attrstr = ', '.join(f'{value}' for _, value in nvlist) + buf.write(attrstr) + + if showcoord: + buf.write(f' (at {self.coord})') + buf.write('\n') + + for (child_name, child) in self.children(): + child.show( + buf, + offset=offset + 2, + attrnames=attrnames, + showemptyattrs=showemptyattrs, + nodenames=nodenames, + showcoord=showcoord, + _my_node_name=child_name) + + +class NodeVisitor: + """ A base NodeVisitor class for visiting c_ast nodes. + Subclass it and define your own visit_XXX methods, where + XXX is the class name you want to visit with these + methods. + + For example: + + class ConstantVisitor(NodeVisitor): + def __init__(self): + self.values = [] + + def visit_Constant(self, node): + self.values.append(node.value) + + Creates a list of values of all the constant nodes + encountered below the given node. To use it: + + cv = ConstantVisitor() + cv.visit(node) + + Notes: + + * generic_visit() will be called for AST nodes for which + no visit_XXX method was defined. + * The children of nodes for which a visit_XXX was + defined will not be visited - if you need this, call + generic_visit() on the node. + You can use: + NodeVisitor.generic_visit(self, node) + * Modeled after Python's own AST visiting facilities + (the ast module of Python 3.0) + """ + + _method_cache = None + + def visit(self, node: Node): + """ Visit a node. + """ + + if self._method_cache is None: + self._method_cache = {} + + visitor = self._method_cache.get(node.__class__.__name__, None) + if visitor is None: + method = 'visit_' + node.__class__.__name__ + visitor = getattr(self, method, self.generic_visit) + self._method_cache[node.__class__.__name__] = visitor + + return visitor(node) + + def generic_visit(self, node: Node): + """ Called if no explicit visitor function exists for a + node. Implements preorder visiting of the node. + """ + for _, c in node.children(): + self.visit(c) + +''' + + +if __name__ == "__main__": + base_dir = os.path.dirname(os.path.abspath(__file__)) + cfg_path = os.path.join(base_dir, "_c_ast.cfg") + out_path = os.path.join(base_dir, "c_ast.py") + ast_gen = ASTCodeGenerator(cfg_path) + with open(out_path, "w") as out: + ast_gen.generate(out) diff --git a/py_modules/pycparser/_c_ast.cfg b/py_modules/pycparser/_c_ast.cfg new file mode 100644 index 0000000..0626533 --- /dev/null +++ b/py_modules/pycparser/_c_ast.cfg @@ -0,0 +1,195 @@ +#----------------------------------------------------------------- +# pycparser: _c_ast.cfg +# +# Defines the AST Node classes used in pycparser. +# +# Each entry is a Node sub-class name, listing the attributes +# and child nodes of the class: +# * - a child node +# ** - a sequence of child nodes +# - an attribute +# +# Eli Bendersky [https://eli.thegreenplace.net/] +# License: BSD +#----------------------------------------------------------------- + +# ArrayDecl is a nested declaration of an array with the given type. +# dim: the dimension (for example, constant 42) +# dim_quals: list of dimension qualifiers, to support C99's allowing 'const' +# and 'static' within the array dimension in function declarations. +ArrayDecl: [type*, dim*, dim_quals] + +ArrayRef: [name*, subscript*] + +# op: =, +=, /= etc. +# +Assignment: [op, lvalue*, rvalue*] + +Alignas: [alignment*] + +BinaryOp: [op, left*, right*] + +Break: [] + +Case: [expr*, stmts**] + +Cast: [to_type*, expr*] + +# Compound statement in C99 is a list of block items (declarations or +# statements). +# +Compound: [block_items**] + +# Compound literal (anonymous aggregate) for C99. +# (type-name) {initializer_list} +# type: the typename +# init: InitList for the initializer list +# +CompoundLiteral: [type*, init*] + +# type: int, char, float, string, etc. +# +Constant: [type, value] + +Continue: [] + +# name: the variable being declared +# quals: list of qualifiers (const, volatile) +# funcspec: list function specifiers (i.e. inline in C99) +# storage: list of storage specifiers (extern, register, etc.) +# type: declaration type (probably nested with all the modifiers) +# init: initialization value, or None +# bitsize: bit field size, or None +# +Decl: [name, quals, align, storage, funcspec, type*, init*, bitsize*] + +DeclList: [decls**] + +Default: [stmts**] + +DoWhile: [cond*, stmt*] + +# Represents the ellipsis (...) parameter in a function +# declaration +# +EllipsisParam: [] + +# An empty statement (a semicolon ';' on its own) +# +EmptyStatement: [] + +# Enumeration type specifier +# name: an optional ID +# values: an EnumeratorList +# +Enum: [name, values*] + +# A name/value pair for enumeration values +# +Enumerator: [name, value*] + +# A list of enumerators +# +EnumeratorList: [enumerators**] + +# A list of expressions separated by the comma operator. +# +ExprList: [exprs**] + +# This is the top of the AST, representing a single C file (a +# translation unit in K&R jargon). It contains a list of +# "external-declaration"s, which is either declarations (Decl), +# Typedef or function definitions (FuncDef). +# +FileAST: [ext**] + +# for (init; cond; next) stmt +# +For: [init*, cond*, next*, stmt*] + +# name: Id +# args: ExprList +# +FuncCall: [name*, args*] + +# type (args) +# +FuncDecl: [args*, type*] + +# Function definition: a declarator for the function name and +# a body, which is a compound statement. +# There's an optional list of parameter declarations for old +# K&R-style definitions +# +FuncDef: [decl*, param_decls**, body*] + +Goto: [name] + +ID: [name] + +# Holder for types that are a simple identifier (e.g. the built +# ins void, char etc. and typedef-defined types) +# +IdentifierType: [names] + +If: [cond*, iftrue*, iffalse*] + +# An initialization list used for compound literals. +# +InitList: [exprs**] + +Label: [name, stmt*] + +# A named initializer for C99. +# The name of a NamedInitializer is a sequence of Nodes, because +# names can be hierarchical and contain constant expressions. +# +NamedInitializer: [name**, expr*] + +# a list of comma separated function parameter declarations +# +ParamList: [params**] + +PtrDecl: [quals, type*] + +Return: [expr*] + +StaticAssert: [cond*, message*] + +# name: struct tag name +# decls: declaration of members +# +Struct: [name, decls**] + +# type: . or -> +# name.field or name->field +# +StructRef: [name*, type, field*] + +Switch: [cond*, stmt*] + +# cond ? iftrue : iffalse +# +TernaryOp: [cond*, iftrue*, iffalse*] + +# A base type declaration +# +TypeDecl: [declname, quals, align, type*] + +# A typedef declaration. +# Very similar to Decl, but without some attributes +# +Typedef: [name, quals, storage, type*] + +Typename: [name, quals, align, type*] + +UnaryOp: [op, expr*] + +# name: union tag name +# decls: declaration of members +# +Union: [name, decls**] + +While: [cond*, stmt*] + +Pragma: [string] diff --git a/py_modules/pycparser/ast_transforms.py b/py_modules/pycparser/ast_transforms.py new file mode 100644 index 0000000..1051737 --- /dev/null +++ b/py_modules/pycparser/ast_transforms.py @@ -0,0 +1,174 @@ +# ------------------------------------------------------------------------------ +# pycparser: ast_transforms.py +# +# Some utilities used by the parser to create a friendlier AST. +# +# Eli Bendersky [https://eli.thegreenplace.net/] +# License: BSD +# ------------------------------------------------------------------------------ + +from typing import Any, List, Tuple, cast + +from . import c_ast + + +def fix_switch_cases(switch_node: c_ast.Switch) -> c_ast.Switch: + """The 'case' statements in a 'switch' come out of parsing with one + child node, so subsequent statements are just tucked to the parent + Compound. Additionally, consecutive (fall-through) case statements + come out messy. This is a peculiarity of the C grammar. The following: + + switch (myvar) { + case 10: + k = 10; + p = k + 1; + return 10; + case 20: + case 30: + return 20; + default: + break; + } + + Creates this tree (pseudo-dump): + + Switch + ID: myvar + Compound: + Case 10: + k = 10 + p = k + 1 + return 10 + Case 20: + Case 30: + return 20 + Default: + break + + The goal of this transform is to fix this mess, turning it into the + following: + + Switch + ID: myvar + Compound: + Case 10: + k = 10 + p = k + 1 + return 10 + Case 20: + Case 30: + return 20 + Default: + break + + A fixed AST node is returned. The argument may be modified. + """ + assert isinstance(switch_node, c_ast.Switch) + if not isinstance(switch_node.stmt, c_ast.Compound): + return switch_node + + # The new Compound child for the Switch, which will collect children in the + # correct order + new_compound = c_ast.Compound([], switch_node.stmt.coord) + + # The last Case/Default node + last_case: c_ast.Case | c_ast.Default | None = None + + # Goes over the children of the Compound below the Switch, adding them + # either directly below new_compound or below the last Case as appropriate + # (for `switch(cond) {}`, block_items would have been None) + for child in switch_node.stmt.block_items or []: + if isinstance(child, (c_ast.Case, c_ast.Default)): + # If it's a Case/Default: + # 1. Add it to the Compound and mark as "last case" + # 2. If its immediate child is also a Case or Default, promote it + # to a sibling. + new_compound.block_items.append(child) + _extract_nested_case(child, new_compound.block_items) + last_case = new_compound.block_items[-1] + else: + # Other statements are added as children to the last case, if it + # exists. + if last_case is None: + new_compound.block_items.append(child) + else: + last_case.stmts.append(child) + + switch_node.stmt = new_compound + return switch_node + + +def _extract_nested_case( + case_node: c_ast.Case | c_ast.Default, stmts_list: List[c_ast.Node] +) -> None: + """Recursively extract consecutive Case statements that are made nested + by the parser and add them to the stmts_list. + """ + if isinstance(case_node.stmts[0], (c_ast.Case, c_ast.Default)): + nested = case_node.stmts.pop() + stmts_list.append(nested) + _extract_nested_case(cast(Any, nested), stmts_list) + + +def fix_atomic_specifiers( + decl: c_ast.Decl | c_ast.Typedef, +) -> c_ast.Decl | c_ast.Typedef: + """Atomic specifiers like _Atomic(type) are unusually structured, + conferring a qualifier upon the contained type. + + This function fixes a decl with atomic specifiers to have a sane AST + structure, by removing spurious Typename->TypeDecl pairs and attaching + the _Atomic qualifier in the right place. + """ + # There can be multiple levels of _Atomic in a decl; fix them until a + # fixed point is reached. + while True: + decl, found = _fix_atomic_specifiers_once(decl) + if not found: + break + + # Make sure to add an _Atomic qual on the topmost decl if needed. Also + # restore the declname on the innermost TypeDecl (it gets placed in the + # wrong place during construction). + typ: Any = decl + while not isinstance(typ, c_ast.TypeDecl): + try: + typ = typ.type + except AttributeError: + return decl + if "_Atomic" in typ.quals and "_Atomic" not in decl.quals: + decl.quals.append("_Atomic") + if typ.declname is None: + typ.declname = decl.name + + return decl + + +def _fix_atomic_specifiers_once( + decl: c_ast.Decl | c_ast.Typedef, +) -> Tuple[c_ast.Decl | c_ast.Typedef, bool]: + """Performs one 'fix' round of atomic specifiers. + Returns (modified_decl, found) where found is True iff a fix was made. + """ + parent: Any = decl + grandparent: Any = None + node: Any = decl.type + while node is not None: + if isinstance(node, c_ast.Typename) and "_Atomic" in node.quals: + break + try: + grandparent = parent + parent = node + node = node.type + except AttributeError: + # If we've reached a node without a `type` field, it means we won't + # find what we're looking for at this point; give up the search + # and return the original decl unmodified. + return decl, False + + assert isinstance(parent, c_ast.TypeDecl) + assert grandparent is not None + cast(Any, grandparent).type = node.type + if "_Atomic" not in node.type.quals: + node.type.quals.append("_Atomic") + return decl, True diff --git a/py_modules/pycparser/c_ast.py b/py_modules/pycparser/c_ast.py new file mode 100644 index 0000000..b6f42af --- /dev/null +++ b/py_modules/pycparser/c_ast.py @@ -0,0 +1,1341 @@ +# ----------------------------------------------------------------- +# ** ATTENTION ** +# This code was automatically generated from _c_ast.cfg +# +# Do not modify it directly. Modify the configuration file and +# run the generator again. +# ** ** *** ** ** +# +# pycparser: c_ast.py +# +# AST Node classes. +# +# Eli Bendersky [https://eli.thegreenplace.net/] +# License: BSD +# ----------------------------------------------------------------- + + +import sys +from typing import Any, ClassVar, IO, Optional + + +def _repr(obj): + """ + Get the representation of an object, with dedicated pprint-like format for lists. + """ + if isinstance(obj, list): + return "[" + (",\n ".join((_repr(e).replace("\n", "\n ") for e in obj))) + "\n]" + else: + return repr(obj) + + +class Node: + __slots__ = () + """ Abstract base class for AST nodes. + """ + attr_names: ClassVar[tuple[str, ...]] = () + coord: Optional[Any] + + def __repr__(self): + """Generates a python representation of the current node""" + result = self.__class__.__name__ + "(" + + indent = "" + separator = "" + for name in self.__slots__[:-2]: + result += separator + result += indent + result += ( + name + + "=" + + ( + _repr(getattr(self, name)).replace( + "\n", + "\n " + (" " * (len(name) + len(self.__class__.__name__))), + ) + ) + ) + + separator = "," + indent = "\n " + (" " * len(self.__class__.__name__)) + + result += indent + ")" + + return result + + def children(self): + """A sequence of all children that are Nodes""" + pass + + def show( + self, + buf: IO[str] = sys.stdout, + offset: int = 0, + attrnames: bool = False, + showemptyattrs: bool = True, + nodenames: bool = False, + showcoord: bool = False, + _my_node_name: Optional[str] = None, + ): + """Pretty print the Node and all its attributes and + children (recursively) to a buffer. + + buf: + Open IO buffer into which the Node is printed. + + offset: + Initial offset (amount of leading spaces) + + attrnames: + True if you want to see the attribute names in + name=value pairs. False to only see the values. + + showemptyattrs: + False if you want to suppress printing empty attributes. + + nodenames: + True if you want to see the actual node names + within their parents. + + showcoord: + Do you want the coordinates of each Node to be + displayed. + """ + lead = " " * offset + if nodenames and _my_node_name is not None: + buf.write(lead + self.__class__.__name__ + " <" + _my_node_name + ">: ") + else: + buf.write(lead + self.__class__.__name__ + ": ") + + if self.attr_names: + + def is_empty(v): + v is None or (hasattr(v, "__len__") and len(v) == 0) + + nvlist = [ + (n, getattr(self, n)) + for n in self.attr_names + if showemptyattrs or not is_empty(getattr(self, n)) + ] + if attrnames: + attrstr = ", ".join(f"{name}={value}" for name, value in nvlist) + else: + attrstr = ", ".join(f"{value}" for _, value in nvlist) + buf.write(attrstr) + + if showcoord: + buf.write(f" (at {self.coord})") + buf.write("\n") + + for child_name, child in self.children(): + child.show( + buf, + offset=offset + 2, + attrnames=attrnames, + showemptyattrs=showemptyattrs, + nodenames=nodenames, + showcoord=showcoord, + _my_node_name=child_name, + ) + + +class NodeVisitor: + """A base NodeVisitor class for visiting c_ast nodes. + Subclass it and define your own visit_XXX methods, where + XXX is the class name you want to visit with these + methods. + + For example: + + class ConstantVisitor(NodeVisitor): + def __init__(self): + self.values = [] + + def visit_Constant(self, node): + self.values.append(node.value) + + Creates a list of values of all the constant nodes + encountered below the given node. To use it: + + cv = ConstantVisitor() + cv.visit(node) + + Notes: + + * generic_visit() will be called for AST nodes for which + no visit_XXX method was defined. + * The children of nodes for which a visit_XXX was + defined will not be visited - if you need this, call + generic_visit() on the node. + You can use: + NodeVisitor.generic_visit(self, node) + * Modeled after Python's own AST visiting facilities + (the ast module of Python 3.0) + """ + + _method_cache = None + + def visit(self, node: Node): + """Visit a node.""" + + if self._method_cache is None: + self._method_cache = {} + + visitor = self._method_cache.get(node.__class__.__name__, None) + if visitor is None: + method = "visit_" + node.__class__.__name__ + visitor = getattr(self, method, self.generic_visit) + self._method_cache[node.__class__.__name__] = visitor + + return visitor(node) + + def generic_visit(self, node: Node): + """Called if no explicit visitor function exists for a + node. Implements preorder visiting of the node. + """ + for _, c in node.children(): + self.visit(c) + + +class ArrayDecl(Node): + __slots__ = ("type", "dim", "dim_quals", "coord", "__weakref__") + + def __init__(self, type, dim, dim_quals, coord=None): + self.type = type + self.dim = dim + self.dim_quals = dim_quals + self.coord = coord + + def children(self): + nodelist = [] + if self.type is not None: + nodelist.append(("type", self.type)) + if self.dim is not None: + nodelist.append(("dim", self.dim)) + return tuple(nodelist) + + def __iter__(self): + if self.type is not None: + yield self.type + if self.dim is not None: + yield self.dim + + attr_names = ("dim_quals",) + + +class ArrayRef(Node): + __slots__ = ("name", "subscript", "coord", "__weakref__") + + def __init__(self, name, subscript, coord=None): + self.name = name + self.subscript = subscript + self.coord = coord + + def children(self): + nodelist = [] + if self.name is not None: + nodelist.append(("name", self.name)) + if self.subscript is not None: + nodelist.append(("subscript", self.subscript)) + return tuple(nodelist) + + def __iter__(self): + if self.name is not None: + yield self.name + if self.subscript is not None: + yield self.subscript + + attr_names = () + + +class Assignment(Node): + __slots__ = ("op", "lvalue", "rvalue", "coord", "__weakref__") + + def __init__(self, op, lvalue, rvalue, coord=None): + self.op = op + self.lvalue = lvalue + self.rvalue = rvalue + self.coord = coord + + def children(self): + nodelist = [] + if self.lvalue is not None: + nodelist.append(("lvalue", self.lvalue)) + if self.rvalue is not None: + nodelist.append(("rvalue", self.rvalue)) + return tuple(nodelist) + + def __iter__(self): + if self.lvalue is not None: + yield self.lvalue + if self.rvalue is not None: + yield self.rvalue + + attr_names = ("op",) + + +class Alignas(Node): + __slots__ = ("alignment", "coord", "__weakref__") + + def __init__(self, alignment, coord=None): + self.alignment = alignment + self.coord = coord + + def children(self): + nodelist = [] + if self.alignment is not None: + nodelist.append(("alignment", self.alignment)) + return tuple(nodelist) + + def __iter__(self): + if self.alignment is not None: + yield self.alignment + + attr_names = () + + +class BinaryOp(Node): + __slots__ = ("op", "left", "right", "coord", "__weakref__") + + def __init__(self, op, left, right, coord=None): + self.op = op + self.left = left + self.right = right + self.coord = coord + + def children(self): + nodelist = [] + if self.left is not None: + nodelist.append(("left", self.left)) + if self.right is not None: + nodelist.append(("right", self.right)) + return tuple(nodelist) + + def __iter__(self): + if self.left is not None: + yield self.left + if self.right is not None: + yield self.right + + attr_names = ("op",) + + +class Break(Node): + __slots__ = ("coord", "__weakref__") + + def __init__(self, coord=None): + self.coord = coord + + def children(self): + return () + + def __iter__(self): + return + yield + + attr_names = () + + +class Case(Node): + __slots__ = ("expr", "stmts", "coord", "__weakref__") + + def __init__(self, expr, stmts, coord=None): + self.expr = expr + self.stmts = stmts + self.coord = coord + + def children(self): + nodelist = [] + if self.expr is not None: + nodelist.append(("expr", self.expr)) + for i, child in enumerate(self.stmts or []): + nodelist.append((f"stmts[{i}]", child)) + return tuple(nodelist) + + def __iter__(self): + if self.expr is not None: + yield self.expr + for child in self.stmts or []: + yield child + + attr_names = () + + +class Cast(Node): + __slots__ = ("to_type", "expr", "coord", "__weakref__") + + def __init__(self, to_type, expr, coord=None): + self.to_type = to_type + self.expr = expr + self.coord = coord + + def children(self): + nodelist = [] + if self.to_type is not None: + nodelist.append(("to_type", self.to_type)) + if self.expr is not None: + nodelist.append(("expr", self.expr)) + return tuple(nodelist) + + def __iter__(self): + if self.to_type is not None: + yield self.to_type + if self.expr is not None: + yield self.expr + + attr_names = () + + +class Compound(Node): + __slots__ = ("block_items", "coord", "__weakref__") + + def __init__(self, block_items, coord=None): + self.block_items = block_items + self.coord = coord + + def children(self): + nodelist = [] + for i, child in enumerate(self.block_items or []): + nodelist.append((f"block_items[{i}]", child)) + return tuple(nodelist) + + def __iter__(self): + for child in self.block_items or []: + yield child + + attr_names = () + + +class CompoundLiteral(Node): + __slots__ = ("type", "init", "coord", "__weakref__") + + def __init__(self, type, init, coord=None): + self.type = type + self.init = init + self.coord = coord + + def children(self): + nodelist = [] + if self.type is not None: + nodelist.append(("type", self.type)) + if self.init is not None: + nodelist.append(("init", self.init)) + return tuple(nodelist) + + def __iter__(self): + if self.type is not None: + yield self.type + if self.init is not None: + yield self.init + + attr_names = () + + +class Constant(Node): + __slots__ = ("type", "value", "coord", "__weakref__") + + def __init__(self, type, value, coord=None): + self.type = type + self.value = value + self.coord = coord + + def children(self): + nodelist = [] + return tuple(nodelist) + + def __iter__(self): + return + yield + + attr_names = ( + "type", + "value", + ) + + +class Continue(Node): + __slots__ = ("coord", "__weakref__") + + def __init__(self, coord=None): + self.coord = coord + + def children(self): + return () + + def __iter__(self): + return + yield + + attr_names = () + + +class Decl(Node): + __slots__ = ( + "name", + "quals", + "align", + "storage", + "funcspec", + "type", + "init", + "bitsize", + "coord", + "__weakref__", + ) + + def __init__( + self, name, quals, align, storage, funcspec, type, init, bitsize, coord=None + ): + self.name = name + self.quals = quals + self.align = align + self.storage = storage + self.funcspec = funcspec + self.type = type + self.init = init + self.bitsize = bitsize + self.coord = coord + + def children(self): + nodelist = [] + if self.type is not None: + nodelist.append(("type", self.type)) + if self.init is not None: + nodelist.append(("init", self.init)) + if self.bitsize is not None: + nodelist.append(("bitsize", self.bitsize)) + return tuple(nodelist) + + def __iter__(self): + if self.type is not None: + yield self.type + if self.init is not None: + yield self.init + if self.bitsize is not None: + yield self.bitsize + + attr_names = ( + "name", + "quals", + "align", + "storage", + "funcspec", + ) + + +class DeclList(Node): + __slots__ = ("decls", "coord", "__weakref__") + + def __init__(self, decls, coord=None): + self.decls = decls + self.coord = coord + + def children(self): + nodelist = [] + for i, child in enumerate(self.decls or []): + nodelist.append((f"decls[{i}]", child)) + return tuple(nodelist) + + def __iter__(self): + for child in self.decls or []: + yield child + + attr_names = () + + +class Default(Node): + __slots__ = ("stmts", "coord", "__weakref__") + + def __init__(self, stmts, coord=None): + self.stmts = stmts + self.coord = coord + + def children(self): + nodelist = [] + for i, child in enumerate(self.stmts or []): + nodelist.append((f"stmts[{i}]", child)) + return tuple(nodelist) + + def __iter__(self): + for child in self.stmts or []: + yield child + + attr_names = () + + +class DoWhile(Node): + __slots__ = ("cond", "stmt", "coord", "__weakref__") + + def __init__(self, cond, stmt, coord=None): + self.cond = cond + self.stmt = stmt + self.coord = coord + + def children(self): + nodelist = [] + if self.cond is not None: + nodelist.append(("cond", self.cond)) + if self.stmt is not None: + nodelist.append(("stmt", self.stmt)) + return tuple(nodelist) + + def __iter__(self): + if self.cond is not None: + yield self.cond + if self.stmt is not None: + yield self.stmt + + attr_names = () + + +class EllipsisParam(Node): + __slots__ = ("coord", "__weakref__") + + def __init__(self, coord=None): + self.coord = coord + + def children(self): + return () + + def __iter__(self): + return + yield + + attr_names = () + + +class EmptyStatement(Node): + __slots__ = ("coord", "__weakref__") + + def __init__(self, coord=None): + self.coord = coord + + def children(self): + return () + + def __iter__(self): + return + yield + + attr_names = () + + +class Enum(Node): + __slots__ = ("name", "values", "coord", "__weakref__") + + def __init__(self, name, values, coord=None): + self.name = name + self.values = values + self.coord = coord + + def children(self): + nodelist = [] + if self.values is not None: + nodelist.append(("values", self.values)) + return tuple(nodelist) + + def __iter__(self): + if self.values is not None: + yield self.values + + attr_names = ("name",) + + +class Enumerator(Node): + __slots__ = ("name", "value", "coord", "__weakref__") + + def __init__(self, name, value, coord=None): + self.name = name + self.value = value + self.coord = coord + + def children(self): + nodelist = [] + if self.value is not None: + nodelist.append(("value", self.value)) + return tuple(nodelist) + + def __iter__(self): + if self.value is not None: + yield self.value + + attr_names = ("name",) + + +class EnumeratorList(Node): + __slots__ = ("enumerators", "coord", "__weakref__") + + def __init__(self, enumerators, coord=None): + self.enumerators = enumerators + self.coord = coord + + def children(self): + nodelist = [] + for i, child in enumerate(self.enumerators or []): + nodelist.append((f"enumerators[{i}]", child)) + return tuple(nodelist) + + def __iter__(self): + for child in self.enumerators or []: + yield child + + attr_names = () + + +class ExprList(Node): + __slots__ = ("exprs", "coord", "__weakref__") + + def __init__(self, exprs, coord=None): + self.exprs = exprs + self.coord = coord + + def children(self): + nodelist = [] + for i, child in enumerate(self.exprs or []): + nodelist.append((f"exprs[{i}]", child)) + return tuple(nodelist) + + def __iter__(self): + for child in self.exprs or []: + yield child + + attr_names = () + + +class FileAST(Node): + __slots__ = ("ext", "coord", "__weakref__") + + def __init__(self, ext, coord=None): + self.ext = ext + self.coord = coord + + def children(self): + nodelist = [] + for i, child in enumerate(self.ext or []): + nodelist.append((f"ext[{i}]", child)) + return tuple(nodelist) + + def __iter__(self): + for child in self.ext or []: + yield child + + attr_names = () + + +class For(Node): + __slots__ = ("init", "cond", "next", "stmt", "coord", "__weakref__") + + def __init__(self, init, cond, next, stmt, coord=None): + self.init = init + self.cond = cond + self.next = next + self.stmt = stmt + self.coord = coord + + def children(self): + nodelist = [] + if self.init is not None: + nodelist.append(("init", self.init)) + if self.cond is not None: + nodelist.append(("cond", self.cond)) + if self.next is not None: + nodelist.append(("next", self.next)) + if self.stmt is not None: + nodelist.append(("stmt", self.stmt)) + return tuple(nodelist) + + def __iter__(self): + if self.init is not None: + yield self.init + if self.cond is not None: + yield self.cond + if self.next is not None: + yield self.next + if self.stmt is not None: + yield self.stmt + + attr_names = () + + +class FuncCall(Node): + __slots__ = ("name", "args", "coord", "__weakref__") + + def __init__(self, name, args, coord=None): + self.name = name + self.args = args + self.coord = coord + + def children(self): + nodelist = [] + if self.name is not None: + nodelist.append(("name", self.name)) + if self.args is not None: + nodelist.append(("args", self.args)) + return tuple(nodelist) + + def __iter__(self): + if self.name is not None: + yield self.name + if self.args is not None: + yield self.args + + attr_names = () + + +class FuncDecl(Node): + __slots__ = ("args", "type", "coord", "__weakref__") + + def __init__(self, args, type, coord=None): + self.args = args + self.type = type + self.coord = coord + + def children(self): + nodelist = [] + if self.args is not None: + nodelist.append(("args", self.args)) + if self.type is not None: + nodelist.append(("type", self.type)) + return tuple(nodelist) + + def __iter__(self): + if self.args is not None: + yield self.args + if self.type is not None: + yield self.type + + attr_names = () + + +class FuncDef(Node): + __slots__ = ("decl", "param_decls", "body", "coord", "__weakref__") + + def __init__(self, decl, param_decls, body, coord=None): + self.decl = decl + self.param_decls = param_decls + self.body = body + self.coord = coord + + def children(self): + nodelist = [] + if self.decl is not None: + nodelist.append(("decl", self.decl)) + if self.body is not None: + nodelist.append(("body", self.body)) + for i, child in enumerate(self.param_decls or []): + nodelist.append((f"param_decls[{i}]", child)) + return tuple(nodelist) + + def __iter__(self): + if self.decl is not None: + yield self.decl + if self.body is not None: + yield self.body + for child in self.param_decls or []: + yield child + + attr_names = () + + +class Goto(Node): + __slots__ = ("name", "coord", "__weakref__") + + def __init__(self, name, coord=None): + self.name = name + self.coord = coord + + def children(self): + nodelist = [] + return tuple(nodelist) + + def __iter__(self): + return + yield + + attr_names = ("name",) + + +class ID(Node): + __slots__ = ("name", "coord", "__weakref__") + + def __init__(self, name, coord=None): + self.name = name + self.coord = coord + + def children(self): + nodelist = [] + return tuple(nodelist) + + def __iter__(self): + return + yield + + attr_names = ("name",) + + +class IdentifierType(Node): + __slots__ = ("names", "coord", "__weakref__") + + def __init__(self, names, coord=None): + self.names = names + self.coord = coord + + def children(self): + nodelist = [] + return tuple(nodelist) + + def __iter__(self): + return + yield + + attr_names = ("names",) + + +class If(Node): + __slots__ = ("cond", "iftrue", "iffalse", "coord", "__weakref__") + + def __init__(self, cond, iftrue, iffalse, coord=None): + self.cond = cond + self.iftrue = iftrue + self.iffalse = iffalse + self.coord = coord + + def children(self): + nodelist = [] + if self.cond is not None: + nodelist.append(("cond", self.cond)) + if self.iftrue is not None: + nodelist.append(("iftrue", self.iftrue)) + if self.iffalse is not None: + nodelist.append(("iffalse", self.iffalse)) + return tuple(nodelist) + + def __iter__(self): + if self.cond is not None: + yield self.cond + if self.iftrue is not None: + yield self.iftrue + if self.iffalse is not None: + yield self.iffalse + + attr_names = () + + +class InitList(Node): + __slots__ = ("exprs", "coord", "__weakref__") + + def __init__(self, exprs, coord=None): + self.exprs = exprs + self.coord = coord + + def children(self): + nodelist = [] + for i, child in enumerate(self.exprs or []): + nodelist.append((f"exprs[{i}]", child)) + return tuple(nodelist) + + def __iter__(self): + for child in self.exprs or []: + yield child + + attr_names = () + + +class Label(Node): + __slots__ = ("name", "stmt", "coord", "__weakref__") + + def __init__(self, name, stmt, coord=None): + self.name = name + self.stmt = stmt + self.coord = coord + + def children(self): + nodelist = [] + if self.stmt is not None: + nodelist.append(("stmt", self.stmt)) + return tuple(nodelist) + + def __iter__(self): + if self.stmt is not None: + yield self.stmt + + attr_names = ("name",) + + +class NamedInitializer(Node): + __slots__ = ("name", "expr", "coord", "__weakref__") + + def __init__(self, name, expr, coord=None): + self.name = name + self.expr = expr + self.coord = coord + + def children(self): + nodelist = [] + if self.expr is not None: + nodelist.append(("expr", self.expr)) + for i, child in enumerate(self.name or []): + nodelist.append((f"name[{i}]", child)) + return tuple(nodelist) + + def __iter__(self): + if self.expr is not None: + yield self.expr + for child in self.name or []: + yield child + + attr_names = () + + +class ParamList(Node): + __slots__ = ("params", "coord", "__weakref__") + + def __init__(self, params, coord=None): + self.params = params + self.coord = coord + + def children(self): + nodelist = [] + for i, child in enumerate(self.params or []): + nodelist.append((f"params[{i}]", child)) + return tuple(nodelist) + + def __iter__(self): + for child in self.params or []: + yield child + + attr_names = () + + +class PtrDecl(Node): + __slots__ = ("quals", "type", "coord", "__weakref__") + + def __init__(self, quals, type, coord=None): + self.quals = quals + self.type = type + self.coord = coord + + def children(self): + nodelist = [] + if self.type is not None: + nodelist.append(("type", self.type)) + return tuple(nodelist) + + def __iter__(self): + if self.type is not None: + yield self.type + + attr_names = ("quals",) + + +class Return(Node): + __slots__ = ("expr", "coord", "__weakref__") + + def __init__(self, expr, coord=None): + self.expr = expr + self.coord = coord + + def children(self): + nodelist = [] + if self.expr is not None: + nodelist.append(("expr", self.expr)) + return tuple(nodelist) + + def __iter__(self): + if self.expr is not None: + yield self.expr + + attr_names = () + + +class StaticAssert(Node): + __slots__ = ("cond", "message", "coord", "__weakref__") + + def __init__(self, cond, message, coord=None): + self.cond = cond + self.message = message + self.coord = coord + + def children(self): + nodelist = [] + if self.cond is not None: + nodelist.append(("cond", self.cond)) + if self.message is not None: + nodelist.append(("message", self.message)) + return tuple(nodelist) + + def __iter__(self): + if self.cond is not None: + yield self.cond + if self.message is not None: + yield self.message + + attr_names = () + + +class Struct(Node): + __slots__ = ("name", "decls", "coord", "__weakref__") + + def __init__(self, name, decls, coord=None): + self.name = name + self.decls = decls + self.coord = coord + + def children(self): + nodelist = [] + for i, child in enumerate(self.decls or []): + nodelist.append((f"decls[{i}]", child)) + return tuple(nodelist) + + def __iter__(self): + for child in self.decls or []: + yield child + + attr_names = ("name",) + + +class StructRef(Node): + __slots__ = ("name", "type", "field", "coord", "__weakref__") + + def __init__(self, name, type, field, coord=None): + self.name = name + self.type = type + self.field = field + self.coord = coord + + def children(self): + nodelist = [] + if self.name is not None: + nodelist.append(("name", self.name)) + if self.field is not None: + nodelist.append(("field", self.field)) + return tuple(nodelist) + + def __iter__(self): + if self.name is not None: + yield self.name + if self.field is not None: + yield self.field + + attr_names = ("type",) + + +class Switch(Node): + __slots__ = ("cond", "stmt", "coord", "__weakref__") + + def __init__(self, cond, stmt, coord=None): + self.cond = cond + self.stmt = stmt + self.coord = coord + + def children(self): + nodelist = [] + if self.cond is not None: + nodelist.append(("cond", self.cond)) + if self.stmt is not None: + nodelist.append(("stmt", self.stmt)) + return tuple(nodelist) + + def __iter__(self): + if self.cond is not None: + yield self.cond + if self.stmt is not None: + yield self.stmt + + attr_names = () + + +class TernaryOp(Node): + __slots__ = ("cond", "iftrue", "iffalse", "coord", "__weakref__") + + def __init__(self, cond, iftrue, iffalse, coord=None): + self.cond = cond + self.iftrue = iftrue + self.iffalse = iffalse + self.coord = coord + + def children(self): + nodelist = [] + if self.cond is not None: + nodelist.append(("cond", self.cond)) + if self.iftrue is not None: + nodelist.append(("iftrue", self.iftrue)) + if self.iffalse is not None: + nodelist.append(("iffalse", self.iffalse)) + return tuple(nodelist) + + def __iter__(self): + if self.cond is not None: + yield self.cond + if self.iftrue is not None: + yield self.iftrue + if self.iffalse is not None: + yield self.iffalse + + attr_names = () + + +class TypeDecl(Node): + __slots__ = ("declname", "quals", "align", "type", "coord", "__weakref__") + + def __init__(self, declname, quals, align, type, coord=None): + self.declname = declname + self.quals = quals + self.align = align + self.type = type + self.coord = coord + + def children(self): + nodelist = [] + if self.type is not None: + nodelist.append(("type", self.type)) + return tuple(nodelist) + + def __iter__(self): + if self.type is not None: + yield self.type + + attr_names = ( + "declname", + "quals", + "align", + ) + + +class Typedef(Node): + __slots__ = ("name", "quals", "storage", "type", "coord", "__weakref__") + + def __init__(self, name, quals, storage, type, coord=None): + self.name = name + self.quals = quals + self.storage = storage + self.type = type + self.coord = coord + + def children(self): + nodelist = [] + if self.type is not None: + nodelist.append(("type", self.type)) + return tuple(nodelist) + + def __iter__(self): + if self.type is not None: + yield self.type + + attr_names = ( + "name", + "quals", + "storage", + ) + + +class Typename(Node): + __slots__ = ("name", "quals", "align", "type", "coord", "__weakref__") + + def __init__(self, name, quals, align, type, coord=None): + self.name = name + self.quals = quals + self.align = align + self.type = type + self.coord = coord + + def children(self): + nodelist = [] + if self.type is not None: + nodelist.append(("type", self.type)) + return tuple(nodelist) + + def __iter__(self): + if self.type is not None: + yield self.type + + attr_names = ( + "name", + "quals", + "align", + ) + + +class UnaryOp(Node): + __slots__ = ("op", "expr", "coord", "__weakref__") + + def __init__(self, op, expr, coord=None): + self.op = op + self.expr = expr + self.coord = coord + + def children(self): + nodelist = [] + if self.expr is not None: + nodelist.append(("expr", self.expr)) + return tuple(nodelist) + + def __iter__(self): + if self.expr is not None: + yield self.expr + + attr_names = ("op",) + + +class Union(Node): + __slots__ = ("name", "decls", "coord", "__weakref__") + + def __init__(self, name, decls, coord=None): + self.name = name + self.decls = decls + self.coord = coord + + def children(self): + nodelist = [] + for i, child in enumerate(self.decls or []): + nodelist.append((f"decls[{i}]", child)) + return tuple(nodelist) + + def __iter__(self): + for child in self.decls or []: + yield child + + attr_names = ("name",) + + +class While(Node): + __slots__ = ("cond", "stmt", "coord", "__weakref__") + + def __init__(self, cond, stmt, coord=None): + self.cond = cond + self.stmt = stmt + self.coord = coord + + def children(self): + nodelist = [] + if self.cond is not None: + nodelist.append(("cond", self.cond)) + if self.stmt is not None: + nodelist.append(("stmt", self.stmt)) + return tuple(nodelist) + + def __iter__(self): + if self.cond is not None: + yield self.cond + if self.stmt is not None: + yield self.stmt + + attr_names = () + + +class Pragma(Node): + __slots__ = ("string", "coord", "__weakref__") + + def __init__(self, string, coord=None): + self.string = string + self.coord = coord + + def children(self): + nodelist = [] + return tuple(nodelist) + + def __iter__(self): + return + yield + + attr_names = ("string",) diff --git a/py_modules/pycparser/c_generator.py b/py_modules/pycparser/c_generator.py new file mode 100644 index 0000000..424e00e --- /dev/null +++ b/py_modules/pycparser/c_generator.py @@ -0,0 +1,573 @@ +# ------------------------------------------------------------------------------ +# pycparser: c_generator.py +# +# C code generator from pycparser AST nodes. +# +# Eli Bendersky [https://eli.thegreenplace.net/] +# License: BSD +# ------------------------------------------------------------------------------ +from typing import Callable, List, Optional + +from . import c_ast + + +class CGenerator: + """Uses the same visitor pattern as c_ast.NodeVisitor, but modified to + return a value from each visit method, using string accumulation in + generic_visit. + """ + + indent_level: int + reduce_parentheses: bool + + def __init__(self, reduce_parentheses: bool = False) -> None: + """Constructs C-code generator + + reduce_parentheses: + if True, eliminates needless parentheses on binary operators + """ + # Statements start with indentation of self.indent_level spaces, using + # the _make_indent method. + self.indent_level = 0 + self.reduce_parentheses = reduce_parentheses + + def _make_indent(self) -> str: + return " " * self.indent_level + + def visit(self, node: c_ast.Node) -> str: + method = "visit_" + node.__class__.__name__ + return getattr(self, method, self.generic_visit)(node) + + def generic_visit(self, node: Optional[c_ast.Node]) -> str: + if node is None: + return "" + else: + return "".join(self.visit(c) for c_name, c in node.children()) + + def visit_Constant(self, n: c_ast.Constant) -> str: + return n.value + + def visit_ID(self, n: c_ast.ID) -> str: + return n.name + + def visit_Pragma(self, n: c_ast.Pragma) -> str: + ret = "#pragma" + if n.string: + ret += " " + n.string + return ret + + def visit_ArrayRef(self, n: c_ast.ArrayRef) -> str: + arrref = self._parenthesize_unless_simple(n.name) + return arrref + "[" + self.visit(n.subscript) + "]" + + def visit_StructRef(self, n: c_ast.StructRef) -> str: + sref = self._parenthesize_unless_simple(n.name) + return sref + n.type + self.visit(n.field) + + def visit_FuncCall(self, n: c_ast.FuncCall) -> str: + fref = self._parenthesize_unless_simple(n.name) + args = self.visit(n.args) if n.args is not None else "" + return fref + "(" + args + ")" + + def visit_UnaryOp(self, n: c_ast.UnaryOp) -> str: + match n.op: + case "sizeof": + # Always parenthesize the argument of sizeof since it can be + # a name. + return f"sizeof({self.visit(n.expr)})" + case "p++": + operand = self._parenthesize_unless_simple(n.expr) + return f"{operand}++" + case "p--": + operand = self._parenthesize_unless_simple(n.expr) + return f"{operand}--" + case _: + operand = self._parenthesize_unless_simple(n.expr) + return f"{n.op}{operand}" + + # Precedence map of binary operators: + precedence_map = { + # Should be in sync with c_parser.CParser.precedence + # Higher numbers are stronger binding + "||": 0, # weakest binding + "&&": 1, + "|": 2, + "^": 3, + "&": 4, + "==": 5, + "!=": 5, + ">": 6, + ">=": 6, + "<": 6, + "<=": 6, + ">>": 7, + "<<": 7, + "+": 8, + "-": 8, + "*": 9, + "/": 9, + "%": 9, # strongest binding + } + + def visit_BinaryOp(self, n: c_ast.BinaryOp) -> str: + # Note: all binary operators are left-to-right associative + # + # If `n.left.op` has a stronger or equally binding precedence in + # comparison to `n.op`, no parenthesis are needed for the left: + # e.g., `(a*b) + c` is equivalent to `a*b + c`, as well as + # `(a+b) - c` is equivalent to `a+b - c` (same precedence). + # If the left operator is weaker binding than the current, then + # parentheses are necessary: + # e.g., `(a+b) * c` is NOT equivalent to `a+b * c`. + lval_str = self._parenthesize_if( + n.left, + lambda d: not ( + self._is_simple_node(d) + or self.reduce_parentheses + and isinstance(d, c_ast.BinaryOp) + and self.precedence_map[d.op] >= self.precedence_map[n.op] + ), + ) + # If `n.right.op` has a stronger -but not equal- binding precedence, + # parenthesis can be omitted on the right: + # e.g., `a + (b*c)` is equivalent to `a + b*c`. + # If the right operator is weaker or equally binding, then parentheses + # are necessary: + # e.g., `a * (b+c)` is NOT equivalent to `a * b+c` and + # `a - (b+c)` is NOT equivalent to `a - b+c` (same precedence). + rval_str = self._parenthesize_if( + n.right, + lambda d: not ( + self._is_simple_node(d) + or self.reduce_parentheses + and isinstance(d, c_ast.BinaryOp) + and self.precedence_map[d.op] > self.precedence_map[n.op] + ), + ) + return f"{lval_str} {n.op} {rval_str}" + + def visit_Assignment(self, n: c_ast.Assignment) -> str: + rval_str = self._parenthesize_if( + n.rvalue, lambda n: isinstance(n, c_ast.Assignment) + ) + return f"{self.visit(n.lvalue)} {n.op} {rval_str}" + + def visit_IdentifierType(self, n: c_ast.IdentifierType) -> str: + return " ".join(n.names) + + def _visit_expr(self, n: c_ast.Node) -> str: + match n: + case c_ast.InitList(): + return "{" + self.visit(n) + "}" + case c_ast.ExprList() | c_ast.Compound(): + return "(" + self.visit(n) + ")" + case _: + return self.visit(n) + + def visit_Decl(self, n: c_ast.Decl, no_type: bool = False) -> str: + # no_type is used when a Decl is part of a DeclList, where the type is + # explicitly only for the first declaration in a list. + # + s = n.name if no_type else self._generate_decl(n) + if n.bitsize: + s += " : " + self.visit(n.bitsize) + if n.init: + s += " = " + self._visit_expr(n.init) + return s + + def visit_DeclList(self, n: c_ast.DeclList) -> str: + s = self.visit(n.decls[0]) + if len(n.decls) > 1: + s += ", " + ", ".join( + self.visit_Decl(decl, no_type=True) for decl in n.decls[1:] + ) + return s + + def visit_Typedef(self, n: c_ast.Typedef) -> str: + s = "" + if n.storage: + s += " ".join(n.storage) + " " + s += self._generate_type(n.type) + return s + + def visit_Cast(self, n: c_ast.Cast) -> str: + s = "(" + self._generate_type(n.to_type, emit_declname=False) + ")" + return s + " " + self._parenthesize_unless_simple(n.expr) + + def visit_ExprList(self, n: c_ast.ExprList) -> str: + visited_subexprs = [] + for expr in n.exprs: + visited_subexprs.append(self._visit_expr(expr)) + return ", ".join(visited_subexprs) + + def visit_InitList(self, n: c_ast.InitList) -> str: + visited_subexprs = [] + for expr in n.exprs: + visited_subexprs.append(self._visit_expr(expr)) + return ", ".join(visited_subexprs) + + def visit_Enum(self, n: c_ast.Enum) -> str: + return self._generate_struct_union_enum(n, name="enum") + + def visit_Alignas(self, n: c_ast.Alignas) -> str: + return "_Alignas({})".format(self.visit(n.alignment)) + + def visit_Enumerator(self, n: c_ast.Enumerator) -> str: + if not n.value: + return "{indent}{name},\n".format( + indent=self._make_indent(), + name=n.name, + ) + else: + return "{indent}{name} = {value},\n".format( + indent=self._make_indent(), + name=n.name, + value=self.visit(n.value), + ) + + def visit_FuncDef(self, n: c_ast.FuncDef) -> str: + decl = self.visit(n.decl) + self.indent_level = 0 + body = self.visit(n.body) + if n.param_decls: + knrdecls = ";\n".join(self.visit(p) for p in n.param_decls) + return decl + "\n" + knrdecls + ";\n" + body + "\n" + else: + return decl + "\n" + body + "\n" + + def visit_FileAST(self, n: c_ast.FileAST) -> str: + s = "" + for ext in n.ext: + match ext: + case c_ast.FuncDef(): + s += self.visit(ext) + case c_ast.Pragma(): + s += self.visit(ext) + "\n" + case _: + s += self.visit(ext) + ";\n" + return s + + def visit_Compound(self, n: c_ast.Compound) -> str: + s = self._make_indent() + "{\n" + self.indent_level += 2 + if n.block_items: + s += "".join(self._generate_stmt(stmt) for stmt in n.block_items) + self.indent_level -= 2 + s += self._make_indent() + "}\n" + return s + + def visit_CompoundLiteral(self, n: c_ast.CompoundLiteral) -> str: + return "(" + self.visit(n.type) + "){" + self.visit(n.init) + "}" + + def visit_EmptyStatement(self, n: c_ast.EmptyStatement) -> str: + return ";" + + def visit_ParamList(self, n: c_ast.ParamList) -> str: + return ", ".join(self.visit(param) for param in n.params) + + def visit_Return(self, n: c_ast.Return) -> str: + s = "return" + if n.expr: + s += " " + self.visit(n.expr) + return s + ";" + + def visit_Break(self, n: c_ast.Break) -> str: + return "break;" + + def visit_Continue(self, n: c_ast.Continue) -> str: + return "continue;" + + def visit_TernaryOp(self, n: c_ast.TernaryOp) -> str: + s = "(" + self._visit_expr(n.cond) + ") ? " + s += "(" + self._visit_expr(n.iftrue) + ") : " + s += "(" + self._visit_expr(n.iffalse) + ")" + return s + + def visit_If(self, n: c_ast.If) -> str: + s = "if (" + if n.cond: + s += self.visit(n.cond) + s += ")\n" + s += self._generate_stmt(n.iftrue, add_indent=True) + if n.iffalse: + s += self._make_indent() + "else\n" + s += self._generate_stmt(n.iffalse, add_indent=True) + return s + + def visit_For(self, n: c_ast.For) -> str: + s = "for (" + if n.init: + s += self.visit(n.init) + s += ";" + if n.cond: + s += " " + self.visit(n.cond) + s += ";" + if n.next: + s += " " + self.visit(n.next) + s += ")\n" + s += self._generate_stmt(n.stmt, add_indent=True) + return s + + def visit_While(self, n: c_ast.While) -> str: + s = "while (" + if n.cond: + s += self.visit(n.cond) + s += ")\n" + s += self._generate_stmt(n.stmt, add_indent=True) + return s + + def visit_DoWhile(self, n: c_ast.DoWhile) -> str: + s = "do\n" + s += self._generate_stmt(n.stmt, add_indent=True) + s += self._make_indent() + "while (" + if n.cond: + s += self.visit(n.cond) + s += ");" + return s + + def visit_StaticAssert(self, n: c_ast.StaticAssert) -> str: + s = "_Static_assert(" + s += self.visit(n.cond) + if n.message: + s += "," + s += self.visit(n.message) + s += ")" + return s + + def visit_Switch(self, n: c_ast.Switch) -> str: + s = "switch (" + self.visit(n.cond) + ")\n" + s += self._generate_stmt(n.stmt, add_indent=True) + return s + + def visit_Case(self, n: c_ast.Case) -> str: + s = "case " + self.visit(n.expr) + ":\n" + for stmt in n.stmts: + s += self._generate_stmt(stmt, add_indent=True) + return s + + def visit_Default(self, n: c_ast.Default) -> str: + s = "default:\n" + for stmt in n.stmts: + s += self._generate_stmt(stmt, add_indent=True) + return s + + def visit_Label(self, n: c_ast.Label) -> str: + return n.name + ":\n" + self._generate_stmt(n.stmt) + + def visit_Goto(self, n: c_ast.Goto) -> str: + return "goto " + n.name + ";" + + def visit_EllipsisParam(self, n: c_ast.EllipsisParam) -> str: + return "..." + + def visit_Struct(self, n: c_ast.Struct) -> str: + return self._generate_struct_union_enum(n, "struct") + + def visit_Typename(self, n: c_ast.Typename) -> str: + return self._generate_type(n.type) + + def visit_Union(self, n: c_ast.Union) -> str: + return self._generate_struct_union_enum(n, "union") + + def visit_NamedInitializer(self, n: c_ast.NamedInitializer) -> str: + s = "" + for name in n.name: + if isinstance(name, c_ast.ID): + s += "." + name.name + else: + s += "[" + self.visit(name) + "]" + s += " = " + self._visit_expr(n.expr) + return s + + def visit_FuncDecl(self, n: c_ast.FuncDecl) -> str: + return self._generate_type(n) + + def visit_ArrayDecl(self, n: c_ast.ArrayDecl) -> str: + return self._generate_type(n, emit_declname=False) + + def visit_TypeDecl(self, n: c_ast.TypeDecl) -> str: + return self._generate_type(n, emit_declname=False) + + def visit_PtrDecl(self, n: c_ast.PtrDecl) -> str: + return self._generate_type(n, emit_declname=False) + + def _generate_struct_union_enum( + self, n: c_ast.Struct | c_ast.Union | c_ast.Enum, name: str + ) -> str: + """Generates code for structs, unions, and enums. name should be + 'struct', 'union', or 'enum'. + """ + if name in ("struct", "union"): + assert isinstance(n, (c_ast.Struct, c_ast.Union)) + members = n.decls + body_function = self._generate_struct_union_body + else: + assert name == "enum" + assert isinstance(n, c_ast.Enum) + members = None if n.values is None else n.values.enumerators + body_function = self._generate_enum_body + s = name + " " + (n.name or "") + if members is not None: + # None means no members + # Empty sequence means an empty list of members + s += "\n" + s += self._make_indent() + self.indent_level += 2 + s += "{\n" + s += body_function(members) + self.indent_level -= 2 + s += self._make_indent() + "}" + return s + + def _generate_struct_union_body(self, members: List[c_ast.Node]) -> str: + return "".join(self._generate_stmt(decl) for decl in members) + + def _generate_enum_body(self, members: List[c_ast.Enumerator]) -> str: + # `[:-2] + '\n'` removes the final `,` from the enumerator list + return "".join(self.visit(value) for value in members)[:-2] + "\n" + + def _generate_stmt(self, n: c_ast.Node, add_indent: bool = False) -> str: + """Generation from a statement node. This method exists as a wrapper + for individual visit_* methods to handle different treatment of + some statements in this context. + """ + if add_indent: + self.indent_level += 2 + indent = self._make_indent() + if add_indent: + self.indent_level -= 2 + + match n: + case ( + c_ast.Decl() + | c_ast.Assignment() + | c_ast.Cast() + | c_ast.UnaryOp() + | c_ast.BinaryOp() + | c_ast.TernaryOp() + | c_ast.FuncCall() + | c_ast.ArrayRef() + | c_ast.StructRef() + | c_ast.Constant() + | c_ast.ID() + | c_ast.Typedef() + | c_ast.ExprList() + ): + # These can also appear in an expression context so no semicolon + # is added to them automatically + # + return indent + self.visit(n) + ";\n" + case c_ast.Compound(): + # No extra indentation required before the opening brace of a + # compound - because it consists of multiple lines it has to + # compute its own indentation. + # + return self.visit(n) + case c_ast.If(): + return indent + self.visit(n) + case _: + return indent + self.visit(n) + "\n" + + def _generate_decl(self, n: c_ast.Decl) -> str: + """Generation from a Decl node.""" + s = "" + if n.funcspec: + s = " ".join(n.funcspec) + " " + if n.storage: + s += " ".join(n.storage) + " " + if n.align: + s += self.visit(n.align[0]) + " " + s += self._generate_type(n.type) + return s + + def _generate_type( + self, + n: c_ast.Node, + modifiers: List[c_ast.Node] = [], + emit_declname: bool = True, + ) -> str: + """Recursive generation from a type node. n is the type node. + modifiers collects the PtrDecl, ArrayDecl and FuncDecl modifiers + encountered on the way down to a TypeDecl, to allow proper + generation from it. + """ + # ~ print(n, modifiers) + match n: + case c_ast.TypeDecl(): + s = "" + if n.quals: + s += " ".join(n.quals) + " " + s += self.visit(n.type) + + nstr = n.declname if n.declname and emit_declname else "" + # Resolve modifiers. + # Wrap in parens to distinguish pointer to array and pointer to + # function syntax. + # + for i, modifier in enumerate(modifiers): + match modifier: + case c_ast.ArrayDecl(): + if i != 0 and isinstance(modifiers[i - 1], c_ast.PtrDecl): + nstr = "(" + nstr + ")" + nstr += "[" + if modifier.dim_quals: + nstr += " ".join(modifier.dim_quals) + " " + if modifier.dim is not None: + nstr += self.visit(modifier.dim) + nstr += "]" + case c_ast.FuncDecl(): + if i != 0 and isinstance(modifiers[i - 1], c_ast.PtrDecl): + nstr = "(" + nstr + ")" + args = ( + self.visit(modifier.args) + if modifier.args is not None + else "" + ) + nstr += "(" + args + ")" + case c_ast.PtrDecl(): + if modifier.quals: + quals = " ".join(modifier.quals) + suffix = f" {nstr}" if nstr else "" + nstr = f"* {quals}{suffix}" + else: + nstr = "*" + nstr + if nstr: + s += " " + nstr + return s + case c_ast.Decl(): + return self._generate_decl(n.type) + case c_ast.Typename(): + return self._generate_type(n.type, emit_declname=emit_declname) + case c_ast.IdentifierType(): + return " ".join(n.names) + " " + case c_ast.ArrayDecl() | c_ast.PtrDecl() | c_ast.FuncDecl(): + return self._generate_type( + n.type, modifiers + [n], emit_declname=emit_declname + ) + case _: + return self.visit(n) + + def _parenthesize_if( + self, n: c_ast.Node, condition: Callable[[c_ast.Node], bool] + ) -> str: + """Visits 'n' and returns its string representation, parenthesized + if the condition function applied to the node returns True. + """ + s = self._visit_expr(n) + if condition(n): + return "(" + s + ")" + else: + return s + + def _parenthesize_unless_simple(self, n: c_ast.Node) -> str: + """Common use case for _parenthesize_if""" + return self._parenthesize_if(n, lambda d: not self._is_simple_node(d)) + + def _is_simple_node(self, n: c_ast.Node) -> bool: + """Returns True for nodes that are "simple" - i.e. nodes that always + have higher precedence than operators. + """ + return isinstance( + n, + (c_ast.Constant, c_ast.ID, c_ast.ArrayRef, c_ast.StructRef, c_ast.FuncCall), + ) diff --git a/py_modules/pycparser/c_lexer.py b/py_modules/pycparser/c_lexer.py new file mode 100644 index 0000000..ef59d69 --- /dev/null +++ b/py_modules/pycparser/c_lexer.py @@ -0,0 +1,706 @@ +# ------------------------------------------------------------------------------ +# pycparser: c_lexer.py +# +# CLexer class: lexer for the C language +# +# Eli Bendersky [https://eli.thegreenplace.net/] +# License: BSD +# ------------------------------------------------------------------------------ +import re +from dataclasses import dataclass +from enum import Enum +from typing import Callable, Dict, List, Optional, Tuple + + +@dataclass(slots=True) +class _Token: + type: str + value: str + lineno: int + column: int + + +class CLexer: + """A standalone lexer for C. + + Parameters for construction: + error_func: + Called with (msg, line, column) on lexing errors. + on_lbrace_func: + Called when an LBRACE token is produced (used for scope tracking). + on_rbrace_func: + Called when an RBRACE token is produced (used for scope tracking). + type_lookup_func: + Called with an identifier name; expected to return True if it is + a typedef name and should be tokenized as TYPEID. + + Call input(text) to initialize lexing, and then keep calling token() to + get the next token, until it returns None (at end of input). + """ + + def __init__( + self, + error_func: Callable[[str, int, int], None], + on_lbrace_func: Callable[[], None], + on_rbrace_func: Callable[[], None], + type_lookup_func: Callable[[str], bool], + ) -> None: + self.error_func = error_func + self.on_lbrace_func = on_lbrace_func + self.on_rbrace_func = on_rbrace_func + self.type_lookup_func = type_lookup_func + self._init_state() + + def input(self, text: str, filename: str = "") -> None: + """Initialize the lexer to the given input text. + + filename is an optional name identifying the file from which the input + comes. The lexer can modify it if #line directives are encountered. + """ + self._init_state() + self._lexdata = text + self._filename = filename + + def _init_state(self) -> None: + self._lexdata = "" + self._filename = "" + self._pos = 0 + self._line_start = 0 + self._pending_tok: Optional[_Token] = None + self._lineno = 1 + + @property + def filename(self) -> str: + return self._filename + + def token(self) -> Optional[_Token]: + # Lexing strategy overview: + # + # - We maintain a current position (self._pos), line number, and the + # byte offset of the current line start. The lexer is a simple loop + # that skips whitespace/newlines and emits one token per call. + # - A small amount of logic is handled manually before regex matching: + # + # * Preprocessor-style directives: if we see '#', we check whether + # it's a #line or #pragma directive and consume it inline. #line + # updates lineno/filename and produces no tokens. #pragma can yield + # both PPPRAGMA and PPPRAGMASTR, but token() returns a single token, + # so we stash the PPPRAGMASTR as _pending_tok to return on the next + # token() call. Otherwise we return PPHASH. + # * Newlines update lineno/line-start tracking so tokens can record + # accurate columns. + # + # - The bulk of tokens are recognized in _match_token: + # + # * _regex_rules: regex patterns for identifiers, literals, and other + # complex tokens (including error-producing patterns). The lexer + # uses a combined _regex_master to scan options at the same time. + # * _fixed_tokens: exact string matches for operators and punctuation, + # resolved by longest match. + # + # - Error patterns call the error callback and advance minimally, which + # keeps lexing resilient while reporting useful diagnostics. + text = self._lexdata + n = len(text) + + if self._pending_tok is not None: + tok = self._pending_tok + self._pending_tok = None + return tok + + while self._pos < n: + match text[self._pos]: + case " " | "\t": + self._pos += 1 + case "\n": + self._lineno += 1 + self._pos += 1 + self._line_start = self._pos + case "#": + if _line_pattern.match(text, self._pos + 1): + self._pos += 1 + self._handle_ppline() + continue + if _pragma_pattern.match(text, self._pos + 1): + self._pos += 1 + toks = self._handle_pppragma() + if len(toks) > 1: + self._pending_tok = toks[1] + if len(toks) > 0: + return toks[0] + continue + tok = self._make_token("PPHASH", "#", self._pos) + self._pos += 1 + return tok + case _: + if tok := self._match_token(): + return tok + else: + continue + + def _match_token(self) -> Optional[_Token]: + """Match one token at the current position. + + Returns a Token on success, or None if no token could be matched and + an error was reported. This method always advances _pos by the matched + length, or by 1 on error/no-match. + """ + text = self._lexdata + pos = self._pos + # We pick the longest match between: + # - the master regex (identifiers, literals, error patterns, etc.) + # - fixed operator/punctuator literals from the bucket for text[pos] + # + # The longest match is required to ensure we properly lex something + # like ".123" (a floating-point constant) as a single entity (with + # FLOAT_CONST), rather than a PERIOD followed by a number. + # + # The fixed-literal buckets are already length-sorted, so within that + # bucket we can take the first match. However, we still compare its + # length to the regex match because the regex may have matched a longer + # token that should take precedence. + best = None + + if m := _regex_master.match(text, pos): + tok_type = m.lastgroup + # All master-regex alternatives are named; lastgroup shouldn't be None. + assert tok_type is not None + value = m.group(tok_type) + length = len(value) + action, msg = _regex_actions[tok_type] + best = (length, tok_type, value, action, msg) + + if bucket := _fixed_tokens_by_first.get(text[pos]): + for entry in bucket: + if text.startswith(entry.literal, pos): + length = len(entry.literal) + if best is None or length > best[0]: + best = ( + length, + entry.tok_type, + entry.literal, + _RegexAction.TOKEN, + None, + ) + break + + if best is None: + msg = f"Illegal character {repr(text[pos])}" + self._error(msg, pos) + self._pos += 1 + return None + + length, tok_type, value, action, msg = best + if action == _RegexAction.ERROR: + if tok_type == "BAD_CHAR_CONST": + msg = f"Invalid char constant {value}" + # All other ERROR rules provide a message. + assert msg is not None + self._error(msg, pos) + self._pos += max(1, length) + return None + + if action == _RegexAction.ID: + tok_type = _keyword_map.get(value, "ID") + if tok_type == "ID" and self.type_lookup_func(value): + tok_type = "TYPEID" + + tok = self._make_token(tok_type, value, pos) + self._pos += length + + if tok.type == "LBRACE": + self.on_lbrace_func() + elif tok.type == "RBRACE": + self.on_rbrace_func() + + return tok + + def _make_token(self, tok_type: str, value: str, pos: int) -> _Token: + """Create a Token at an absolute input position. + + Expects tok_type/value and the absolute byte offset pos in the current + input. Does not advance lexer state; callers manage _pos themselves. + Returns a Token with lineno/column computed from current line tracking. + """ + column = pos - self._line_start + 1 + tok = _Token(tok_type, value, self._lineno, column) + return tok + + def _error(self, msg: str, pos: int) -> None: + column = pos - self._line_start + 1 + self.error_func(msg, self._lineno, column) + + def _handle_ppline(self) -> None: + # Since #line directives aren't supposed to return tokens but should + # only affect the lexer's state (update line/filename for coords), this + # method does a bit of parsing on its own. It doesn't return anything, + # but its side effect is to update self._pos past the directive, and + # potentially update self._lineno and self._filename, based on the + # directive's contents. + # + # Accepted #line forms from preprocessors: + # - "#line 66 \"kwas\\df.h\"" + # - "# 9" + # - "#line 10 \"include/me.h\" 1 2 3" (extra numeric flags) + # - "# 1 \"file.h\" 3" + # Errors we must report: + # - "#line \"file.h\"" (filename before line number) + # - "#line df" (garbage instead of number/string) + # + # We scan the directive line once (after an optional 'line' keyword), + # validating the order: NUMBER, optional STRING, then any NUMBERs. + # The NUMBERs tail is only accepted if a filename STRING was present. + text = self._lexdata + n = len(text) + line_end = text.find("\n", self._pos) + if line_end == -1: + line_end = n + line = text[self._pos : line_end] + pos = 0 + line_len = len(line) + + def skip_ws() -> None: + nonlocal pos + while pos < line_len and line[pos] in " \t": + pos += 1 + + skip_ws() + if line.startswith("line", pos): + pos += 4 + + def success(pp_line: Optional[str], pp_filename: Optional[str]) -> None: + if pp_line is None: + self._error("line number missing in #line", self._pos + line_len) + else: + self._lineno = int(pp_line) + if pp_filename is not None: + self._filename = pp_filename + self._pos = line_end + 1 + self._line_start = self._pos + + def fail(msg: str, offset: int) -> None: + self._error(msg, self._pos + offset) + self._pos = line_end + 1 + self._line_start = self._pos + + skip_ws() + if pos >= line_len: + success(None, None) + return + if line[pos] == '"': + fail("filename before line number in #line", pos) + return + + m = re.match(_decimal_constant, line[pos:]) + if not m: + fail("invalid #line directive", pos) + return + + pp_line = m.group(0) + pos += len(pp_line) + skip_ws() + if pos >= line_len: + success(pp_line, None) + return + + if line[pos] != '"': + fail("invalid #line directive", pos) + return + + m = re.match(_string_literal, line[pos:]) + if not m: + fail("invalid #line directive", pos) + return + + pp_filename = m.group(0).lstrip('"').rstrip('"') + pos += len(m.group(0)) + + # Consume arbitrary sequence of numeric flags after the directive + while True: + skip_ws() + if pos >= line_len: + break + m = re.match(_decimal_constant, line[pos:]) + if not m: + fail("invalid #line directive", pos) + return + pos += len(m.group(0)) + + success(pp_line, pp_filename) + + def _handle_pppragma(self) -> List[_Token]: + # Parse a full #pragma line; returns a list of tokens with 1 or 2 + # tokens - PPPRAGMA and an optional PPPRAGMASTR. If an empty list is + # returned, it means an error occurred, or we're at the end of input. + # + # Examples: + # - "#pragma" -> PPPRAGMA only + # - "#pragma once" -> PPPRAGMA, PPPRAGMASTR("once") + # - "# pragma omp parallel private(th_id)" -> PPPRAGMA, PPPRAGMASTR("omp parallel private(th_id)") + # - "#\tpragma {pack: 2, smack: 3}" -> PPPRAGMA, PPPRAGMASTR("{pack: 2, smack: 3}") + text = self._lexdata + n = len(text) + pos = self._pos + + while pos < n and text[pos] in " \t": + pos += 1 + if pos >= n: + self._pos = pos + return [] + + if not text.startswith("pragma", pos): + self._error("invalid #pragma directive", pos) + self._pos = pos + 1 + return [] + + pragma_pos = pos + pos += len("pragma") + toks = [self._make_token("PPPRAGMA", "pragma", pragma_pos)] + + while pos < n and text[pos] in " \t": + pos += 1 + + start = pos + while pos < n and text[pos] != "\n": + pos += 1 + if pos > start: + toks.append(self._make_token("PPPRAGMASTR", text[start:pos], start)) + if pos < n and text[pos] == "\n": + self._lineno += 1 + pos += 1 + self._line_start = pos + self._pos = pos + return toks + + +## +## Reserved keywords +## +_keywords: Tuple[str, ...] = ( + "AUTO", + "BREAK", + "CASE", + "CHAR", + "CONST", + "CONTINUE", + "DEFAULT", + "DO", + "DOUBLE", + "ELSE", + "ENUM", + "EXTERN", + "FLOAT", + "FOR", + "GOTO", + "IF", + "INLINE", + "INT", + "LONG", + "REGISTER", + "OFFSETOF", + "RESTRICT", + "RETURN", + "SHORT", + "SIGNED", + "SIZEOF", + "STATIC", + "STRUCT", + "SWITCH", + "TYPEDEF", + "UNION", + "UNSIGNED", + "VOID", + "VOLATILE", + "WHILE", + "__INT128", + "_BOOL", + "_COMPLEX", + "_NORETURN", + "_THREAD_LOCAL", + "_STATIC_ASSERT", + "_ATOMIC", + "_ALIGNOF", + "_ALIGNAS", + "_PRAGMA", +) + +_keyword_map: Dict[str, str] = {} + +for keyword in _keywords: + # Keywords from new C standard are mixed-case, like _Bool, _Alignas, etc. + if keyword.startswith("_") and len(keyword) > 1 and keyword[1].isalpha(): + _keyword_map[keyword[:2].upper() + keyword[2:].lower()] = keyword + else: + _keyword_map[keyword.lower()] = keyword + +## +## Regexes for use in tokens +## + +# valid C identifiers (K&R2: A.2.3), plus '$' (supported by some compilers) +_identifier = r"[a-zA-Z_$][0-9a-zA-Z_$]*" + +_hex_prefix = "0[xX]" +_hex_digits = "[0-9a-fA-F]+" +_bin_prefix = "0[bB]" +_bin_digits = "[01]+" + +# integer constants (K&R2: A.2.5.1) +_integer_suffix_opt = ( + r"(([uU]ll)|([uU]LL)|(ll[uU]?)|(LL[uU]?)|([uU][lL])|([lL][uU]?)|[uU])?" +) +_decimal_constant = ( + "(0" + _integer_suffix_opt + ")|([1-9][0-9]*" + _integer_suffix_opt + ")" +) +_octal_constant = "0[0-7]*" + _integer_suffix_opt +_hex_constant = _hex_prefix + _hex_digits + _integer_suffix_opt +_bin_constant = _bin_prefix + _bin_digits + _integer_suffix_opt + +_bad_octal_constant = "0[0-7]*[89]" + +# comments are not supported +_unsupported_c_style_comment = r"\/\*" +_unsupported_cxx_style_comment = r"\/\/" + +# character constants (K&R2: A.2.5.2) +# Note: a-zA-Z and '.-~^_!=&;,' are allowed as escape chars to support #line +# directives with Windows paths as filenames (..\..\dir\file) +# For the same reason, decimal_escape allows all digit sequences. We want to +# parse all correct code, even if it means to sometimes parse incorrect +# code. +# +# The original regexes were taken verbatim from the C syntax definition, +# and were later modified to avoid worst-case exponential running time. +# +# simple_escape = r"""([a-zA-Z._~!=&\^\-\\?'"])""" +# decimal_escape = r"""(\d+)""" +# hex_escape = r"""(x[0-9a-fA-F]+)""" +# bad_escape = r"""([\\][^a-zA-Z._~^!=&\^\-\\?'"x0-7])""" +# +# The following modifications were made to avoid the ambiguity that allowed +# backtracking: (https://github.com/eliben/pycparser/issues/61) +# +# - \x was removed from simple_escape, unless it was not followed by a hex +# digit, to avoid ambiguity with hex_escape. +# - hex_escape allows one or more hex characters, but requires that the next +# character(if any) is not hex +# - decimal_escape allows one or more decimal characters, but requires that the +# next character(if any) is not a decimal +# - bad_escape does not allow any decimals (8-9), to avoid conflicting with the +# permissive decimal_escape. +# +# Without this change, python's `re` module would recursively try parsing each +# ambiguous escape sequence in multiple ways. e.g. `\123` could be parsed as +# `\1`+`23`, `\12`+`3`, and `\123`. + +_simple_escape = r"""([a-wyzA-Z._~!=&\^\-\\?'"]|x(?![0-9a-fA-F]))""" +_decimal_escape = r"""(\d+)(?!\d)""" +_hex_escape = r"""(x[0-9a-fA-F]+)(?![0-9a-fA-F])""" +_bad_escape = r"""([\\][^a-zA-Z._~^!=&\^\-\\?'"x0-9])""" + +_escape_sequence = ( + r"""(\\(""" + _simple_escape + "|" + _decimal_escape + "|" + _hex_escape + "))" +) + +# This complicated regex with lookahead might be slow for strings, so because +# all of the valid escapes (including \x) allowed +# 0 or more non-escaped characters after the first character, +# simple_escape+decimal_escape+hex_escape got simplified to + +_escape_sequence_start_in_string = r"""(\\[0-9a-zA-Z._~!=&\^\-\\?'"])""" + +_cconst_char = r"""([^'\\\n]|""" + _escape_sequence + ")" +_char_const = "'" + _cconst_char + "'" +_wchar_const = "L" + _char_const +_u8char_const = "u8" + _char_const +_u16char_const = "u" + _char_const +_u32char_const = "U" + _char_const +_multicharacter_constant = "'" + _cconst_char + "{2,4}'" +_unmatched_quote = "('" + _cconst_char + "*\\n)|('" + _cconst_char + "*$)" +_bad_char_const = ( + r"""('""" + _cconst_char + """[^'\n]+')|('')|('""" + _bad_escape + r"""[^'\n]*')""" +) + +# string literals (K&R2: A.2.6) +_string_char = r"""([^"\\\n]|""" + _escape_sequence_start_in_string + ")" +_string_literal = '"' + _string_char + '*"' +_wstring_literal = "L" + _string_literal +_u8string_literal = "u8" + _string_literal +_u16string_literal = "u" + _string_literal +_u32string_literal = "U" + _string_literal +_bad_string_literal = '"' + _string_char + "*" + _bad_escape + _string_char + '*"' + +# floating constants (K&R2: A.2.5.3) +_exponent_part = r"""([eE][-+]?[0-9]+)""" +_fractional_constant = r"""([0-9]*\.[0-9]+)|([0-9]+\.)""" +_floating_constant = ( + "((((" + + _fractional_constant + + ")" + + _exponent_part + + "?)|([0-9]+" + + _exponent_part + + "))[FfLl]?)" +) +_binary_exponent_part = r"""([pP][+-]?[0-9]+)""" +_hex_fractional_constant = ( + "(((" + _hex_digits + r""")?\.""" + _hex_digits + ")|(" + _hex_digits + r"""\.))""" +) +_hex_floating_constant = ( + "(" + + _hex_prefix + + "(" + + _hex_digits + + "|" + + _hex_fractional_constant + + ")" + + _binary_exponent_part + + "[FfLl]?)" +) + + +class _RegexAction(Enum): + TOKEN = 0 + ID = 1 + ERROR = 2 + + +@dataclass(frozen=True) +class _RegexRule: + # tok_type: name of the token emitted for a match + # regex_pattern: the raw regex (no anchors) to match at the current position + # action: TOKEN for normal tokens, ID for identifiers, ERROR to report + # error_message: message used for ERROR entries + tok_type: str + regex_pattern: str + action: _RegexAction + error_message: Optional[str] + + +_regex_rules: List[_RegexRule] = [ + _RegexRule( + "UNSUPPORTED_C_STYLE_COMMENT", + _unsupported_c_style_comment, + _RegexAction.ERROR, + "Comments are not supported, see https://github.com/eliben/pycparser#3using.", + ), + _RegexRule( + "UNSUPPORTED_CXX_STYLE_COMMENT", + _unsupported_cxx_style_comment, + _RegexAction.ERROR, + "Comments are not supported, see https://github.com/eliben/pycparser#3using.", + ), + _RegexRule( + "BAD_STRING_LITERAL", + _bad_string_literal, + _RegexAction.ERROR, + "String contains invalid escape code", + ), + _RegexRule("WSTRING_LITERAL", _wstring_literal, _RegexAction.TOKEN, None), + _RegexRule("U8STRING_LITERAL", _u8string_literal, _RegexAction.TOKEN, None), + _RegexRule("U16STRING_LITERAL", _u16string_literal, _RegexAction.TOKEN, None), + _RegexRule("U32STRING_LITERAL", _u32string_literal, _RegexAction.TOKEN, None), + _RegexRule("STRING_LITERAL", _string_literal, _RegexAction.TOKEN, None), + _RegexRule("HEX_FLOAT_CONST", _hex_floating_constant, _RegexAction.TOKEN, None), + _RegexRule("FLOAT_CONST", _floating_constant, _RegexAction.TOKEN, None), + _RegexRule("INT_CONST_HEX", _hex_constant, _RegexAction.TOKEN, None), + _RegexRule("INT_CONST_BIN", _bin_constant, _RegexAction.TOKEN, None), + _RegexRule( + "BAD_CONST_OCT", + _bad_octal_constant, + _RegexAction.ERROR, + "Invalid octal constant", + ), + _RegexRule("INT_CONST_OCT", _octal_constant, _RegexAction.TOKEN, None), + _RegexRule("INT_CONST_DEC", _decimal_constant, _RegexAction.TOKEN, None), + _RegexRule("INT_CONST_CHAR", _multicharacter_constant, _RegexAction.TOKEN, None), + _RegexRule("CHAR_CONST", _char_const, _RegexAction.TOKEN, None), + _RegexRule("WCHAR_CONST", _wchar_const, _RegexAction.TOKEN, None), + _RegexRule("U8CHAR_CONST", _u8char_const, _RegexAction.TOKEN, None), + _RegexRule("U16CHAR_CONST", _u16char_const, _RegexAction.TOKEN, None), + _RegexRule("U32CHAR_CONST", _u32char_const, _RegexAction.TOKEN, None), + _RegexRule("UNMATCHED_QUOTE", _unmatched_quote, _RegexAction.ERROR, "Unmatched '"), + _RegexRule("BAD_CHAR_CONST", _bad_char_const, _RegexAction.ERROR, None), + _RegexRule("ID", _identifier, _RegexAction.ID, None), +] + +_regex_actions: Dict[str, Tuple[_RegexAction, Optional[str]]] = {} +_regex_pattern_parts: List[str] = [] +for _rule in _regex_rules: + _regex_actions[_rule.tok_type] = (_rule.action, _rule.error_message) + _regex_pattern_parts.append(f"(?P<{_rule.tok_type}>{_rule.regex_pattern})") +# The master regex is a single alternation of all token patterns, each wrapped +# in a named group. We match once at the current position and then use +# `lastgroup` to recover which token kind fired; this avoids iterating over all +# regexes on every character while keeping the same token-level semantics. +_regex_master: re.Pattern[str] = re.compile("|".join(_regex_pattern_parts)) + + +@dataclass(frozen=True) +class _FixedToken: + tok_type: str + literal: str + + +_fixed_tokens: List[_FixedToken] = [ + _FixedToken("ELLIPSIS", "..."), + _FixedToken("LSHIFTEQUAL", "<<="), + _FixedToken("RSHIFTEQUAL", ">>="), + _FixedToken("PLUSPLUS", "++"), + _FixedToken("MINUSMINUS", "--"), + _FixedToken("ARROW", "->"), + _FixedToken("LAND", "&&"), + _FixedToken("LOR", "||"), + _FixedToken("LSHIFT", "<<"), + _FixedToken("RSHIFT", ">>"), + _FixedToken("LE", "<="), + _FixedToken("GE", ">="), + _FixedToken("EQ", "=="), + _FixedToken("NE", "!="), + _FixedToken("TIMESEQUAL", "*="), + _FixedToken("DIVEQUAL", "/="), + _FixedToken("MODEQUAL", "%="), + _FixedToken("PLUSEQUAL", "+="), + _FixedToken("MINUSEQUAL", "-="), + _FixedToken("ANDEQUAL", "&="), + _FixedToken("OREQUAL", "|="), + _FixedToken("XOREQUAL", "^="), + _FixedToken("EQUALS", "="), + _FixedToken("PLUS", "+"), + _FixedToken("MINUS", "-"), + _FixedToken("TIMES", "*"), + _FixedToken("DIVIDE", "/"), + _FixedToken("MOD", "%"), + _FixedToken("OR", "|"), + _FixedToken("AND", "&"), + _FixedToken("NOT", "~"), + _FixedToken("XOR", "^"), + _FixedToken("LNOT", "!"), + _FixedToken("LT", "<"), + _FixedToken("GT", ">"), + _FixedToken("CONDOP", "?"), + _FixedToken("LPAREN", "("), + _FixedToken("RPAREN", ")"), + _FixedToken("LBRACKET", "["), + _FixedToken("RBRACKET", "]"), + _FixedToken("LBRACE", "{"), + _FixedToken("RBRACE", "}"), + _FixedToken("COMMA", ","), + _FixedToken("PERIOD", "."), + _FixedToken("SEMI", ";"), + _FixedToken("COLON", ":"), +] + +# To avoid scanning all fixed tokens on every character, we bucket them by the +# first character. When matching at position i, we only look at the bucket for +# text[i], and we pre-sort that bucket by token length so the first match is +# also the longest. This preserves longest-match semantics (e.g. '>>=' before +# '>>' before '>') while reducing the number of comparisons. +_fixed_tokens_by_first: Dict[str, List[_FixedToken]] = {} +for _entry in _fixed_tokens: + _fixed_tokens_by_first.setdefault(_entry.literal[0], []).append(_entry) +for _bucket in _fixed_tokens_by_first.values(): + _bucket.sort(key=lambda item: len(item.literal), reverse=True) + +_line_pattern: re.Pattern[str] = re.compile(r"([ \t]*line\W)|([ \t]*\d+)") +_pragma_pattern: re.Pattern[str] = re.compile(r"[ \t]*pragma\W") diff --git a/py_modules/pycparser/c_parser.py b/py_modules/pycparser/c_parser.py new file mode 100644 index 0000000..f980672 --- /dev/null +++ b/py_modules/pycparser/c_parser.py @@ -0,0 +1,2376 @@ +# ------------------------------------------------------------------------------ +# pycparser: c_parser.py +# +# Recursive-descent parser for the C language. +# +# Eli Bendersky [https://eli.thegreenplace.net/] +# License: BSD +# ------------------------------------------------------------------------------ +from dataclasses import dataclass +from typing import ( + Any, + Dict, + List, + Literal, + NoReturn, + Optional, + Tuple, + TypedDict, + cast, +) + +from . import c_ast +from .c_lexer import CLexer, _Token +from .ast_transforms import fix_switch_cases, fix_atomic_specifiers + + +@dataclass +class Coord: + """Coordinates of a syntactic element. Consists of: + - File name + - Line number + - Column number + """ + + file: str + line: int + column: Optional[int] = None + + def __str__(self) -> str: + text = f"{self.file}:{self.line}" + if self.column: + text += f":{self.column}" + return text + + +class ParseError(Exception): + pass + + +class CParser: + """Recursive-descent C parser. + + Usage: + parser = CParser() + ast = parser.parse(text, filename) + + The `lexer` parameter lets you inject a lexer class (defaults to CLexer). + The parameters after `lexer` are accepted for backward compatibility with + the old PLY-based parser and are otherwise unused. + """ + + def __init__( + self, + lex_optimize: bool = True, + lexer: type[CLexer] = CLexer, + lextab: str = "pycparser.lextab", + yacc_optimize: bool = True, + yacctab: str = "pycparser.yacctab", + yacc_debug: bool = False, + taboutputdir: str = "", + ) -> None: + self.clex: CLexer = lexer( + error_func=self._lex_error_func, + on_lbrace_func=self._lex_on_lbrace_func, + on_rbrace_func=self._lex_on_rbrace_func, + type_lookup_func=self._lex_type_lookup_func, + ) + + # Stack of scopes for keeping track of symbols. _scope_stack[-1] is + # the current (topmost) scope. Each scope is a dictionary that + # specifies whether a name is a type. If _scope_stack[n][name] is + # True, 'name' is currently a type in the scope. If it's False, + # 'name' is used in the scope but not as a type (for instance, if we + # saw: int name; + # If 'name' is not a key in _scope_stack[n] then 'name' was not defined + # in this scope at all. + self._scope_stack: List[Dict[str, bool]] = [dict()] + self._tokens: _TokenStream = _TokenStream(self.clex) + + def parse( + self, text: str, filename: str = "", debug: bool = False + ) -> c_ast.FileAST: + """Parses C code and returns an AST. + + text: + A string containing the C source code + + filename: + Name of the file being parsed (for meaningful + error messages) + + debug: + Deprecated debug flag (unused); for backwards compatibility. + """ + self._scope_stack = [dict()] + self.clex.input(text, filename) + self._tokens = _TokenStream(self.clex) + + ast = self._parse_translation_unit_or_empty() + tok = self._peek() + if tok is not None: + self._parse_error(f"before: {tok.value}", self._tok_coord(tok)) + return ast + + # ------------------------------------------------------------------ + # Scope and declaration helpers + # ------------------------------------------------------------------ + def _coord(self, lineno: int, column: Optional[int] = None) -> Coord: + return Coord(file=self.clex.filename, line=lineno, column=column) + + def _parse_error(self, msg: str, coord: Coord | str | None) -> NoReturn: + raise ParseError(f"{coord}: {msg}") + + def _push_scope(self) -> None: + self._scope_stack.append(dict()) + + def _pop_scope(self) -> None: + assert len(self._scope_stack) > 1 + self._scope_stack.pop() + + def _add_typedef_name(self, name: str, coord: Optional[Coord]) -> None: + """Add a new typedef name (ie a TYPEID) to the current scope""" + if not self._scope_stack[-1].get(name, True): + self._parse_error( + f"Typedef {name!r} previously declared as non-typedef in this scope", + coord, + ) + self._scope_stack[-1][name] = True + + def _add_identifier(self, name: str, coord: Optional[Coord]) -> None: + """Add a new object, function, or enum member name (ie an ID) to the + current scope + """ + if self._scope_stack[-1].get(name, False): + self._parse_error( + f"Non-typedef {name!r} previously declared as typedef in this scope", + coord, + ) + self._scope_stack[-1][name] = False + + def _is_type_in_scope(self, name: str) -> bool: + """Is *name* a typedef-name in the current scope?""" + for scope in reversed(self._scope_stack): + # If name is an identifier in this scope it shadows typedefs in + # higher scopes. + in_scope = scope.get(name) + if in_scope is not None: + return in_scope + return False + + def _lex_error_func(self, msg: str, line: int, column: int) -> None: + self._parse_error(msg, self._coord(line, column)) + + def _lex_on_lbrace_func(self) -> None: + self._push_scope() + + def _lex_on_rbrace_func(self) -> None: + self._pop_scope() + + def _lex_type_lookup_func(self, name: str) -> bool: + """Looks up types that were previously defined with + typedef. + Passed to the lexer for recognizing identifiers that + are types. + """ + return self._is_type_in_scope(name) + + # To understand what's going on here, read sections A.8.5 and + # A.8.6 of K&R2 very carefully. + # + # A C type consists of a basic type declaration, with a list + # of modifiers. For example: + # + # int *c[5]; + # + # The basic declaration here is 'int c', and the pointer and + # the array are the modifiers. + # + # Basic declarations are represented by TypeDecl (from module c_ast) and the + # modifiers are FuncDecl, PtrDecl and ArrayDecl. + # + # The standard states that whenever a new modifier is parsed, it should be + # added to the end of the list of modifiers. For example: + # + # K&R2 A.8.6.2: Array Declarators + # + # In a declaration T D where D has the form + # D1 [constant-expression-opt] + # and the type of the identifier in the declaration T D1 is + # "type-modifier T", the type of the + # identifier of D is "type-modifier array of T" + # + # This is what this method does. The declarator it receives + # can be a list of declarators ending with TypeDecl. It + # tacks the modifier to the end of this list, just before + # the TypeDecl. + # + # Additionally, the modifier may be a list itself. This is + # useful for pointers, that can come as a chain from the rule + # p_pointer. In this case, the whole modifier list is spliced + # into the new location. + def _type_modify_decl(self, decl: Any, modifier: Any) -> c_ast.Node: + """Tacks a type modifier on a declarator, and returns + the modified declarator. + + Note: the declarator and modifier may be modified + """ + modifier_head = modifier + modifier_tail = modifier + + # The modifier may be a nested list. Reach its tail. + while modifier_tail.type: + modifier_tail = modifier_tail.type + + # If the decl is a basic type, just tack the modifier onto it. + if isinstance(decl, c_ast.TypeDecl): + modifier_tail.type = decl + return modifier + else: + # Otherwise, the decl is a list of modifiers. Reach + # its tail and splice the modifier onto the tail, + # pointing to the underlying basic type. + decl_tail = decl + while not isinstance(decl_tail.type, c_ast.TypeDecl): + decl_tail = decl_tail.type + + modifier_tail.type = decl_tail.type + decl_tail.type = modifier_head + return decl + + # Due to the order in which declarators are constructed, + # they have to be fixed in order to look like a normal AST. + # + # When a declaration arrives from syntax construction, it has + # these problems: + # * The innermost TypeDecl has no type (because the basic + # type is only known at the uppermost declaration level) + # * The declaration has no variable name, since that is saved + # in the innermost TypeDecl + # * The typename of the declaration is a list of type + # specifiers, and not a node. Here, basic identifier types + # should be separated from more complex types like enums + # and structs. + # + # This method fixes these problems. + def _fix_decl_name_type( + self, + decl: c_ast.Decl | c_ast.Typedef | c_ast.Typename, + typename: List[Any], + ) -> c_ast.Decl | c_ast.Typedef | c_ast.Typename: + """Fixes a declaration. Modifies decl.""" + # Reach the underlying basic type + typ = decl + while not isinstance(typ, c_ast.TypeDecl): + typ = typ.type + + decl.name = typ.declname + typ.quals = decl.quals[:] + + # The typename is a list of types. If any type in this + # list isn't an IdentifierType, it must be the only + # type in the list (it's illegal to declare "int enum ..") + # If all the types are basic, they're collected in the + # IdentifierType holder. + for tn in typename: + if not isinstance(tn, c_ast.IdentifierType): + if len(typename) > 1: + self._parse_error("Invalid multiple types specified", tn.coord) + else: + typ.type = tn + return decl + + if not typename: + # Functions default to returning int + if not isinstance(decl.type, c_ast.FuncDecl): + self._parse_error("Missing type in declaration", decl.coord) + typ.type = c_ast.IdentifierType(["int"], coord=decl.coord) + else: + # At this point, we know that typename is a list of IdentifierType + # nodes. Concatenate all the names into a single list. + typ.type = c_ast.IdentifierType( + [name for id in typename for name in id.names], coord=typename[0].coord + ) + return decl + + def _add_declaration_specifier( + self, + declspec: Optional["_DeclSpec"], + newspec: Any, + kind: "_DeclSpecKind", + append: bool = False, + ) -> "_DeclSpec": + """See _DeclSpec for the specifier dictionary layout.""" + if declspec is None: + spec: _DeclSpec = dict( + qual=[], storage=[], type=[], function=[], alignment=[] + ) + else: + spec = declspec + + if append: + spec[kind].append(newspec) + else: + spec[kind].insert(0, newspec) + + return spec + + def _build_declarations( + self, + spec: "_DeclSpec", + decls: List["_DeclInfo"], + typedef_namespace: bool = False, + ) -> List[c_ast.Node]: + """Builds a list of declarations all sharing the given specifiers. + If typedef_namespace is true, each declared name is added + to the "typedef namespace", which also includes objects, + functions, and enum constants. + """ + is_typedef = "typedef" in spec["storage"] + declarations = [] + + # Bit-fields are allowed to be unnamed. + if decls[0].get("bitsize") is None: + # When redeclaring typedef names as identifiers in inner scopes, a + # problem can occur where the identifier gets grouped into + # spec['type'], leaving decl as None. This can only occur for the + # first declarator. + if decls[0]["decl"] is None: + if ( + len(spec["type"]) < 2 + or len(spec["type"][-1].names) != 1 + or not self._is_type_in_scope(spec["type"][-1].names[0]) + ): + coord = "?" + for t in spec["type"]: + if hasattr(t, "coord"): + coord = t.coord + break + self._parse_error("Invalid declaration", coord) + + # Make this look as if it came from "direct_declarator:ID" + decls[0]["decl"] = c_ast.TypeDecl( + declname=spec["type"][-1].names[0], + type=None, + quals=None, + align=spec["alignment"], + coord=spec["type"][-1].coord, + ) + # Remove the "new" type's name from the end of spec['type'] + del spec["type"][-1] + # A similar problem can occur where the declaration ends up + # looking like an abstract declarator. Give it a name if this is + # the case. + elif not isinstance( + decls[0]["decl"], + (c_ast.Enum, c_ast.Struct, c_ast.Union, c_ast.IdentifierType), + ): + decls_0_tail = cast(Any, decls[0]["decl"]) + while not isinstance(decls_0_tail, c_ast.TypeDecl): + decls_0_tail = decls_0_tail.type + if decls_0_tail.declname is None: + decls_0_tail.declname = spec["type"][-1].names[0] + del spec["type"][-1] + + for decl in decls: + assert decl["decl"] is not None + if is_typedef: + declaration = c_ast.Typedef( + name=None, + quals=spec["qual"], + storage=spec["storage"], + type=decl["decl"], + coord=decl["decl"].coord, + ) + else: + declaration = c_ast.Decl( + name=None, + quals=spec["qual"], + align=spec["alignment"], + storage=spec["storage"], + funcspec=spec["function"], + type=decl["decl"], + init=decl.get("init"), + bitsize=decl.get("bitsize"), + coord=decl["decl"].coord, + ) + + if isinstance( + declaration.type, + (c_ast.Enum, c_ast.Struct, c_ast.Union, c_ast.IdentifierType), + ): + fixed_decl = declaration + else: + fixed_decl = self._fix_decl_name_type(declaration, spec["type"]) + + # Add the type name defined by typedef to a + # symbol table (for usage in the lexer) + if typedef_namespace: + if is_typedef: + self._add_typedef_name(fixed_decl.name, fixed_decl.coord) + else: + self._add_identifier(fixed_decl.name, fixed_decl.coord) + + fixed_decl = fix_atomic_specifiers( + cast(c_ast.Decl | c_ast.Typedef, fixed_decl) + ) + declarations.append(fixed_decl) + + return declarations + + def _build_function_definition( + self, + spec: "_DeclSpec", + decl: c_ast.Node, + param_decls: Optional[List[c_ast.Node]], + body: c_ast.Node, + ) -> c_ast.Node: + """Builds a function definition.""" + if "typedef" in spec["storage"]: + self._parse_error("Invalid typedef", decl.coord) + + declaration = self._build_declarations( + spec=spec, + decls=[dict(decl=decl, init=None, bitsize=None)], + typedef_namespace=True, + )[0] + + return c_ast.FuncDef( + decl=declaration, param_decls=param_decls, body=body, coord=decl.coord + ) + + def _select_struct_union_class(self, token: str) -> type: + """Given a token (either STRUCT or UNION), selects the + appropriate AST class. + """ + if token == "struct": + return c_ast.Struct + else: + return c_ast.Union + + # ------------------------------------------------------------------ + # Token helpers + # ------------------------------------------------------------------ + def _peek(self, k: int = 1) -> Optional[_Token]: + """Return the k-th next token without consuming it (1-based).""" + return self._tokens.peek(k) + + def _peek_type(self, k: int = 1) -> Optional[str]: + """Return the type of the k-th next token, or None if absent (1-based).""" + tok = self._peek(k) + return tok.type if tok is not None else None + + def _advance(self) -> _Token: + tok = self._tokens.next() + if tok is None: + self._parse_error("At end of input", self.clex.filename) + else: + return tok + + def _accept(self, token_type: str) -> Optional[_Token]: + """Conditionally consume next token, only if it's of token_type. + + If it is of the expected type, consume and return it. + Otherwise, leaves the token intact and returns None. + """ + tok = self._peek() + if tok is not None and tok.type == token_type: + return self._advance() + return None + + def _expect(self, token_type: str) -> _Token: + tok = self._advance() + if tok.type != token_type: + self._parse_error(f"before: {tok.value}", self._tok_coord(tok)) + return tok + + def _mark(self) -> int: + return self._tokens.mark() + + def _reset(self, mark: int) -> None: + self._tokens.reset(mark) + + def _tok_coord(self, tok: _Token) -> Coord: + return self._coord(tok.lineno, tok.column) + + def _starts_declaration(self, tok: Optional[_Token] = None) -> bool: + tok = tok or self._peek() + if tok is None: + return False + return tok.type in _DECL_START + + def _starts_expression(self, tok: Optional[_Token] = None) -> bool: + tok = tok or self._peek() + if tok is None: + return False + return tok.type in _STARTS_EXPRESSION + + def _starts_statement(self) -> bool: + tok_type = self._peek_type() + if tok_type is None: + return False + if tok_type in _STARTS_STATEMENT: + return True + return self._starts_expression() + + def _starts_declarator(self, id_only: bool = False) -> bool: + tok_type = self._peek_type() + if tok_type is None: + return False + if tok_type in {"TIMES", "LPAREN"}: + return True + if id_only: + return tok_type == "ID" + return tok_type in {"ID", "TYPEID"} + + def _peek_declarator_name_info(self) -> Tuple[Optional[str], bool]: + mark = self._mark() + tok_type, saw_paren = self._scan_declarator_name_info() + self._reset(mark) + return tok_type, saw_paren + + def _parse_any_declarator( + self, allow_abstract: bool = False, typeid_paren_as_abstract: bool = False + ) -> Tuple[Optional[c_ast.Node], bool]: + # C declarators are ambiguous without lookahead. For example: + # int foo(int (aa)); -> aa is a name (ID) + # typedef char TT; + # int bar(int (TT)); -> TT is a type (TYPEID) in parens + name_type, saw_paren = self._peek_declarator_name_info() + if name_type is None or ( + typeid_paren_as_abstract and name_type == "TYPEID" and saw_paren + ): + if not allow_abstract: + tok = self._peek() + coord = self._tok_coord(tok) if tok is not None else self.clex.filename + self._parse_error("Invalid declarator", coord) + decl = self._parse_abstract_declarator_opt() + return decl, False + + if name_type == "TYPEID": + if typeid_paren_as_abstract: + decl = self._parse_typeid_noparen_declarator() + else: + decl = self._parse_typeid_declarator() + else: + decl = self._parse_id_declarator() + return decl, True + + def _scan_declarator_name_info(self) -> Tuple[Optional[str], bool]: + saw_paren = False + while self._accept("TIMES"): + while self._peek_type() in _TYPE_QUALIFIER: + self._advance() + + tok = self._peek() + if tok is None: + return None, saw_paren + if tok.type in {"ID", "TYPEID"}: + self._advance() + return tok.type, saw_paren + if tok.type == "LPAREN": + saw_paren = True + self._advance() + tok_type, nested_paren = self._scan_declarator_name_info() + if nested_paren: + saw_paren = True + depth = 1 + while True: + tok = self._peek() + if tok is None: + return None, saw_paren + if tok.type == "LPAREN": + depth += 1 + elif tok.type == "RPAREN": + depth -= 1 + self._advance() + if depth == 0: + break + continue + self._advance() + return tok_type, saw_paren + return None, saw_paren + + def _starts_direct_abstract_declarator(self) -> bool: + return self._peek_type() in {"LPAREN", "LBRACKET"} + + def _is_assignment_op(self) -> bool: + tok = self._peek() + return tok is not None and tok.type in _ASSIGNMENT_OPS + + def _try_parse_paren_type_name( + self, + ) -> Optional[Tuple[c_ast.Typename, int, _Token]]: + """Parse and return a parenthesized type name if present. + + Returns (typ, mark, lparen_tok) when the next tokens look like + '(' type_name ')', where typ is the parsed type name, mark is the + token-stream position before parsing, and lparen_tok is the LPAREN + token. Returns None if no parenthesized type name is present. + """ + mark = self._mark() + lparen_tok = self._accept("LPAREN") + if lparen_tok is None: + return None + if not self._starts_declaration(): + self._reset(mark) + return None + typ = self._parse_type_name() + if self._accept("RPAREN") is None: + self._reset(mark) + return None + return typ, mark, lparen_tok + + # ------------------------------------------------------------------ + # Top-level + # ------------------------------------------------------------------ + # BNF: translation_unit_or_empty : translation_unit | empty + def _parse_translation_unit_or_empty(self) -> c_ast.FileAST: + if self._peek() is None: + return c_ast.FileAST([]) + return c_ast.FileAST(self._parse_translation_unit()) + + # BNF: translation_unit : external_declaration+ + def _parse_translation_unit(self) -> List[c_ast.Node]: + ext = [] + while self._peek() is not None: + ext.extend(self._parse_external_declaration()) + return ext + + # BNF: external_declaration : function_definition + # | declaration + # | pp_directive + # | pppragma_directive + # | static_assert + # | ';' + def _parse_external_declaration(self) -> List[c_ast.Node]: + tok = self._peek() + if tok is None: + return [] + if tok.type == "PPHASH": + self._parse_pp_directive() + return [] + if tok.type in {"PPPRAGMA", "_PRAGMA"}: + return [self._parse_pppragma_directive()] + if self._accept("SEMI"): + return [] + if tok.type == "_STATIC_ASSERT": + return self._parse_static_assert() + + if not self._starts_declaration(tok): + # Special handling for old-style function definitions that have an + # implicit return type, e.g. + # + # foo() { + # return 5; + # } + # + # These get an implicit 'int' return type. + decl = self._parse_id_declarator() + param_decls = None + if self._peek_type() != "LBRACE": + self._parse_error("Invalid function definition", decl.coord) + spec: _DeclSpec = dict( + qual=[], + alignment=[], + storage=[], + type=[c_ast.IdentifierType(["int"], coord=decl.coord)], + function=[], + ) + func = self._build_function_definition( + spec=spec, + decl=decl, + param_decls=param_decls, + body=self._parse_compound_statement(), + ) + return [func] + + # From here on, parsing a standard declatation/definition. + spec, saw_type, spec_coord = self._parse_declaration_specifiers( + allow_no_type=True + ) + + name_type, _ = self._peek_declarator_name_info() + if name_type != "ID": + decls = self._parse_decl_body_with_spec(spec, saw_type) + self._expect("SEMI") + return decls + + decl = self._parse_id_declarator() + + if self._peek_type() == "LBRACE" or self._starts_declaration(): + param_decls = None + if self._starts_declaration(): + param_decls = self._parse_declaration_list() + if self._peek_type() != "LBRACE": + self._parse_error("Invalid function definition", decl.coord) + if not spec["type"]: + spec["type"] = [c_ast.IdentifierType(["int"], coord=spec_coord)] + func = self._build_function_definition( + spec=spec, + decl=decl, + param_decls=param_decls, + body=self._parse_compound_statement(), + ) + return [func] + + decl_dict: "_DeclInfo" = dict(decl=decl, init=None, bitsize=None) + if self._accept("EQUALS"): + decl_dict["init"] = self._parse_initializer() + decls = self._parse_init_declarator_list(first=decl_dict) + decls = self._build_declarations(spec=spec, decls=decls, typedef_namespace=True) + self._expect("SEMI") + return decls + + # ------------------------------------------------------------------ + # Declarations + # + # Declarations always come as lists (because they can be several in one + # line). When returning parsed declarations, a list is always returned - + # even if it contains a single element. + # ------------------------------------------------------------------ + def _parse_declaration(self) -> List[c_ast.Node]: + decls = self._parse_decl_body() + self._expect("SEMI") + return decls + + # BNF: decl_body : declaration_specifiers decl_body_with_spec + def _parse_decl_body(self) -> List[c_ast.Node]: + spec, saw_type, _ = self._parse_declaration_specifiers(allow_no_type=True) + return self._parse_decl_body_with_spec(spec, saw_type) + + # BNF: decl_body_with_spec : init_declarator_list + # | struct_or_union_or_enum_only + def _parse_decl_body_with_spec( + self, spec: "_DeclSpec", saw_type: bool + ) -> List[c_ast.Node]: + decls = None + if saw_type: + if self._starts_declarator(): + decls = self._parse_init_declarator_list() + else: + if self._starts_declarator(id_only=True): + decls = self._parse_init_declarator_list(id_only=True) + + if decls is None: + ty = spec["type"] + s_u_or_e = (c_ast.Struct, c_ast.Union, c_ast.Enum) + if len(ty) == 1 and isinstance(ty[0], s_u_or_e): + decls = [ + c_ast.Decl( + name=None, + quals=spec["qual"], + align=spec["alignment"], + storage=spec["storage"], + funcspec=spec["function"], + type=ty[0], + init=None, + bitsize=None, + coord=ty[0].coord, + ) + ] + else: + decls = self._build_declarations( + spec=spec, + decls=[dict(decl=None, init=None, bitsize=None)], + typedef_namespace=True, + ) + else: + decls = self._build_declarations( + spec=spec, decls=decls, typedef_namespace=True + ) + + return decls + + # BNF: declaration_list : declaration+ + def _parse_declaration_list(self) -> List[c_ast.Node]: + decls = [] + while self._starts_declaration(): + decls.extend(self._parse_declaration()) + return decls + + # BNF: declaration_specifiers : (storage_class_specifier + # | type_specifier + # | type_qualifier + # | function_specifier + # | alignment_specifier)+ + def _parse_declaration_specifiers( + self, allow_no_type: bool = False + ) -> Tuple["_DeclSpec", bool, Optional[Coord]]: + """Parse declaration-specifier sequence. + + allow_no_type: + If True, allow a missing type specifier without error. + + Returns: + (spec, saw_type, first_coord) where spec is a dict with + qual/storage/type/function/alignment entries, saw_type is True + if a type specifier was consumed, and first_coord is the coord + of the first specifier token (used for diagnostics). + """ + spec = None + saw_type = False + first_coord = None + + while True: + tok = self._peek() + if tok is None: + break + + if tok.type == "_ALIGNAS": + if first_coord is None: + first_coord = self._tok_coord(tok) + spec = self._add_declaration_specifier( + spec, self._parse_alignment_specifier(), "alignment", append=True + ) + continue + + if tok.type == "_ATOMIC" and self._peek_type(2) == "LPAREN": + if first_coord is None: + first_coord = self._tok_coord(tok) + spec = self._add_declaration_specifier( + spec, self._parse_atomic_specifier(), "type", append=True + ) + saw_type = True + continue + + if tok.type in _TYPE_QUALIFIER: + if first_coord is None: + first_coord = self._tok_coord(tok) + spec = self._add_declaration_specifier( + spec, self._advance().value, "qual", append=True + ) + continue + + if tok.type in _STORAGE_CLASS: + if first_coord is None: + first_coord = self._tok_coord(tok) + spec = self._add_declaration_specifier( + spec, self._advance().value, "storage", append=True + ) + continue + + if tok.type in _FUNCTION_SPEC: + if first_coord is None: + first_coord = self._tok_coord(tok) + spec = self._add_declaration_specifier( + spec, self._advance().value, "function", append=True + ) + continue + + if tok.type in _TYPE_SPEC_SIMPLE: + if first_coord is None: + first_coord = self._tok_coord(tok) + tok = self._advance() + spec = self._add_declaration_specifier( + spec, + c_ast.IdentifierType([tok.value], coord=self._tok_coord(tok)), + "type", + append=True, + ) + saw_type = True + continue + + if tok.type == "TYPEID": + if saw_type: + break + if first_coord is None: + first_coord = self._tok_coord(tok) + tok = self._advance() + spec = self._add_declaration_specifier( + spec, + c_ast.IdentifierType([tok.value], coord=self._tok_coord(tok)), + "type", + append=True, + ) + saw_type = True + continue + + if tok.type in {"STRUCT", "UNION"}: + if first_coord is None: + first_coord = self._tok_coord(tok) + spec = self._add_declaration_specifier( + spec, self._parse_struct_or_union_specifier(), "type", append=True + ) + saw_type = True + continue + + if tok.type == "ENUM": + if first_coord is None: + first_coord = self._tok_coord(tok) + spec = self._add_declaration_specifier( + spec, self._parse_enum_specifier(), "type", append=True + ) + saw_type = True + continue + + break + + if spec is None: + self._parse_error("Invalid declaration", self.clex.filename) + + if not saw_type and not allow_no_type: + self._parse_error("Missing type in declaration", first_coord) + + return spec, saw_type, first_coord + + # BNF: specifier_qualifier_list : (type_specifier + # | type_qualifier + # | alignment_specifier)+ + def _parse_specifier_qualifier_list(self) -> "_DeclSpec": + spec = None + saw_type = False + saw_alignment = False + first_coord = None + + while True: + tok = self._peek() + if tok is None: + break + + if tok.type == "_ALIGNAS": + if first_coord is None: + first_coord = self._tok_coord(tok) + spec = self._add_declaration_specifier( + spec, self._parse_alignment_specifier(), "alignment", append=True + ) + saw_alignment = True + continue + + if tok.type == "_ATOMIC" and self._peek_type(2) == "LPAREN": + if first_coord is None: + first_coord = self._tok_coord(tok) + spec = self._add_declaration_specifier( + spec, self._parse_atomic_specifier(), "type", append=True + ) + saw_type = True + continue + + if tok.type in _TYPE_QUALIFIER: + if first_coord is None: + first_coord = self._tok_coord(tok) + spec = self._add_declaration_specifier( + spec, self._advance().value, "qual", append=True + ) + continue + + if tok.type in _TYPE_SPEC_SIMPLE: + if first_coord is None: + first_coord = self._tok_coord(tok) + tok = self._advance() + spec = self._add_declaration_specifier( + spec, + c_ast.IdentifierType([tok.value], coord=self._tok_coord(tok)), + "type", + append=True, + ) + saw_type = True + continue + + if tok.type == "TYPEID": + if saw_type: + break + if first_coord is None: + first_coord = self._tok_coord(tok) + tok = self._advance() + spec = self._add_declaration_specifier( + spec, + c_ast.IdentifierType([tok.value], coord=self._tok_coord(tok)), + "type", + append=True, + ) + saw_type = True + continue + + if tok.type in {"STRUCT", "UNION"}: + if first_coord is None: + first_coord = self._tok_coord(tok) + spec = self._add_declaration_specifier( + spec, self._parse_struct_or_union_specifier(), "type", append=True + ) + saw_type = True + continue + + if tok.type == "ENUM": + if first_coord is None: + first_coord = self._tok_coord(tok) + spec = self._add_declaration_specifier( + spec, self._parse_enum_specifier(), "type", append=True + ) + saw_type = True + continue + + break + + if spec is None: + self._parse_error("Invalid specifier list", self.clex.filename) + + if not saw_type and not saw_alignment: + self._parse_error("Missing type in declaration", first_coord) + + if spec.get("storage") is None: + spec["storage"] = [] + if spec.get("function") is None: + spec["function"] = [] + + return spec + + # BNF: type_qualifier_list : type_qualifier+ + def _parse_type_qualifier_list(self) -> List[str]: + quals = [] + while self._peek_type() in _TYPE_QUALIFIER: + quals.append(self._advance().value) + return quals + + # BNF: alignment_specifier : _ALIGNAS '(' type_name | constant_expression ')' + def _parse_alignment_specifier(self) -> c_ast.Node: + tok = self._expect("_ALIGNAS") + self._expect("LPAREN") + + if self._starts_declaration(): + typ = self._parse_type_name() + self._expect("RPAREN") + return c_ast.Alignas(typ, self._tok_coord(tok)) + + expr = self._parse_constant_expression() + self._expect("RPAREN") + return c_ast.Alignas(expr, self._tok_coord(tok)) + + # BNF: atomic_specifier : _ATOMIC '(' type_name ')' + def _parse_atomic_specifier(self) -> c_ast.Node: + self._expect("_ATOMIC") + self._expect("LPAREN") + typ = self._parse_type_name() + self._expect("RPAREN") + typ.quals.append("_Atomic") + return typ + + # BNF: init_declarator_list : init_declarator (',' init_declarator)* + def _parse_init_declarator_list( + self, first: Optional["_DeclInfo"] = None, id_only: bool = False + ) -> List["_DeclInfo"]: + decls = ( + [first] + if first is not None + else [self._parse_init_declarator(id_only=id_only)] + ) + + while self._accept("COMMA"): + decls.append(self._parse_init_declarator(id_only=id_only)) + return decls + + # BNF: init_declarator : declarator ('=' initializer)? + def _parse_init_declarator(self, id_only: bool = False) -> "_DeclInfo": + decl = self._parse_id_declarator() if id_only else self._parse_declarator() + init = None + if self._accept("EQUALS"): + init = self._parse_initializer() + return dict(decl=decl, init=init, bitsize=None) + + # ------------------------------------------------------------------ + # Structs/unions/enums + # ------------------------------------------------------------------ + # BNF: struct_or_union_specifier : struct_or_union ID? '{' struct_declaration_list? '}' + # | struct_or_union ID + def _parse_struct_or_union_specifier(self) -> c_ast.Node: + tok = self._advance() + klass = self._select_struct_union_class(tok.value) + + if self._peek_type() in {"ID", "TYPEID"}: + name_tok = self._advance() + if self._peek_type() == "LBRACE": + self._advance() + if self._accept("RBRACE"): + return klass( + name=name_tok.value, decls=[], coord=self._tok_coord(name_tok) + ) + decls = self._parse_struct_declaration_list() + self._expect("RBRACE") + return klass( + name=name_tok.value, decls=decls, coord=self._tok_coord(name_tok) + ) + + return klass( + name=name_tok.value, decls=None, coord=self._tok_coord(name_tok) + ) + + if self._peek_type() == "LBRACE": + brace_tok = self._advance() + if self._accept("RBRACE"): + return klass(name=None, decls=[], coord=self._tok_coord(brace_tok)) + decls = self._parse_struct_declaration_list() + self._expect("RBRACE") + return klass(name=None, decls=decls, coord=self._tok_coord(brace_tok)) + + self._parse_error("Invalid struct/union declaration", self._tok_coord(tok)) + + # BNF: struct_declaration_list : struct_declaration+ + def _parse_struct_declaration_list(self) -> List[c_ast.Node]: + decls = [] + while self._peek_type() not in {None, "RBRACE"}: + items = self._parse_struct_declaration() + if items is None: + continue + decls.extend(items) + return decls + + # BNF: struct_declaration : specifier_qualifier_list struct_declarator_list? ';' + # | static_assert + # | pppragma_directive + def _parse_struct_declaration(self) -> Optional[List[c_ast.Node]]: + if self._peek_type() == "SEMI": + self._advance() + return None + if self._peek_type() in {"PPPRAGMA", "_PRAGMA"}: + return [self._parse_pppragma_directive()] + + spec = self._parse_specifier_qualifier_list() + assert "typedef" not in spec.get("storage", []) + + decls = None + if self._starts_declarator() or self._peek_type() == "COLON": + decls = self._parse_struct_declarator_list() + if decls is not None: + self._expect("SEMI") + return self._build_declarations(spec=spec, decls=decls) + + if len(spec["type"]) == 1: + node = spec["type"][0] + if isinstance(node, c_ast.Node): + decl_type = node + else: + decl_type = c_ast.IdentifierType(node) + self._expect("SEMI") + return self._build_declarations( + spec=spec, decls=[dict(decl=decl_type, init=None, bitsize=None)] + ) + + self._expect("SEMI") + return self._build_declarations( + spec=spec, decls=[dict(decl=None, init=None, bitsize=None)] + ) + + # BNF: struct_declarator_list : struct_declarator (',' struct_declarator)* + def _parse_struct_declarator_list(self) -> List["_DeclInfo"]: + decls = [self._parse_struct_declarator()] + while self._accept("COMMA"): + decls.append(self._parse_struct_declarator()) + return decls + + # BNF: struct_declarator : declarator? ':' constant_expression + # | declarator (':' constant_expression)? + def _parse_struct_declarator(self) -> "_DeclInfo": + if self._accept("COLON"): + bitsize = self._parse_constant_expression() + return { + "decl": c_ast.TypeDecl(None, None, None, None), + "init": None, + "bitsize": bitsize, + } + + decl = self._parse_declarator() + if self._accept("COLON"): + bitsize = self._parse_constant_expression() + return {"decl": decl, "init": None, "bitsize": bitsize} + + return {"decl": decl, "init": None, "bitsize": None} + + # BNF: enum_specifier : ENUM ID? '{' enumerator_list? '}' + # | ENUM ID + def _parse_enum_specifier(self) -> c_ast.Node: + tok = self._expect("ENUM") + if self._peek_type() in {"ID", "TYPEID"}: + name_tok = self._advance() + if self._peek_type() == "LBRACE": + self._advance() + enums = self._parse_enumerator_list() + self._expect("RBRACE") + return c_ast.Enum(name_tok.value, enums, self._tok_coord(tok)) + return c_ast.Enum(name_tok.value, None, self._tok_coord(tok)) + + self._expect("LBRACE") + enums = self._parse_enumerator_list() + self._expect("RBRACE") + return c_ast.Enum(None, enums, self._tok_coord(tok)) + + # BNF: enumerator_list : enumerator (',' enumerator)* ','? + def _parse_enumerator_list(self) -> c_ast.Node: + enum = self._parse_enumerator() + enum_list = c_ast.EnumeratorList([enum], enum.coord) + while self._accept("COMMA"): + if self._peek_type() == "RBRACE": + break + enum = self._parse_enumerator() + enum_list.enumerators.append(enum) + return enum_list + + # BNF: enumerator : ID ('=' constant_expression)? + def _parse_enumerator(self) -> c_ast.Node: + name_tok = self._expect("ID") + if self._accept("EQUALS"): + value = self._parse_constant_expression() + else: + value = None + enum = c_ast.Enumerator(name_tok.value, value, self._tok_coord(name_tok)) + self._add_identifier(enum.name, enum.coord) + return enum + + # ------------------------------------------------------------------ + # Declarators + # ------------------------------------------------------------------ + # BNF: declarator : pointer? direct_declarator + def _parse_declarator(self) -> c_ast.Node: + decl, _ = self._parse_any_declarator( + allow_abstract=False, typeid_paren_as_abstract=False + ) + assert decl is not None + return decl + + # BNF: id_declarator : declarator with ID name + def _parse_id_declarator(self) -> c_ast.Node: + return self._parse_declarator_kind(kind="id", allow_paren=True) + + # BNF: typeid_declarator : declarator with TYPEID name + def _parse_typeid_declarator(self) -> c_ast.Node: + return self._parse_declarator_kind(kind="typeid", allow_paren=True) + + # BNF: typeid_noparen_declarator : declarator without parenthesized name + def _parse_typeid_noparen_declarator(self) -> c_ast.Node: + return self._parse_declarator_kind(kind="typeid", allow_paren=False) + + # BNF: declarator_kind : pointer? direct_declarator(kind) + def _parse_declarator_kind(self, kind: str, allow_paren: bool) -> c_ast.Node: + ptr = None + if self._peek_type() == "TIMES": + ptr = self._parse_pointer() + direct = self._parse_direct_declarator(kind, allow_paren=allow_paren) + if ptr is not None: + return self._type_modify_decl(direct, ptr) + return direct + + # BNF: direct_declarator : ID | TYPEID | '(' declarator ')' + # | direct_declarator '[' ... ']' + # | direct_declarator '(' ... ')' + def _parse_direct_declarator( + self, kind: str, allow_paren: bool = True + ) -> c_ast.Node: + if allow_paren and self._accept("LPAREN"): + decl = self._parse_declarator_kind(kind, allow_paren=True) + self._expect("RPAREN") + else: + if kind == "id": + name_tok = self._expect("ID") + else: + name_tok = self._expect("TYPEID") + decl = c_ast.TypeDecl( + declname=name_tok.value, + type=None, + quals=None, + align=None, + coord=self._tok_coord(name_tok), + ) + + return self._parse_decl_suffixes(decl) + + def _parse_decl_suffixes(self, decl: c_ast.Node) -> c_ast.Node: + """Parse a chain of array/function suffixes and attach them to decl.""" + while True: + if self._peek_type() == "LBRACKET": + decl = self._type_modify_decl(decl, self._parse_array_decl(decl)) + continue + if self._peek_type() == "LPAREN": + func = self._parse_function_decl(decl) + decl = self._type_modify_decl(decl, func) + continue + break + return decl + + # BNF: array_decl : '[' array_specifiers? assignment_expression? ']' + def _parse_array_decl(self, base_decl: c_ast.Node) -> c_ast.Node: + return self._parse_array_decl_common(base_type=None, coord=base_decl.coord) + + def _parse_array_decl_common( + self, base_type: Optional[c_ast.Node], coord: Optional[Coord] = None + ) -> c_ast.Node: + """Parse an array declarator suffix and return an ArrayDecl node. + + base_type: + Base declarator node to attach (None for direct-declarator parsing, + TypeDecl for abstract declarators). + + coord: + Coordinate to use for the ArrayDecl. If None, uses the '[' token. + """ + lbrack_tok = self._expect("LBRACKET") + if coord is None: + coord = self._tok_coord(lbrack_tok) + + def make_array_decl(dim, dim_quals): + return c_ast.ArrayDecl( + type=base_type, dim=dim, dim_quals=dim_quals, coord=coord + ) + + if self._accept("STATIC"): + dim_quals = ["static"] + (self._parse_type_qualifier_list() or []) + dim = self._parse_assignment_expression() + self._expect("RBRACKET") + return make_array_decl(dim, dim_quals) + + if self._peek_type() in _TYPE_QUALIFIER: + dim_quals = self._parse_type_qualifier_list() or [] + if self._accept("STATIC"): + dim_quals = dim_quals + ["static"] + dim = self._parse_assignment_expression() + self._expect("RBRACKET") + return make_array_decl(dim, dim_quals) + times_tok = self._accept("TIMES") + if times_tok: + self._expect("RBRACKET") + dim = c_ast.ID(times_tok.value, self._tok_coord(times_tok)) + return make_array_decl(dim, dim_quals) + dim = None + if self._starts_expression(): + dim = self._parse_assignment_expression() + self._expect("RBRACKET") + return make_array_decl(dim, dim_quals) + + times_tok = self._accept("TIMES") + if times_tok: + self._expect("RBRACKET") + dim = c_ast.ID(times_tok.value, self._tok_coord(times_tok)) + return make_array_decl(dim, []) + + dim = None + if self._starts_expression(): + dim = self._parse_assignment_expression() + self._expect("RBRACKET") + return make_array_decl(dim, []) + + # BNF: function_decl : '(' parameter_type_list_opt | identifier_list_opt ')' + def _parse_function_decl(self, base_decl: c_ast.Node) -> c_ast.Node: + self._expect("LPAREN") + if self._accept("RPAREN"): + args = None + else: + args = ( + self._parse_parameter_type_list() + if self._starts_declaration() + else self._parse_identifier_list_opt() + ) + self._expect("RPAREN") + + func = c_ast.FuncDecl(args=args, type=None, coord=base_decl.coord) + + if self._peek_type() == "LBRACE": + if func.args is not None: + for param in func.args.params: + if isinstance(param, c_ast.EllipsisParam): + break + name = getattr(param, "name", None) + if name: + self._add_identifier(name, param.coord) + + return func + + # BNF: pointer : '*' type_qualifier_list? pointer? + def _parse_pointer(self) -> Optional[c_ast.Node]: + stars = [] + times_tok = self._accept("TIMES") + while times_tok: + quals = self._parse_type_qualifier_list() or [] + stars.append((quals, self._tok_coord(times_tok))) + times_tok = self._accept("TIMES") + + if not stars: + return None + + ptr = None + for quals, coord in stars: + ptr = c_ast.PtrDecl(quals=quals, type=ptr, coord=coord) + return ptr + + # BNF: parameter_type_list : parameter_list (',' ELLIPSIS)? + def _parse_parameter_type_list(self) -> c_ast.ParamList: + params = self._parse_parameter_list() + if self._peek_type() == "COMMA" and self._peek_type(2) == "ELLIPSIS": + self._advance() + ell_tok = self._advance() + params.params.append(c_ast.EllipsisParam(self._tok_coord(ell_tok))) + return params + + # BNF: parameter_list : parameter_declaration (',' parameter_declaration)* + def _parse_parameter_list(self) -> c_ast.ParamList: + first = self._parse_parameter_declaration() + params = c_ast.ParamList([first], first.coord) + while self._peek_type() == "COMMA" and self._peek_type(2) != "ELLIPSIS": + self._advance() + params.params.append(self._parse_parameter_declaration()) + return params + + # BNF: parameter_declaration : declaration_specifiers declarator? + # | declaration_specifiers abstract_declarator_opt + def _parse_parameter_declaration(self) -> c_ast.Node: + spec, _, spec_coord = self._parse_declaration_specifiers(allow_no_type=True) + + if not spec["type"]: + spec["type"] = [c_ast.IdentifierType(["int"], coord=spec_coord)] + + if self._starts_declarator(): + decl, is_named = self._parse_any_declarator( + allow_abstract=True, typeid_paren_as_abstract=True + ) + if is_named: + return self._build_declarations( + spec=spec, decls=[dict(decl=decl, init=None, bitsize=None)] + )[0] + return self._build_parameter_declaration(spec, decl, spec_coord) + + decl = self._parse_abstract_declarator_opt() + return self._build_parameter_declaration(spec, decl, spec_coord) + + def _build_parameter_declaration( + self, spec: "_DeclSpec", decl: Optional[c_ast.Node], spec_coord: Optional[Coord] + ) -> c_ast.Node: + if ( + len(spec["type"]) > 1 + and len(spec["type"][-1].names) == 1 + and self._is_type_in_scope(spec["type"][-1].names[0]) + ): + return self._build_declarations( + spec=spec, decls=[dict(decl=decl, init=None, bitsize=None)] + )[0] + + decl = c_ast.Typename( + name="", + quals=spec["qual"], + align=None, + type=decl or c_ast.TypeDecl(None, None, None, None), + coord=spec_coord, + ) + return self._fix_decl_name_type(decl, spec["type"]) + + # BNF: identifier_list_opt : identifier_list | empty + def _parse_identifier_list_opt(self) -> Optional[c_ast.Node]: + if self._peek_type() == "RPAREN": + return None + return self._parse_identifier_list() + + # BNF: identifier_list : identifier (',' identifier)* + def _parse_identifier_list(self) -> c_ast.Node: + first = self._parse_identifier() + params = c_ast.ParamList([first], first.coord) + while self._accept("COMMA"): + params.params.append(self._parse_identifier()) + return params + + # ------------------------------------------------------------------ + # Abstract declarators + # ------------------------------------------------------------------ + # BNF: type_name : specifier_qualifier_list abstract_declarator_opt + def _parse_type_name(self) -> c_ast.Typename: + spec = self._parse_specifier_qualifier_list() + decl = self._parse_abstract_declarator_opt() + + coord = None + if decl is not None: + coord = decl.coord + elif spec["type"]: + coord = spec["type"][0].coord + + typename = c_ast.Typename( + name="", + quals=spec["qual"][:], + align=None, + type=decl or c_ast.TypeDecl(None, None, None, None), + coord=coord, + ) + return cast(c_ast.Typename, self._fix_decl_name_type(typename, spec["type"])) + + # BNF: abstract_declarator_opt : pointer? direct_abstract_declarator? + def _parse_abstract_declarator_opt(self) -> Optional[c_ast.Node]: + if self._peek_type() == "TIMES": + ptr = self._parse_pointer() + if self._starts_direct_abstract_declarator(): + decl = self._parse_direct_abstract_declarator() + else: + decl = c_ast.TypeDecl(None, None, None, None) + assert ptr is not None + return self._type_modify_decl(decl, ptr) + + if self._starts_direct_abstract_declarator(): + return self._parse_direct_abstract_declarator() + + return None + + # BNF: direct_abstract_declarator : '(' parameter_type_list_opt ')' + # | '(' abstract_declarator ')' + # | '[' ... ']' + def _parse_direct_abstract_declarator(self) -> c_ast.Node: + lparen_tok = self._accept("LPAREN") + if lparen_tok: + if self._starts_declaration() or self._peek_type() == "RPAREN": + params = self._parse_parameter_type_list_opt() + self._expect("RPAREN") + decl = c_ast.FuncDecl( + args=params, + type=c_ast.TypeDecl(None, None, None, None), + coord=self._tok_coord(lparen_tok), + ) + else: + decl = self._parse_abstract_declarator_opt() + self._expect("RPAREN") + assert decl is not None + elif self._peek_type() == "LBRACKET": + decl = self._parse_abstract_array_base() + else: + self._parse_error("Invalid abstract declarator", self.clex.filename) + + return self._parse_decl_suffixes(decl) + + # BNF: parameter_type_list_opt : parameter_type_list | empty + def _parse_parameter_type_list_opt(self) -> Optional[c_ast.ParamList]: + if self._peek_type() == "RPAREN": + return None + return self._parse_parameter_type_list() + + # BNF: abstract_array_base : '[' array_specifiers? assignment_expression? ']' + def _parse_abstract_array_base(self) -> c_ast.Node: + return self._parse_array_decl_common( + base_type=c_ast.TypeDecl(None, None, None, None), coord=None + ) + + # ------------------------------------------------------------------ + # Statements + # ------------------------------------------------------------------ + # BNF: statement : labeled_statement | compound_statement + # | selection_statement | iteration_statement + # | jump_statement | expression_statement + # | static_assert | pppragma_directive + def _parse_statement(self) -> c_ast.Node | List[c_ast.Node]: + tok_type = self._peek_type() + match tok_type: + case "CASE" | "DEFAULT": + return self._parse_labeled_statement() + case "ID" if self._peek_type(2) == "COLON": + return self._parse_labeled_statement() + case "LBRACE": + return self._parse_compound_statement() + case "IF" | "SWITCH": + return self._parse_selection_statement() + case "WHILE" | "DO" | "FOR": + return self._parse_iteration_statement() + case "GOTO" | "BREAK" | "CONTINUE" | "RETURN": + return self._parse_jump_statement() + case "PPPRAGMA" | "_PRAGMA": + return self._parse_pppragma_directive() + case "_STATIC_ASSERT": + return self._parse_static_assert() + case _: + return self._parse_expression_statement() + + # BNF: pragmacomp_or_statement : pppragma_directive* statement + def _parse_pragmacomp_or_statement(self) -> c_ast.Node | List[c_ast.Node]: + if self._peek_type() in {"PPPRAGMA", "_PRAGMA"}: + pragmas = self._parse_pppragma_directive_list() + stmt = self._parse_statement() + return c_ast.Compound(block_items=pragmas + [stmt], coord=pragmas[0].coord) + return self._parse_statement() + + # BNF: block_item : declaration | statement + def _parse_block_item(self) -> c_ast.Node | List[c_ast.Node]: + if self._starts_declaration(): + return self._parse_declaration() + return self._parse_statement() + + # BNF: block_item_list : block_item+ + def _parse_block_item_list(self) -> List[c_ast.Node]: + items = [] + while self._peek_type() not in {"RBRACE", None}: + item = self._parse_block_item() + if isinstance(item, list): + if item == [None]: + continue + items.extend(item) + else: + items.append(item) + return items + + # BNF: compound_statement : '{' block_item_list? '}' + def _parse_compound_statement(self) -> c_ast.Node: + lbrace_tok = self._expect("LBRACE") + if self._accept("RBRACE"): + return c_ast.Compound(block_items=None, coord=self._tok_coord(lbrace_tok)) + block_items = self._parse_block_item_list() + self._expect("RBRACE") + return c_ast.Compound( + block_items=block_items, coord=self._tok_coord(lbrace_tok) + ) + + # BNF: labeled_statement : ID ':' statement + # | CASE constant_expression ':' statement + # | DEFAULT ':' statement + def _parse_labeled_statement(self) -> c_ast.Node: + tok_type = self._peek_type() + match tok_type: + case "ID": + name_tok = self._advance() + self._expect("COLON") + if self._starts_statement(): + stmt = self._parse_pragmacomp_or_statement() + else: + stmt = c_ast.EmptyStatement(self._tok_coord(name_tok)) + return c_ast.Label(name_tok.value, stmt, self._tok_coord(name_tok)) + case "CASE": + case_tok = self._advance() + expr = self._parse_constant_expression() + self._expect("COLON") + if self._starts_statement(): + stmt = self._parse_pragmacomp_or_statement() + else: + stmt = c_ast.EmptyStatement(self._tok_coord(case_tok)) + return c_ast.Case(expr, [stmt], self._tok_coord(case_tok)) + case "DEFAULT": + def_tok = self._advance() + self._expect("COLON") + if self._starts_statement(): + stmt = self._parse_pragmacomp_or_statement() + else: + stmt = c_ast.EmptyStatement(self._tok_coord(def_tok)) + return c_ast.Default([stmt], self._tok_coord(def_tok)) + case _: + self._parse_error("Invalid labeled statement", self.clex.filename) + + # BNF: selection_statement : IF '(' expression ')' statement (ELSE statement)? + # | SWITCH '(' expression ')' statement + def _parse_selection_statement(self) -> c_ast.Node: + tok = self._advance() + match tok.type: + case "IF": + self._expect("LPAREN") + cond = self._parse_expression() + self._expect("RPAREN") + then_stmt = self._parse_pragmacomp_or_statement() + if self._accept("ELSE"): + else_stmt = self._parse_pragmacomp_or_statement() + return c_ast.If(cond, then_stmt, else_stmt, self._tok_coord(tok)) + return c_ast.If(cond, then_stmt, None, self._tok_coord(tok)) + case "SWITCH": + self._expect("LPAREN") + expr = self._parse_expression() + self._expect("RPAREN") + stmt = self._parse_pragmacomp_or_statement() + return fix_switch_cases(c_ast.Switch(expr, stmt, self._tok_coord(tok))) + case _: + self._parse_error("Invalid selection statement", self._tok_coord(tok)) + + # BNF: iteration_statement : WHILE '(' expression ')' statement + # | DO statement WHILE '(' expression ')' ';' + # | FOR '(' (declaration | expression_opt) ';' + # expression_opt ';' expression_opt ')' statement + def _parse_iteration_statement(self) -> c_ast.Node: + tok = self._advance() + match tok.type: + case "WHILE": + self._expect("LPAREN") + cond = self._parse_expression() + self._expect("RPAREN") + stmt = self._parse_pragmacomp_or_statement() + return c_ast.While(cond, stmt, self._tok_coord(tok)) + case "DO": + stmt = self._parse_pragmacomp_or_statement() + self._expect("WHILE") + self._expect("LPAREN") + cond = self._parse_expression() + self._expect("RPAREN") + self._expect("SEMI") + return c_ast.DoWhile(cond, stmt, self._tok_coord(tok)) + case "FOR": + self._expect("LPAREN") + if self._starts_declaration(): + decls = self._parse_declaration() + init = c_ast.DeclList(decls, self._tok_coord(tok)) + cond = self._parse_expression_opt() + self._expect("SEMI") + next_expr = self._parse_expression_opt() + self._expect("RPAREN") + stmt = self._parse_pragmacomp_or_statement() + return c_ast.For(init, cond, next_expr, stmt, self._tok_coord(tok)) + + init = self._parse_expression_opt() + self._expect("SEMI") + cond = self._parse_expression_opt() + self._expect("SEMI") + next_expr = self._parse_expression_opt() + self._expect("RPAREN") + stmt = self._parse_pragmacomp_or_statement() + return c_ast.For(init, cond, next_expr, stmt, self._tok_coord(tok)) + case _: + self._parse_error("Invalid iteration statement", self._tok_coord(tok)) + + # BNF: jump_statement : GOTO ID ';' | BREAK ';' | CONTINUE ';' + # | RETURN expression? ';' + def _parse_jump_statement(self) -> c_ast.Node: + tok = self._advance() + match tok.type: + case "GOTO": + name_tok = self._expect("ID") + self._expect("SEMI") + return c_ast.Goto(name_tok.value, self._tok_coord(tok)) + case "BREAK": + self._expect("SEMI") + return c_ast.Break(self._tok_coord(tok)) + case "CONTINUE": + self._expect("SEMI") + return c_ast.Continue(self._tok_coord(tok)) + case "RETURN": + if self._accept("SEMI"): + return c_ast.Return(None, self._tok_coord(tok)) + expr = self._parse_expression() + self._expect("SEMI") + return c_ast.Return(expr, self._tok_coord(tok)) + case _: + self._parse_error("Invalid jump statement", self._tok_coord(tok)) + + # BNF: expression_statement : expression_opt ';' + def _parse_expression_statement(self) -> c_ast.Node: + expr = self._parse_expression_opt() + semi_tok = self._expect("SEMI") + if expr is None: + return c_ast.EmptyStatement(self._tok_coord(semi_tok)) + return expr + + # ------------------------------------------------------------------ + # Expressions + # ------------------------------------------------------------------ + # BNF: expression_opt : expression | empty + def _parse_expression_opt(self) -> Optional[c_ast.Node]: + if self._starts_expression(): + return self._parse_expression() + return None + + # BNF: expression : assignment_expression (',' assignment_expression)* + def _parse_expression(self) -> c_ast.Node: + expr = self._parse_assignment_expression() + if not self._accept("COMMA"): + return expr + exprs = [expr, self._parse_assignment_expression()] + while self._accept("COMMA"): + exprs.append(self._parse_assignment_expression()) + return c_ast.ExprList(exprs, expr.coord) + + # BNF: assignment_expression : conditional_expression + # | unary_expression assignment_op assignment_expression + def _parse_assignment_expression(self) -> c_ast.Node: + if self._peek_type() == "LPAREN" and self._peek_type(2) == "LBRACE": + self._advance() + comp = self._parse_compound_statement() + self._expect("RPAREN") + return comp + + expr = self._parse_conditional_expression() + if self._is_assignment_op(): + op = self._advance().value + rhs = self._parse_assignment_expression() + return c_ast.Assignment(op, expr, rhs, expr.coord) + return expr + + # BNF: conditional_expression : binary_expression + # | binary_expression '?' expression ':' conditional_expression + def _parse_conditional_expression(self) -> c_ast.Node: + expr = self._parse_binary_expression() + if self._accept("CONDOP"): + iftrue = self._parse_expression() + self._expect("COLON") + iffalse = self._parse_conditional_expression() + return c_ast.TernaryOp(expr, iftrue, iffalse, expr.coord) + return expr + + # BNF: binary_expression : cast_expression (binary_op cast_expression)* + def _parse_binary_expression( + self, min_prec: int = 0, lhs: Optional[c_ast.Node] = None + ) -> c_ast.Node: + if lhs is None: + lhs = self._parse_cast_expression() + + while True: + tok = self._peek() + if tok is None or tok.type not in _BINARY_PRECEDENCE: + break + prec = _BINARY_PRECEDENCE[tok.type] + if prec < min_prec: + break + + op = tok.value + self._advance() + rhs = self._parse_cast_expression() + + while True: + next_tok = self._peek() + if next_tok is None or next_tok.type not in _BINARY_PRECEDENCE: + break + next_prec = _BINARY_PRECEDENCE[next_tok.type] + if next_prec > prec: + rhs = self._parse_binary_expression(next_prec, rhs) + else: + break + + lhs = c_ast.BinaryOp(op, lhs, rhs, lhs.coord) + + return lhs + + # BNF: cast_expression : '(' type_name ')' cast_expression + # | unary_expression + def _parse_cast_expression(self) -> c_ast.Node: + result = self._try_parse_paren_type_name() + if result is not None: + typ, mark, lparen_tok = result + if self._peek_type() == "LBRACE": + # (type){...} is a compound literal, not a cast. Examples: + # (int){1} -> compound literal, handled in postfix + # (int) x -> cast, handled below + self._reset(mark) + else: + expr = self._parse_cast_expression() + return c_ast.Cast(typ, expr, self._tok_coord(lparen_tok)) + return self._parse_unary_expression() + + # BNF: unary_expression : postfix_expression + # | '++' unary_expression + # | '--' unary_expression + # | unary_op cast_expression + # | 'sizeof' unary_expression + # | 'sizeof' '(' type_name ')' + # | '_Alignof' '(' type_name ')' + def _parse_unary_expression(self) -> c_ast.Node: + tok_type = self._peek_type() + if tok_type in {"PLUSPLUS", "MINUSMINUS"}: + tok = self._advance() + expr = self._parse_unary_expression() + return c_ast.UnaryOp(tok.value, expr, expr.coord) + + if tok_type in {"AND", "TIMES", "PLUS", "MINUS", "NOT", "LNOT"}: + tok = self._advance() + expr = self._parse_cast_expression() + return c_ast.UnaryOp(tok.value, expr, expr.coord) + + if tok_type == "SIZEOF": + tok = self._advance() + result = self._try_parse_paren_type_name() + if result is not None: + typ, _, _ = result + return c_ast.UnaryOp(tok.value, typ, self._tok_coord(tok)) + expr = self._parse_unary_expression() + return c_ast.UnaryOp(tok.value, expr, self._tok_coord(tok)) + + if tok_type == "_ALIGNOF": + tok = self._advance() + self._expect("LPAREN") + typ = self._parse_type_name() + self._expect("RPAREN") + return c_ast.UnaryOp(tok.value, typ, self._tok_coord(tok)) + + return self._parse_postfix_expression() + + # BNF: postfix_expression : primary_expression postfix_suffix* + # | '(' type_name ')' '{' initializer_list ','? '}' + def _parse_postfix_expression(self) -> c_ast.Node: + result = self._try_parse_paren_type_name() + if result is not None: + typ, mark, _ = result + # Disambiguate between casts and compound literals: + # (int) x -> cast + # (int) {1} -> compound literal + if self._accept("LBRACE"): + init = self._parse_initializer_list() + self._accept("COMMA") + self._expect("RBRACE") + return c_ast.CompoundLiteral(typ, init) + else: + self._reset(mark) + + expr = self._parse_primary_expression() + while True: + if self._accept("LBRACKET"): + sub = self._parse_expression() + self._expect("RBRACKET") + expr = c_ast.ArrayRef(expr, sub, expr.coord) + continue + if self._accept("LPAREN"): + if self._peek_type() == "RPAREN": + self._advance() + args = None + else: + args = self._parse_argument_expression_list() + self._expect("RPAREN") + expr = c_ast.FuncCall(expr, args, expr.coord) + continue + if self._peek_type() in {"PERIOD", "ARROW"}: + op_tok = self._advance() + name_tok = self._advance() + if name_tok.type not in {"ID", "TYPEID"}: + self._parse_error( + "Invalid struct reference", self._tok_coord(name_tok) + ) + field = c_ast.ID(name_tok.value, self._tok_coord(name_tok)) + expr = c_ast.StructRef(expr, op_tok.value, field, expr.coord) + continue + if self._peek_type() in {"PLUSPLUS", "MINUSMINUS"}: + tok = self._advance() + expr = c_ast.UnaryOp("p" + tok.value, expr, expr.coord) + continue + break + return expr + + # BNF: primary_expression : ID | constant | string_literal + # | '(' expression ')' | offsetof + def _parse_primary_expression(self) -> c_ast.Node: + tok_type = self._peek_type() + if tok_type == "ID": + return self._parse_identifier() + if ( + tok_type in _INT_CONST + or tok_type in _FLOAT_CONST + or tok_type in _CHAR_CONST + ): + return self._parse_constant() + if tok_type in _STRING_LITERAL: + return self._parse_unified_string_literal() + if tok_type in _WSTR_LITERAL: + return self._parse_unified_wstring_literal() + if tok_type == "LPAREN": + self._advance() + expr = self._parse_expression() + self._expect("RPAREN") + return expr + if tok_type == "OFFSETOF": + off_tok = self._advance() + self._expect("LPAREN") + typ = self._parse_type_name() + self._expect("COMMA") + designator = self._parse_offsetof_member_designator() + self._expect("RPAREN") + coord = self._tok_coord(off_tok) + return c_ast.FuncCall( + c_ast.ID(off_tok.value, coord), + c_ast.ExprList([typ, designator], coord), + coord, + ) + + self._parse_error("Invalid expression", self.clex.filename) + + # BNF: offsetof_member_designator : identifier_or_typeid + # ('.' identifier_or_typeid | '[' expression ']')* + def _parse_offsetof_member_designator(self) -> c_ast.Node: + node = self._parse_identifier_or_typeid() + while True: + if self._accept("PERIOD"): + field = self._parse_identifier_or_typeid() + node = c_ast.StructRef(node, ".", field, node.coord) + continue + if self._accept("LBRACKET"): + expr = self._parse_expression() + self._expect("RBRACKET") + node = c_ast.ArrayRef(node, expr, node.coord) + continue + break + return node + + # BNF: argument_expression_list : assignment_expression (',' assignment_expression)* + def _parse_argument_expression_list(self) -> c_ast.Node: + expr = self._parse_assignment_expression() + exprs = [expr] + while self._accept("COMMA"): + exprs.append(self._parse_assignment_expression()) + return c_ast.ExprList(exprs, expr.coord) + + # BNF: constant_expression : conditional_expression + def _parse_constant_expression(self) -> c_ast.Node: + return self._parse_conditional_expression() + + # ------------------------------------------------------------------ + # Terminals + # ------------------------------------------------------------------ + # BNF: identifier : ID + def _parse_identifier(self) -> c_ast.Node: + tok = self._expect("ID") + return c_ast.ID(tok.value, self._tok_coord(tok)) + + # BNF: identifier_or_typeid : ID | TYPEID + def _parse_identifier_or_typeid(self) -> c_ast.Node: + tok = self._advance() + if tok.type not in {"ID", "TYPEID"}: + self._parse_error("Expected identifier", self._tok_coord(tok)) + return c_ast.ID(tok.value, self._tok_coord(tok)) + + # BNF: constant : INT_CONST | FLOAT_CONST | CHAR_CONST + def _parse_constant(self) -> c_ast.Node: + tok = self._advance() + if tok.type in _INT_CONST: + u_count = 0 + l_count = 0 + for ch in tok.value[-3:]: + if ch in ("l", "L"): + l_count += 1 + elif ch in ("u", "U"): + u_count += 1 + if u_count > 1: + raise ValueError("Constant cannot have more than one u/U suffix.") + if l_count > 2: + raise ValueError("Constant cannot have more than two l/L suffix.") + prefix = "unsigned " * u_count + "long " * l_count + return c_ast.Constant(prefix + "int", tok.value, self._tok_coord(tok)) + + if tok.type in _FLOAT_CONST: + if tok.value[-1] in ("f", "F"): + t = "float" + elif tok.value[-1] in ("l", "L"): + t = "long double" + else: + t = "double" + return c_ast.Constant(t, tok.value, self._tok_coord(tok)) + + if tok.type in _CHAR_CONST: + return c_ast.Constant("char", tok.value, self._tok_coord(tok)) + + self._parse_error("Invalid constant", self._tok_coord(tok)) + + # BNF: unified_string_literal : STRING_LITERAL+ + def _parse_unified_string_literal(self) -> c_ast.Node: + tok = self._expect("STRING_LITERAL") + node = c_ast.Constant("string", tok.value, self._tok_coord(tok)) + while self._peek_type() == "STRING_LITERAL": + tok2 = self._advance() + node.value = node.value[:-1] + tok2.value[1:] + return node + + # BNF: unified_wstring_literal : WSTRING_LITERAL+ + def _parse_unified_wstring_literal(self) -> c_ast.Node: + tok = self._advance() + if tok.type not in _WSTR_LITERAL: + self._parse_error("Invalid string literal", self._tok_coord(tok)) + node = c_ast.Constant("string", tok.value, self._tok_coord(tok)) + while self._peek_type() in _WSTR_LITERAL: + tok2 = self._advance() + node.value = node.value.rstrip()[:-1] + tok2.value[2:] + return node + + # ------------------------------------------------------------------ + # Initializers + # ------------------------------------------------------------------ + # BNF: initializer : assignment_expression + # | '{' initializer_list ','? '}' + # | '{' '}' + def _parse_initializer(self) -> c_ast.Node: + lbrace_tok = self._accept("LBRACE") + if lbrace_tok: + if self._accept("RBRACE"): + return c_ast.InitList([], self._tok_coord(lbrace_tok)) + init_list = self._parse_initializer_list() + self._accept("COMMA") + self._expect("RBRACE") + return init_list + + return self._parse_assignment_expression() + + # BNF: initializer_list : initializer_item (',' initializer_item)* ','? + def _parse_initializer_list(self) -> c_ast.Node: + items = [self._parse_initializer_item()] + while self._accept("COMMA"): + if self._peek_type() == "RBRACE": + break + items.append(self._parse_initializer_item()) + return c_ast.InitList(items, items[0].coord) + + # BNF: initializer_item : designation? initializer + def _parse_initializer_item(self) -> c_ast.Node: + designation = None + if self._peek_type() in {"LBRACKET", "PERIOD"}: + designation = self._parse_designation() + init = self._parse_initializer() + if designation is not None: + return c_ast.NamedInitializer(designation, init) + return init + + # BNF: designation : designator_list '=' + def _parse_designation(self) -> List[c_ast.Node]: + designators = self._parse_designator_list() + self._expect("EQUALS") + return designators + + # BNF: designator_list : designator+ + def _parse_designator_list(self) -> List[c_ast.Node]: + designators = [] + while self._peek_type() in {"LBRACKET", "PERIOD"}: + designators.append(self._parse_designator()) + return designators + + # BNF: designator : '[' constant_expression ']' + # | '.' identifier_or_typeid + def _parse_designator(self) -> c_ast.Node: + if self._accept("LBRACKET"): + expr = self._parse_constant_expression() + self._expect("RBRACKET") + return expr + if self._accept("PERIOD"): + return self._parse_identifier_or_typeid() + self._parse_error("Invalid designator", self.clex.filename) + + # ------------------------------------------------------------------ + # Preprocessor-like directives + # ------------------------------------------------------------------ + # BNF: pp_directive : '#' ... (unsupported) + def _parse_pp_directive(self) -> NoReturn: + tok = self._expect("PPHASH") + self._parse_error("Directives not supported yet", self._tok_coord(tok)) + + # BNF: pppragma_directive : PPPRAGMA PPPRAGMASTR? + # | _PRAGMA '(' string_literal ')' + def _parse_pppragma_directive(self) -> c_ast.Node: + if self._peek_type() == "PPPRAGMA": + tok = self._advance() + if self._peek_type() == "PPPRAGMASTR": + str_tok = self._advance() + return c_ast.Pragma(str_tok.value, self._tok_coord(str_tok)) + return c_ast.Pragma("", self._tok_coord(tok)) + + if self._peek_type() == "_PRAGMA": + tok = self._advance() + lparen = self._expect("LPAREN") + literal = self._parse_unified_string_literal() + self._expect("RPAREN") + return c_ast.Pragma(literal, self._tok_coord(lparen)) + + self._parse_error("Invalid pragma", self.clex.filename) + + # BNF: pppragma_directive_list : pppragma_directive+ + def _parse_pppragma_directive_list(self) -> List[c_ast.Node]: + pragmas = [] + while self._peek_type() in {"PPPRAGMA", "_PRAGMA"}: + pragmas.append(self._parse_pppragma_directive()) + return pragmas + + # BNF: static_assert : _STATIC_ASSERT '(' constant_expression (',' string_literal)? ')' + def _parse_static_assert(self) -> List[c_ast.Node]: + tok = self._expect("_STATIC_ASSERT") + self._expect("LPAREN") + cond = self._parse_constant_expression() + msg = None + if self._accept("COMMA"): + msg = self._parse_unified_string_literal() + self._expect("RPAREN") + return [c_ast.StaticAssert(cond, msg, self._tok_coord(tok))] + + +_ASSIGNMENT_OPS = { + "EQUALS", + "XOREQUAL", + "TIMESEQUAL", + "DIVEQUAL", + "MODEQUAL", + "PLUSEQUAL", + "MINUSEQUAL", + "LSHIFTEQUAL", + "RSHIFTEQUAL", + "ANDEQUAL", + "OREQUAL", +} + +# Precedence of operators (lower number = weather binding) +# If this changes, c_generator.CGenerator.precedence_map needs to change as +# well +_BINARY_PRECEDENCE = { + "LOR": 0, + "LAND": 1, + "OR": 2, + "XOR": 3, + "AND": 4, + "EQ": 5, + "NE": 5, + "GT": 6, + "GE": 6, + "LT": 6, + "LE": 6, + "RSHIFT": 7, + "LSHIFT": 7, + "PLUS": 8, + "MINUS": 8, + "TIMES": 9, + "DIVIDE": 9, + "MOD": 9, +} + +_STORAGE_CLASS = {"AUTO", "REGISTER", "STATIC", "EXTERN", "TYPEDEF", "_THREAD_LOCAL"} + +_FUNCTION_SPEC = {"INLINE", "_NORETURN"} + +_TYPE_QUALIFIER = {"CONST", "RESTRICT", "VOLATILE", "_ATOMIC"} + +_TYPE_SPEC_SIMPLE = { + "VOID", + "_BOOL", + "CHAR", + "SHORT", + "INT", + "LONG", + "FLOAT", + "DOUBLE", + "_COMPLEX", + "SIGNED", + "UNSIGNED", + "__INT128", +} + +_DECL_START = ( + _STORAGE_CLASS + | _FUNCTION_SPEC + | _TYPE_QUALIFIER + | _TYPE_SPEC_SIMPLE + | {"TYPEID", "STRUCT", "UNION", "ENUM", "_ALIGNAS", "_ATOMIC"} +) + +_EXPR_START = { + "ID", + "LPAREN", + "PLUSPLUS", + "MINUSMINUS", + "PLUS", + "MINUS", + "TIMES", + "AND", + "NOT", + "LNOT", + "SIZEOF", + "_ALIGNOF", + "OFFSETOF", +} + +_INT_CONST = { + "INT_CONST_DEC", + "INT_CONST_OCT", + "INT_CONST_HEX", + "INT_CONST_BIN", + "INT_CONST_CHAR", +} + +_FLOAT_CONST = {"FLOAT_CONST", "HEX_FLOAT_CONST"} + +_CHAR_CONST = { + "CHAR_CONST", + "WCHAR_CONST", + "U8CHAR_CONST", + "U16CHAR_CONST", + "U32CHAR_CONST", +} + +_STRING_LITERAL = {"STRING_LITERAL"} + +_WSTR_LITERAL = { + "WSTRING_LITERAL", + "U8STRING_LITERAL", + "U16STRING_LITERAL", + "U32STRING_LITERAL", +} + +_STARTS_EXPRESSION = ( + _EXPR_START + | _INT_CONST + | _FLOAT_CONST + | _CHAR_CONST + | _STRING_LITERAL + | _WSTR_LITERAL +) + +_STARTS_STATEMENT = { + "LBRACE", + "IF", + "SWITCH", + "WHILE", + "DO", + "FOR", + "GOTO", + "BREAK", + "CONTINUE", + "RETURN", + "CASE", + "DEFAULT", + "PPPRAGMA", + "_PRAGMA", + "_STATIC_ASSERT", + "SEMI", +} + + +class _TokenStream: + """Wraps a lexer to provide convenient, buffered access to the underlying + token stream. The lexer is expected to be initialized with the input + string already. + """ + + def __init__(self, lexer: CLexer) -> None: + self._lexer = lexer + self._buffer: List[Optional[_Token]] = [] + self._index = 0 + + def peek(self, k: int = 1) -> Optional[_Token]: + """Peek at the k-th next token in the stream, without consuming it. + + Examples: + k=1 returns the immediate next token. + k=2 returns the token after that. + """ + if k <= 0: + return None + self._fill(k) + return self._buffer[self._index + k - 1] + + def next(self) -> Optional[_Token]: + """Consume a single token and return it.""" + self._fill(1) + tok = self._buffer[self._index] + self._index += 1 + return tok + + # The 'mark' and 'reset' methods are useful for speculative parsing with + # backtracking; when the parser needs to examine a sequence of tokens + # and potentially decide to try a different path on the same sequence, it + # can call 'mark' to obtain the current token position, and if the first + # path fails restore the position with `reset(pos)`. + def mark(self) -> int: + return self._index + + def reset(self, mark: int) -> None: + self._index = mark + + def _fill(self, n: int) -> None: + while len(self._buffer) < self._index + n: + tok = self._lexer.token() + self._buffer.append(tok) + if tok is None: + break + + +# Declaration specifiers are represented by a dictionary with entries: +# - qual: a list of type qualifiers +# - storage: a list of storage class specifiers +# - type: a list of type specifiers +# - function: a list of function specifiers +# - alignment: a list of alignment specifiers +class _DeclSpec(TypedDict): + qual: List[Any] + storage: List[Any] + type: List[Any] + function: List[Any] + alignment: List[Any] + + +_DeclSpecKind = Literal["qual", "storage", "type", "function", "alignment"] + + +class _DeclInfo(TypedDict): + # Declarator payloads used by declaration/initializer parsing: + # - decl: the declarator node (may be None for abstract/implicit cases) + # - init: optional initializer expression + # - bitsize: optional bit-field width expression (for struct declarators) + decl: Optional[c_ast.Node] + init: Optional[c_ast.Node] + bitsize: Optional[c_ast.Node] diff --git a/py_modules/pyotp/__init__.py b/py_modules/pyotp/__init__.py new file mode 100644 index 0000000..037f1f5 --- /dev/null +++ b/py_modules/pyotp/__init__.py @@ -0,0 +1,115 @@ +import hashlib +from re import split +from typing import Any, Dict, Sequence +from urllib.parse import parse_qsl, unquote, urlparse + +from . import contrib # noqa:F401 +from .compat import random +from .hotp import HOTP as HOTP +from .otp import OTP as OTP +from .totp import TOTP as TOTP + + +def random_base32(length: int = 32, chars: Sequence[str] = "ABCDEFGHIJKLMNOPQRSTUVWXYZ234567") -> str: + # Note: the otpauth scheme DOES NOT use base32 padding for secret lengths not divisible by 8. + # Some third-party tools have bugs when dealing with such secrets. + # We might consider warning the user when generating a secret of length not divisible by 8. + if length < 32: + raise ValueError("Secrets should be at least 160 bits") + + return "".join(random.choice(chars) for _ in range(length)) + + +def random_hex(length: int = 40, chars: Sequence[str] = "ABCDEF0123456789") -> str: + if length < 40: + raise ValueError("Secrets should be at least 160 bits") + return random_base32(length=length, chars=chars) + + +def parse_uri(uri: str) -> OTP: + """ + Parses the provisioning URI for the OTP; works for either TOTP or HOTP. + + See also: + https://github.com/google/google-authenticator/wiki/Key-Uri-Format + + :param uri: the hotp/totp URI to parse + :returns: OTP object + """ + + # Secret (to be filled in later) + secret = None + + # Encoder (to be filled in later) + encoder = None + + # Digits (to be filled in later) + digits = None + + # Data we'll parse to the correct constructor + otp_data: Dict[str, Any] = {} + + # Parse the URI structure first, then unquote individual components. + # Unquoting the whole URI before splitting the label would turn an + # encoded colon (%3A) inside the issuer or account name into a literal + # ":" and mis-parse the issuer:account separator (see GitHub issue #174). + parsed_uri = urlparse(uri) + + if parsed_uri.scheme != "otpauth": + raise ValueError("Not an otpauth URI") + + # Parse issuer/accountname info. The label uses a literal ":" as the + # issuer:account separator; a percent-encoded "%3A" is a colon that is + # part of the issuer or account name, not a separator. Split on the + # literal ":" while the components are still percent-encoded, then + # unquote each component on its own. + accountinfo_parts = split(":", parsed_uri.path[1:], maxsplit=1) + if len(accountinfo_parts) == 1: + otp_data["name"] = unquote(accountinfo_parts[0]) + else: + otp_data["issuer"] = unquote(accountinfo_parts[0]) + otp_data["name"] = unquote(accountinfo_parts[1]) + + # Parse values + for key, value in parse_qsl(parsed_uri.query): + if key == "secret": + secret = value + elif key == "issuer": + if "issuer" in otp_data and otp_data["issuer"] is not None and otp_data["issuer"] != value: + raise ValueError("If issuer is specified in both label and parameters, it should be equal.") + otp_data["issuer"] = value + elif key == "algorithm": + if value == "SHA1": + otp_data["digest"] = hashlib.sha1 + elif value == "SHA256": + otp_data["digest"] = hashlib.sha256 + elif value == "SHA512": + otp_data["digest"] = hashlib.sha512 + else: + raise ValueError("Invalid value for algorithm, must be SHA1, SHA256 or SHA512") + elif key == "encoder": + encoder = value + elif key == "digits": + digits = int(value) + otp_data["digits"] = digits + elif key == "period": + otp_data["interval"] = int(value) + elif key == "counter": + otp_data["initial_count"] = int(value) + + if encoder != "steam": + if digits is not None and digits not in [6, 7, 8]: + raise ValueError("Digits may only be 6, 7, or 8") + + if not secret: + raise ValueError("No secret found in URI") + + # Create objects + if encoder == "steam": + return contrib.Steam(secret, **otp_data) + if parsed_uri.netloc == "totp": + return TOTP(secret, **otp_data) + elif parsed_uri.netloc == "hotp": + return HOTP(secret, **otp_data) + + raise ValueError("Not a supported OTP type") diff --git a/py_modules/pyotp/compat.py b/py_modules/pyotp/compat.py new file mode 100644 index 0000000..b2f058f --- /dev/null +++ b/py_modules/pyotp/compat.py @@ -0,0 +1,9 @@ +import sys + +# Use secrets module if available (Python version >= 3.6) per PEP 506 +if sys.version_info >= (3, 6): + from secrets import SystemRandom +else: + from random import SystemRandom + +random = SystemRandom() diff --git a/py_modules/pyotp/contrib/__init__.py b/py_modules/pyotp/contrib/__init__.py new file mode 100644 index 0000000..cdd567d --- /dev/null +++ b/py_modules/pyotp/contrib/__init__.py @@ -0,0 +1 @@ +from .steam import Steam # noqa:F401 diff --git a/py_modules/pyotp/contrib/steam.py b/py_modules/pyotp/contrib/steam.py new file mode 100644 index 0000000..35c6cc6 --- /dev/null +++ b/py_modules/pyotp/contrib/steam.py @@ -0,0 +1,40 @@ +import hashlib +from typing import Optional + +from ..totp import TOTP + +STEAM_CHARS = "23456789BCDFGHJKMNPQRTVWXY" # steam's custom alphabet +STEAM_DEFAULT_DIGITS = 5 # Steam TOTP code length + + +class Steam(TOTP): + """ + Steam's custom TOTP. Subclass of `pyotp.totp.TOTP`. + """ + + def __init__( + self, s: str, name: Optional[str] = None, issuer: Optional[str] = None, interval: int = 30, digits: int = 5 + ) -> None: + """ + :param s: secret in base32 format + :param interval: the time interval in seconds for OTP. This defaults to 30. + :param name: account name + :param issuer: issuer + """ + self.interval = interval + super().__init__(s=s, digits=10, digest=hashlib.sha1, name=name, issuer=issuer) + + def generate_otp(self, input: int) -> str: + """ + :param input: the HMAC counter value to use as the OTP input. + Usually either the counter, or the computed integer based on the Unix timestamp + """ + int_code = int(super().generate_otp(input)) + total_chars = len(STEAM_CHARS) + + digits = [] + for _ in range(STEAM_DEFAULT_DIGITS): + digits.append(STEAM_CHARS[int_code % total_chars]) + int_code //= total_chars + + return "".join(digits) diff --git a/py_modules/pyotp/hotp.py b/py_modules/pyotp/hotp.py new file mode 100644 index 0000000..c0276cf --- /dev/null +++ b/py_modules/pyotp/hotp.py @@ -0,0 +1,85 @@ +import hashlib +from typing import Any, Optional + +from . import utils +from .otp import OTP + + +class HOTP(OTP): + """ + Handler for HMAC-based OTP counters. + """ + + def __init__( + self, + s: str, + digits: int = 6, + digest: Any = None, + name: Optional[str] = None, + issuer: Optional[str] = None, + initial_count: int = 0, + ) -> None: + """ + :param s: secret in base32 format + :param initial_count: starting HMAC counter value, defaults to 0 + :param digits: number of integers in the OTP. Some apps expect this to be 6 digits, others support more. + :param digest: digest function to use in the HMAC (expected to be SHA1) + :param name: account name + :param issuer: issuer + """ + if digest is None: + digest = hashlib.sha1 + elif digest in [hashlib.md5, hashlib.shake_128]: + raise ValueError("selected digest function must generate digest size greater than or equals to 18 bytes") + + self.initial_count = initial_count + super().__init__(s=s, digits=digits, digest=digest, name=name, issuer=issuer) + + def at(self, count: int) -> str: + """ + Generates the OTP for the given count. + + :param count: the OTP HMAC counter + :returns: OTP + """ + return self.generate_otp(self.initial_count + count) + + def verify(self, otp: str, counter: int) -> bool: + """ + Verifies the OTP passed in against the current counter OTP. + + :param otp: the OTP to check against + :param counter: the OTP HMAC counter + """ + return utils.strings_equal(str(otp), str(self.at(counter))) + + def provisioning_uri( + self, + name: Optional[str] = None, + initial_count: Optional[int] = None, + issuer_name: Optional[str] = None, + **kwargs, + ) -> str: + """ + Returns the provisioning URI for the OTP. This can then be + encoded in a QR Code and used to provision an OTP app like + Google Authenticator. + + See also: + https://github.com/google/google-authenticator/wiki/Key-Uri-Format + + :param name: name of the user account + :param initial_count: starting HMAC counter value, defaults to 0 + :param issuer_name: the name of the OTP issuer; this will be the + organization title of the OTP entry in Authenticator + :returns: provisioning URI + """ + return utils.build_uri( + self.secret, + name=name if name else self.name, + initial_count=initial_count if initial_count else self.initial_count, + issuer=issuer_name if issuer_name else self.issuer, + algorithm=self.digest().name, + digits=self.digits, + **kwargs, + ) diff --git a/py_modules/pyotp/otp.py b/py_modules/pyotp/otp.py new file mode 100644 index 0000000..6420018 --- /dev/null +++ b/py_modules/pyotp/otp.py @@ -0,0 +1,72 @@ +import base64 +import hashlib +import hmac +from typing import Any, Optional + + +class OTP(object): + """ + Base class for OTP handlers. + """ + + def __init__( + self, + s: str, + digits: int = 6, + digest: Any = hashlib.sha1, + name: Optional[str] = None, + issuer: Optional[str] = None, + ) -> None: + self.digits = digits + if digits > 10: + raise ValueError("digits must be no greater than 10") + self.digest = digest + if digest in [hashlib.md5, hashlib.shake_128]: + raise ValueError("selected digest function must generate digest size greater than or equals to 18 bytes") + self.secret = s + self.name = name or "Secret" + self.issuer = issuer + + def generate_otp(self, input: int) -> str: + """ + :param input: the HMAC counter value to use as the OTP input. + Usually either the counter, or the computed integer based on the Unix timestamp + """ + if input < 0: + raise ValueError("input must be positive integer") + hasher = hmac.new(self.byte_secret(), self.int_to_bytestring(input), self.digest) + if hasher.digest_size < 18: + raise ValueError("digest size is lower than 18 bytes, which will trigger error on otp generation") + hmac_hash = bytearray(hasher.digest()) + offset = hmac_hash[-1] & 0xF + code = ( + (hmac_hash[offset] & 0x7F) << 24 + | (hmac_hash[offset + 1] & 0xFF) << 16 + | (hmac_hash[offset + 2] & 0xFF) << 8 + | (hmac_hash[offset + 3] & 0xFF) + ) + str_code = str(10_000_000_000 + (code % 10**self.digits)) + return str_code[-self.digits :] + + def byte_secret(self) -> bytes: + secret = self.secret + missing_padding = len(secret) % 8 + if missing_padding != 0: + secret += "=" * (8 - missing_padding) + return base64.b32decode(secret, casefold=True) + + @staticmethod + def int_to_bytestring(i: int, padding: int = 8) -> bytes: + """ + Turns an integer to the OATH specified + bytestring, which is fed to the HMAC + along with the secret + """ + result = bytearray() + while i != 0: + result.append(i & 0xFF) + i >>= 8 + # It's necessary to convert the final result from bytearray to bytes + # because the hmac functions in python 2.6 and 3.3 don't work with + # bytearray + return bytes(bytearray(reversed(result)).rjust(padding, b"\0")) diff --git a/py_modules/pyotp/py.typed b/py_modules/pyotp/py.typed new file mode 100644 index 0000000..e69de29 diff --git a/py_modules/pyotp/totp.py b/py_modules/pyotp/totp.py new file mode 100644 index 0000000..9908d55 --- /dev/null +++ b/py_modules/pyotp/totp.py @@ -0,0 +1,118 @@ +import calendar +import datetime +import hashlib +import time +from typing import Any, Optional, Union + +from . import utils +from .otp import OTP + + +class TOTP(OTP): + """ + Handler for time-based OTP counters. + """ + + def __init__( + self, + s: str, + digits: int = 6, + digest: Any = None, + name: Optional[str] = None, + issuer: Optional[str] = None, + interval: int = 30, + ) -> None: + """ + :param s: secret in base32 format + :param digits: number of integers in the OTP. Some apps expect this to be 6 digits, others support more. + :param digest: digest function to use in the HMAC (expected to be SHA1) + :param name: account name + :param issuer: issuer + :param interval: the time interval in seconds for OTP. This defaults to 30. + """ + if digest is None: + digest = hashlib.sha1 + elif digest in [hashlib.md5, hashlib.shake_128]: + raise ValueError("selected digest function must generate digest size greater than or equals to 18 bytes") + + self.interval = interval + super().__init__(s=s, digits=digits, digest=digest, name=name, issuer=issuer) + + def at(self, for_time: Union[int, datetime.datetime], counter_offset: int = 0) -> str: + """ + Accepts either a Unix timestamp integer or a datetime object. + + To get the time until the next timecode change (seconds until the current OTP expires), use this instead: + + .. code:: python + + totp = pyotp.TOTP(...) + time_remaining = totp.interval - datetime.datetime.now().timestamp() % totp.interval + + :param for_time: the time to generate an OTP for + :param counter_offset: the amount of ticks to add to the time counter + :returns: OTP value + """ + if not isinstance(for_time, datetime.datetime): + for_time = datetime.datetime.fromtimestamp(int(for_time)) + return self.generate_otp(self.timecode(for_time) + counter_offset) + + def now(self) -> str: + """ + Generate the current time OTP + + :returns: OTP value + """ + return self.generate_otp(self.timecode(datetime.datetime.now())) + + def verify(self, otp: str, for_time: Optional[datetime.datetime] = None, valid_window: int = 0) -> bool: + """ + Verifies the OTP passed in against the current time OTP. + + :param otp: the OTP to check against + :param for_time: Time to check OTP at (defaults to now) + :param valid_window: extends the validity to this many counter ticks before and after the current one + :returns: True if verification succeeded, False otherwise + """ + if for_time is None: + for_time = datetime.datetime.now() + + if valid_window: + for i in range(-valid_window, valid_window + 1): + if utils.strings_equal(str(otp), str(self.at(for_time, i))): + return True + return False + + return utils.strings_equal(str(otp), str(self.at(for_time))) + + def provisioning_uri(self, name: Optional[str] = None, issuer_name: Optional[str] = None, **kwargs) -> str: + """ + Returns the provisioning URI for the OTP. This can then be + encoded in a QR Code and used to provision an OTP app like + Google Authenticator. + + See also: + https://github.com/google/google-authenticator/wiki/Key-Uri-Format + + """ + return utils.build_uri( + self.secret, + name if name else self.name, + issuer=issuer_name if issuer_name else self.issuer, + algorithm=self.digest().name, + digits=self.digits, + period=self.interval, + **kwargs, + ) + + def timecode(self, for_time: datetime.datetime) -> int: + """ + Accepts either a timezone naive (`for_time.tzinfo is None`) or + a timezone aware datetime as argument and returns the + corresponding counter value (timecode). + + """ + if for_time.tzinfo: + return int(calendar.timegm(for_time.utctimetuple()) / self.interval) + else: + return int(time.mktime(for_time.timetuple()) / self.interval) diff --git a/py_modules/pyotp/utils.py b/py_modules/pyotp/utils.py new file mode 100644 index 0000000..ff14d8f --- /dev/null +++ b/py_modules/pyotp/utils.py @@ -0,0 +1,91 @@ +import unicodedata +from hmac import compare_digest +from typing import Dict, Optional, Union +from urllib.parse import quote, urlencode, urlparse + + +def build_uri( + secret: str, + name: str, + initial_count: Optional[int] = None, + issuer: Optional[str] = None, + algorithm: Optional[str] = None, + digits: Optional[int] = None, + period: Optional[int] = None, + **kwargs, +) -> str: + """ + Returns the provisioning URI for the OTP; works for either TOTP or HOTP. + + This can then be encoded in a QR Code and used to provision the Google + Authenticator app. + + For module-internal use. + + See also: + https://github.com/google/google-authenticator/wiki/Key-Uri-Format + + :param secret: the hotp/totp secret used to generate the URI + :param name: name of the account + :param initial_count: starting counter value, defaults to None. + If none, the OTP type will be assumed as TOTP. + :param issuer: the name of the OTP issuer; this will be the + organization title of the OTP entry in Authenticator + :param algorithm: the algorithm used in the OTP generation. + :param digits: the length of the OTP generated code. + :param period: the number of seconds the OTP generator is set to + expire every code. + :param kwargs: other query string parameters to include in the URI + :returns: provisioning uri + """ + # initial_count may be 0 as a valid param + is_initial_count_present = initial_count is not None + + # Handling values different from defaults + is_algorithm_set = algorithm is not None and algorithm != "sha1" + is_digits_set = digits is not None and digits != 6 + is_period_set = period is not None and period != 30 + + otp_type = "hotp" if is_initial_count_present else "totp" + base_uri = "otpauth://{0}/{1}?{2}" + + url_args: Dict[str, Union[None, int, str]] = {"secret": secret} + + label = quote(name) + if issuer is not None: + label = quote(issuer) + ":" + label + url_args["issuer"] = issuer + + if is_initial_count_present: + url_args["counter"] = initial_count + if is_algorithm_set: + url_args["algorithm"] = algorithm.upper() # type: ignore + if is_digits_set: + url_args["digits"] = digits + if is_period_set: + url_args["period"] = period + for k, v in kwargs.items(): + if not isinstance(v, str): + raise ValueError("All otpauth uri parameters must be strings") + if k == "image": + image_uri = urlparse(v) + if image_uri.scheme != "https" or not image_uri.netloc or not image_uri.path: + raise ValueError("{} is not a valid url".format(image_uri)) + url_args[k] = v + + uri = base_uri.format(otp_type, label, urlencode(url_args).replace("+", "%20")) + return uri + + +def strings_equal(s1: str, s2: str) -> bool: + """ + Timing-attack resistant string comparison. + + Normal comparison using == will short-circuit on the first mismatching + character. This avoids that by scanning the whole string, though we + still reveal to a timing attack whether the strings are the same + length. + """ + s1 = unicodedata.normalize("NFKC", s1) + s2 = unicodedata.normalize("NFKC", s2) + return compare_digest(s1.encode("utf-8"), s2.encode("utf-8")) diff --git a/py_modules/yarl/__init__.py b/py_modules/yarl/__init__.py new file mode 100644 index 0000000..76c409e --- /dev/null +++ b/py_modules/yarl/__init__.py @@ -0,0 +1,14 @@ +from ._query import Query, QueryVariable, SimpleQuery +from ._url import URL, cache_clear, cache_configure, cache_info + +__version__ = "1.24.5" + +__all__ = ( + "URL", + "SimpleQuery", + "QueryVariable", + "Query", + "cache_clear", + "cache_configure", + "cache_info", +) diff --git a/py_modules/yarl/_parse.py b/py_modules/yarl/_parse.py new file mode 100644 index 0000000..99dab1a --- /dev/null +++ b/py_modules/yarl/_parse.py @@ -0,0 +1,232 @@ +"""URL parsing utilities.""" + +import re +import unicodedata +from functools import lru_cache +from urllib.parse import scheme_chars, uses_netloc + +from ._quoters import QUOTER, UNQUOTER_PLUS + +# Leading and trailing C0 control and space to be stripped per WHATWG spec. +# == "".join([chr(i) for i in range(0, 0x20 + 1)]) +WHATWG_C0_CONTROL_OR_SPACE = ( + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\t\n\x0b\x0c\r\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f " +) + +# Unsafe bytes to be removed per WHATWG spec +UNSAFE_URL_BYTES_TO_REMOVE = ["\t", "\r", "\n"] +USES_AUTHORITY = frozenset(uses_netloc) + +SplitURLType = tuple[str, str, str, str, str] + + +def split_url(url: str) -> SplitURLType: + """Split URL into parts.""" + # Adapted from urllib.parse.urlsplit + # Only lstrip url as some applications rely on preserving trailing space. + # (https://url.spec.whatwg.org/#concept-basic-url-parser would strip both) + url = url.lstrip(WHATWG_C0_CONTROL_OR_SPACE) + for b in UNSAFE_URL_BYTES_TO_REMOVE: + if b in url: + url = url.replace(b, "") + + scheme = netloc = query = fragment = "" + i = url.find(":") + if i > 0 and url[0] in scheme_chars: + for c in url[1:i]: + if c not in scheme_chars: + break + else: + scheme, url = url[:i].lower(), url[i + 1 :] + has_hash = "#" in url + has_question_mark = "?" in url + if url[:2] == "//": + delim = len(url) # position of end of domain part of url, default is end + if has_hash and has_question_mark: + delim_chars = "/?#" + elif has_question_mark: + delim_chars = "/?" + elif has_hash: + delim_chars = "/#" + else: + delim_chars = "/" + for c in delim_chars: # look for delimiters; the order is NOT important + wdelim = url.find(c, 2) # find first of this delim + if wdelim >= 0 and wdelim < delim: # if found + delim = wdelim # use earliest delim position + netloc = url[2:delim] + url = url[delim:] + # Backslash is not valid in the authority component per RFC 3986. + # WHATWG parsers treat \ as a path separator for special schemes, so + # accepting it in the authority can cause host parsing ambiguity. + if "\\" in netloc: + raise ValueError( + "Invalid URL: backslash ('\\') is not allowed in the authority " + "component per RFC 3986." + ) + has_left_bracket = "[" in netloc + has_right_bracket = "]" in netloc + if (has_left_bracket and not has_right_bracket) or ( + has_right_bracket and not has_left_bracket + ): + raise ValueError("Invalid IPv6 URL") + if has_left_bracket: + # Per RFC 3986, brackets are only valid at the START of the host + # for IP-literal addresses. Text before '[' (e.g. '127.0.0.1[::1]') + # is invalid and must be rejected to prevent SSRF bypasses. The + # count checks reject URLs with more than one bracket pair in the + # host subcomponent (e.g. 'http://[:localhost[]].google:80'), + # which would otherwise resolve to an unintended host. + hostinfo = netloc.rpartition("@")[2] + if hostinfo[0] != "[" or hostinfo.count("[") > 1 or hostinfo.count("]") > 1: + raise ValueError("Invalid IPv6 URL") + bracketed_host, _, after_bracket = hostinfo[1:].partition("]") + # Per RFC 3986 §3.2.2, after the closing ']' of an IP-literal + # only ":" or end-of-authority is valid. Any other text + # (e.g. '[::1]allowed.example:1') must be rejected to prevent + # host-confusion where the suffix is silently dropped. + if after_bracket and after_bracket[0] != ":": + raise ValueError("Invalid IPv6 URL") + # Valid bracketed hosts are defined in + # https://www.rfc-editor.org/rfc/rfc3986#page-49 + # https://url.spec.whatwg.org/ + if bracketed_host and bracketed_host[0] == "v": + if not re.match(r"\Av[a-fA-F0-9]+\..+\Z", bracketed_host): + raise ValueError("IPvFuture address is invalid") + elif ":" not in bracketed_host: + raise ValueError("The IPv6 content between brackets is not valid") + if has_hash: + url, _, fragment = url.partition("#") + if has_question_mark: + url, _, query = url.partition("?") + if netloc and not netloc.isascii(): + _check_netloc(netloc) + return scheme, netloc, url, query, fragment + + +def _check_netloc(netloc: str) -> None: + # Adapted from urllib.parse._checknetloc + # looking for characters like \u2100 that expand to 'a/c' + # IDNA uses NFKC equivalence, so normalize for this check + + # ignore characters already included + # but not the surrounding text + n = netloc.replace("@", "").replace(":", "").replace("#", "").replace("?", "") + normalized_netloc = unicodedata.normalize("NFKC", n) + if n == normalized_netloc: + return + # Note that there are no unicode decompositions for the character '@' so + # its currently impossible to have test coverage for this branch, however if the + # one should be added in the future we want to make sure its still checked. + for c in "/?#@:%": # pragma: no branch + if c in normalized_netloc: + raise ValueError( + f"netloc '{netloc}' contains invalid " + "characters under NFKC normalization" + ) + + +@lru_cache # match the same size as urlsplit +def split_netloc( + netloc: str, +) -> tuple[str | None, str | None, str | None, int | None]: + """Split netloc into username, password, host and port.""" + if "@" not in netloc: + username: str | None = None + password: str | None = None + hostinfo = netloc + else: + userinfo, _, hostinfo = netloc.rpartition("@") + username, have_password, password = userinfo.partition(":") + if not have_password: + password = None + + if "[" in hostinfo: + if hostinfo[0] != "[" or hostinfo.count("[") > 1 or hostinfo.count("]") > 1: + raise ValueError("Invalid IPv6 URL") + _, _, bracketed = hostinfo.partition("[") + hostname, _, port_str = bracketed.partition("]") + # Defense-in-depth: after ']' only ':port' or empty is valid. + # split_url() should have already rejected invalid suffixes, + # but guard here too for callers that use split_netloc() directly. + if port_str and port_str[0] != ":": + raise ValueError("Invalid IPv6 URL") + _, _, port_str = port_str.partition(":") + else: + hostname, _, port_str = hostinfo.partition(":") + + if not port_str: + return username or None, password, hostname or None, None + + try: + port = int(port_str) + except ValueError: + raise ValueError("Invalid URL: port can't be converted to integer") + if not (0 <= port <= 65535): + raise ValueError("Port out of range 0-65535") + return username or None, password, hostname or None, port + + +def unsplit_result( + scheme: str, netloc: str, url: str, query: str, fragment: str +) -> str: + """Unsplit a URL without any normalization.""" + if netloc or (scheme and scheme in USES_AUTHORITY) or url[:2] == "//": + if url and url[:1] != "/": + url = f"{scheme}://{netloc}/{url}" if scheme else f"{scheme}:{url}" + else: + url = f"{scheme}://{netloc}{url}" if scheme else f"//{netloc}{url}" + elif scheme: + url = f"{scheme}:{url}" + if query: + url = f"{url}?{query}" + return f"{url}#{fragment}" if fragment else url + + +@lru_cache # match the same size as urlsplit +def make_netloc( + user: str | None, + password: str | None, + host: str | None, + port: int | None, + encode: bool = False, +) -> str: + """Make netloc from parts. + + The user and password are encoded if encode is True. + + The host must already be encoded with _encode_host. + """ + if host is None: + return "" + ret = host + if port is not None: + ret = f"{ret}:{port}" + if user is None and password is None: + return ret + if password is not None: + if not user: + user = "" + elif encode: + user = QUOTER(user) + if encode: + password = QUOTER(password) + user = f"{user}:{password}" + elif user and encode: + user = QUOTER(user) + return f"{user}@{ret}" if user else ret + + +def query_to_pairs(query_string: str) -> list[tuple[str, str]]: + """Parse a query given as a string argument. + + Works like urllib.parse.parse_qsl with keep empty values. + """ + pairs: list[tuple[str, str]] = [] + if not query_string: + return pairs + for k_v in query_string.split("&"): + k, _, v = k_v.partition("=") + pairs.append((UNQUOTER_PLUS(k), UNQUOTER_PLUS(v))) + return pairs diff --git a/py_modules/yarl/_path.py b/py_modules/yarl/_path.py new file mode 100644 index 0000000..c22f0b4 --- /dev/null +++ b/py_modules/yarl/_path.py @@ -0,0 +1,41 @@ +"""Utilities for working with paths.""" + +from collections.abc import Sequence +from contextlib import suppress + + +def normalize_path_segments(segments: Sequence[str]) -> list[str]: + """Drop '.' and '..' from a sequence of str segments""" + + resolved_path: list[str] = [] + + for seg in segments: + if seg == "..": + # ignore any .. segments that would otherwise cause an + # IndexError when popped from resolved_path if + # resolving for rfc3986 + with suppress(IndexError): + resolved_path.pop() + elif seg != ".": + resolved_path.append(seg) + + if segments and segments[-1] in (".", ".."): + # do some post-processing here. + # if the last segment was a relative dir, + # then we need to append the trailing '/' + resolved_path.append("") + + return resolved_path + + +def normalize_path(path: str) -> str: + # Drop '.' and '..' from str path + prefix = "" + if path and path[0] == "/": + # preserve the "/" root element of absolute paths, copying it to the + # normalised output as per sections 5.2.4 and 6.2.2.3 of rfc3986. + prefix = "/" + path = path[1:] + + segments = path.split("/") + return prefix + "/".join(normalize_path_segments(segments)) diff --git a/py_modules/yarl/_query.py b/py_modules/yarl/_query.py new file mode 100644 index 0000000..3f5ca22 --- /dev/null +++ b/py_modules/yarl/_query.py @@ -0,0 +1,127 @@ +"""Query string handling.""" + +import math +from collections.abc import Iterable, Mapping, Sequence +from typing import TYPE_CHECKING, Any, SupportsInt, Union, cast + +from multidict import istr + +from ._quoters import QUERY_PART_QUOTER, QUERY_QUOTER + +SimpleQuery = Union[str, SupportsInt, float] +QueryVariable = Union[SimpleQuery, Sequence[SimpleQuery]] +Query = Union[ + None, str, Mapping[str, QueryVariable], Sequence[tuple[str, QueryVariable]] +] + + +def query_var(v: SimpleQuery) -> str: + """Convert a query variable to a string. + + Note: Objects implementing the ``__int__`` data model method (typed as + ``SupportsInt``; e.g. ``uuid.UUID``) are converted via ``int()`` first. + Callers should convert such values to ``str`` explicitly if the string + representation is desired. + """ + cls = type(v) + if cls is int: # Fast path for non-subclassed int + return str(v) + if isinstance(v, str): + return v + if isinstance(v, float): + if math.isinf(v): + raise ValueError("float('inf') is not supported") + if math.isnan(v): + raise ValueError("float('nan') is not supported") + return str(float(v)) + if cls is not bool and isinstance(v, SupportsInt): + return str(int(v)) + raise TypeError( + "Invalid variable type: value " + "should be str, int or float, got {!r} " + "of type {}".format(v, cls) + ) + + +def get_str_query_from_sequence_iterable( + items: Iterable[tuple[str | istr, QueryVariable]], +) -> str: + """Return a query string from a sequence of (key, value) pairs. + + value is a single value or a sequence of values for the key + + The sequence of values must be a list or tuple. + """ + quoter = QUERY_PART_QUOTER + pairs = [ + f"{quoter(k)}={quoter(v if type(v) is str else query_var(v))}" + for k, val in items + for v in ( + val if type(val) is not str and isinstance(val, (list, tuple)) else (val,) + ) + ] + return "&".join(pairs) + + +def get_str_query_from_iterable( + items: Iterable[tuple[str | istr, SimpleQuery]], +) -> str: + """Return a query string from an iterable. + + The iterable must contain (key, value) pairs. + + The values are not allowed to be sequences, only single values are + allowed. For sequences, use `_get_str_query_from_sequence_iterable`. + """ + quoter = QUERY_PART_QUOTER + # A listcomp is used since listcomps are inlined on CPython 3.12+ and + # they are a bit faster than a generator expression. + pairs = [ + f"{quoter(k)}={quoter(v if type(v) is str else query_var(v))}" for k, v in items + ] + return "&".join(pairs) + + +def get_str_query(*args: Any, **kwargs: Any) -> str | None: + """Return a query string from supported args.""" + query: ( + str + | Mapping[str, QueryVariable] + | Sequence[tuple[str | istr, SimpleQuery]] + | None + ) + if kwargs: + if args: + msg = "Either kwargs or single query parameter must be present" + raise ValueError(msg) + query = kwargs + elif len(args) == 1: + query = args[0] + else: + raise ValueError("Either kwargs or single query parameter must be present") + + if query is None: + return None + if not query: + return "" + if type(query) is dict: + return get_str_query_from_sequence_iterable(query.items()) + if type(query) is str or isinstance(query, str): + return QUERY_QUOTER(query) + if isinstance(query, Mapping): + return get_str_query_from_sequence_iterable(query.items()) + if isinstance(query, (bytes, bytearray, memoryview)): + msg = "Invalid query type: bytes, bytearray and memoryview are forbidden" + raise TypeError(msg) + if isinstance(query, Sequence): + # We don't expect sequence values if we're given a list of pairs + # already; only mappings like builtin `dict` which can't have the + # same key pointing to multiple values are allowed to use + # `_query_seq_pairs`. + if TYPE_CHECKING: + query = cast(Sequence[tuple[Union[str, istr], SimpleQuery]], query) + return get_str_query_from_iterable(query) + raise TypeError( + "Invalid query type: only str, mapping or " + "sequence of (key, value) pairs is allowed" + ) diff --git a/py_modules/yarl/_quoters.py b/py_modules/yarl/_quoters.py new file mode 100644 index 0000000..78f77ca --- /dev/null +++ b/py_modules/yarl/_quoters.py @@ -0,0 +1,32 @@ +"""Quoting and unquoting utilities for URL parts.""" + +from urllib.parse import quote + +from ._quoting import _Quoter, _Unquoter + +QUOTER = _Quoter(requote=False) +REQUOTER = _Quoter() +PATH_QUOTER = _Quoter(safe="@:", protected="/+", requote=False) +PATH_REQUOTER = _Quoter(safe="@:", protected="/+") +QUERY_QUOTER = _Quoter(safe="?/:@", protected="=+&;", qs=True, requote=False) +QUERY_REQUOTER = _Quoter(safe="?/:@", protected="=+&;", qs=True) +QUERY_PART_QUOTER = _Quoter(safe="?/:@", qs=True, requote=False) +FRAGMENT_QUOTER = _Quoter(safe="?/:@", requote=False) +FRAGMENT_REQUOTER = _Quoter(safe="?/:@") + +UNQUOTER = _Unquoter() +PATH_UNQUOTER = _Unquoter(unsafe="+") +PATH_SAFE_UNQUOTER = _Unquoter(ignore="/%", unsafe="+") +QS_UNQUOTER = _Unquoter(qs=True) +UNQUOTER_PLUS = _Unquoter(plus=True) # to match urllib.parse.unquote_plus + + +def human_quote(s: str | None, unsafe: str) -> str | None: + if not s: + return s + for c in "%" + unsafe: + if c in s: + s = s.replace(c, f"%{ord(c):02X}") + if s.isprintable(): + return s + return "".join(c if c.isprintable() else quote(c) for c in s) diff --git a/py_modules/yarl/_quoting.py b/py_modules/yarl/_quoting.py new file mode 100644 index 0000000..25d76c8 --- /dev/null +++ b/py_modules/yarl/_quoting.py @@ -0,0 +1,19 @@ +import os +import sys +from typing import TYPE_CHECKING + +__all__ = ("_Quoter", "_Unquoter") + + +NO_EXTENSIONS = bool(os.environ.get("YARL_NO_EXTENSIONS")) # type: bool +if sys.implementation.name != "cpython": + NO_EXTENSIONS = True + + +if TYPE_CHECKING or NO_EXTENSIONS: + from ._quoting_py import _Quoter, _Unquoter +else: + try: + from ._quoting_c import _Quoter, _Unquoter + except ImportError: # pragma: no cover + from ._quoting_py import _Quoter, _Unquoter # type: ignore[assignment] diff --git a/py_modules/yarl/_quoting_c.cpython-311-x86_64-linux-gnu.so b/py_modules/yarl/_quoting_c.cpython-311-x86_64-linux-gnu.so new file mode 100755 index 0000000..1d1b554 Binary files /dev/null and b/py_modules/yarl/_quoting_c.cpython-311-x86_64-linux-gnu.so differ diff --git a/py_modules/yarl/_quoting_c.pyx b/py_modules/yarl/_quoting_c.pyx new file mode 100644 index 0000000..be3e6e0 --- /dev/null +++ b/py_modules/yarl/_quoting_c.pyx @@ -0,0 +1,483 @@ +from cpython.exc cimport PyErr_NoMemory +from cpython.mem cimport PyMem_Free, PyMem_Malloc, PyMem_Realloc +from cpython.unicode cimport ( + PyUnicode_DATA, + PyUnicode_DecodeASCII, + PyUnicode_DecodeUTF8Stateful, + PyUnicode_GET_LENGTH, + PyUnicode_KIND, + PyUnicode_READ, +) +from libc.stdint cimport uint8_t, uint64_t +from libc.string cimport memcpy, memset + +from string import ascii_letters, digits + + +cdef str GEN_DELIMS = ":/?#[]@" +cdef str SUB_DELIMS_WITHOUT_QS = "!$'()*," +cdef str SUB_DELIMS = SUB_DELIMS_WITHOUT_QS + '+?=;' +cdef str RESERVED = GEN_DELIMS + SUB_DELIMS +cdef str UNRESERVED = ascii_letters + digits + '-._~' +cdef str ALLOWED = UNRESERVED + SUB_DELIMS_WITHOUT_QS +cdef str QS = '+&=;' + +DEF BUF_SIZE = 8 * 1024 # 8KiB + +cdef inline Py_UCS4 _to_hex(uint8_t v) noexcept: + if v < 10: + return (v+0x30) # ord('0') == 0x30 + else: + return (v+0x41-10) # ord('A') == 0x41 + + +cdef inline int _from_hex(Py_UCS4 v) noexcept: + if '0' <= v <= '9': + return (v) - 0x30 # ord('0') == 0x30 + elif 'A' <= v <= 'F': + return (v) - 0x41 + 10 # ord('A') == 0x41 + elif 'a' <= v <= 'f': + return (v) - 0x61 + 10 # ord('a') == 0x61 + else: + return -1 + + +cdef inline int _is_lower_hex(Py_UCS4 v) noexcept: + return 'a' <= v <= 'f' + + +cdef inline bint _is_surrogate(Py_UCS4 ch) noexcept: + return 0xD800 <= ch <= 0xDFFF + + +cdef inline Py_ssize_t _skip_surrogates( + int kind, const void *data, Py_ssize_t idx, Py_ssize_t length +) noexcept: + # Advance past lone surrogates; they cannot be UTF-8 encoded and are + # dropped, so they must not break up a percent escape during requoting. + while idx < length and _is_surrogate(PyUnicode_READ(kind, data, idx)): + idx += 1 + return idx + + +cdef inline long _restore_ch(Py_UCS4 d1, Py_UCS4 d2): + cdef int digit1 = _from_hex(d1) + if digit1 < 0: + return -1 + cdef int digit2 = _from_hex(d2) + if digit2 < 0: + return -1 + return digit1 << 4 | digit2 + + +cdef uint8_t ALLOWED_TABLE[16] +cdef uint8_t ALLOWED_NOTQS_TABLE[16] + + +cdef inline bint bit_at(uint8_t array[], uint64_t ch) noexcept: + return array[ch >> 3] & (1 << (ch & 7)) + + +cdef inline void set_bit(uint8_t array[], uint64_t ch) noexcept: + array[ch >> 3] |= (1 << (ch & 7)) + + +memset(ALLOWED_TABLE, 0, sizeof(ALLOWED_TABLE)) +memset(ALLOWED_NOTQS_TABLE, 0, sizeof(ALLOWED_NOTQS_TABLE)) + +for i in range(128): + if chr(i) in ALLOWED: + set_bit(ALLOWED_TABLE, i) + set_bit(ALLOWED_NOTQS_TABLE, i) + if chr(i) in QS: + set_bit(ALLOWED_NOTQS_TABLE, i) + +# ----------------- writer --------------------------- + +cdef struct Writer: + char *buf + bint heap_allocated_buf + Py_ssize_t size + Py_ssize_t pos + bint changed + + +cdef inline void _init_writer(Writer* writer, char* buf): + writer.buf = buf + writer.heap_allocated_buf = False + writer.size = BUF_SIZE + writer.pos = 0 + writer.changed = 0 + + +cdef inline void _release_writer(Writer* writer): + if writer.heap_allocated_buf: + PyMem_Free(writer.buf) + + +cdef inline int _write_char(Writer* writer, Py_UCS4 ch, bint changed): + cdef char * buf + cdef Py_ssize_t size + + if writer.pos == writer.size: + # reallocate + size = writer.size + BUF_SIZE + if not writer.heap_allocated_buf: + buf = PyMem_Malloc(size) + if buf == NULL: + PyErr_NoMemory() + return -1 + memcpy(buf, writer.buf, writer.size) + writer.heap_allocated_buf = True + else: + buf = PyMem_Realloc(writer.buf, size) + if buf == NULL: + PyErr_NoMemory() + return -1 + writer.buf = buf + writer.size = size + writer.buf[writer.pos] = ch + writer.pos += 1 + writer.changed |= changed + return 0 + + +cdef inline int _write_pct(Writer* writer, uint8_t ch, bint changed): + if _write_char(writer, '%', changed) < 0: + return -1 + if _write_char(writer, _to_hex(ch >> 4), changed) < 0: + return -1 + return _write_char(writer, _to_hex(ch & 0x0f), changed) + + +cdef inline int _write_utf8(Writer* writer, Py_UCS4 symbol): + cdef uint64_t utf = symbol + + if utf < 0x80: + return _write_pct(writer, utf, True) + elif utf < 0x800: + if _write_pct(writer, (0xc0 | (utf >> 6)), True) < 0: + return -1 + return _write_pct(writer, (0x80 | (utf & 0x3f)), True) + elif _is_surrogate(symbol): + # lone surrogate; invalid in UTF-8 so it is dropped, matching the + # pure-Python quoter's errors="ignore" encode. Mark the writer as + # changed so _do_quote returns the surrogate-free buffer rather than + # the untouched input string. + writer.changed = True + return 0 + elif utf < 0x10000: + if _write_pct(writer, (0xe0 | (utf >> 12)), True) < 0: + return -1 + if _write_pct(writer, (0x80 | ((utf >> 6) & 0x3f)), + True) < 0: + return -1 + return _write_pct(writer, (0x80 | (utf & 0x3f)), True) + elif utf > 0x10FFFF: + # symbol is too large + return 0 + else: + if _write_pct(writer, (0xf0 | (utf >> 18)), True) < 0: + return -1 + if _write_pct(writer, (0x80 | ((utf >> 12) & 0x3f)), + True) < 0: + return -1 + if _write_pct(writer, (0x80 | ((utf >> 6) & 0x3f)), + True) < 0: + return -1 + return _write_pct(writer, (0x80 | (utf & 0x3f)), True) + + +# --------------------- end writer -------------------------- + + +cdef class _Quoter: + cdef bint _qs + cdef bint _requote + + cdef uint8_t _safe_table[16] + cdef uint8_t _protected_table[16] + + def __init__( + self, *, str safe='', str protected='', bint qs=False, bint requote=True, + ): + cdef Py_UCS4 ch + + self._qs = qs + self._requote = requote + + if not self._qs: + memcpy(self._safe_table, + ALLOWED_NOTQS_TABLE, + sizeof(self._safe_table)) + else: + memcpy(self._safe_table, + ALLOWED_TABLE, + sizeof(self._safe_table)) + for ch in safe: + if ord(ch) > 127: + raise ValueError("Only safe symbols with ORD < 128 are allowed") + set_bit(self._safe_table, ch) + + memset(self._protected_table, 0, sizeof(self._protected_table)) + for ch in protected: + if ord(ch) > 127: + raise ValueError("Only safe symbols with ORD < 128 are allowed") + set_bit(self._safe_table, ch) + set_bit(self._protected_table, ch) + + def __call__(self, val): + if val is None: + return None + if type(val) is not str: + if isinstance(val, str): + # derived from str + val = str(val) + else: + raise TypeError("Argument should be str") + return self._do_quote_or_skip(val) + + cdef str _do_quote_or_skip(self, str val): + cdef char[BUF_SIZE] buffer + cdef Py_UCS4 ch + cdef Py_ssize_t length = PyUnicode_GET_LENGTH(val) + cdef Py_ssize_t idx = length + cdef bint must_quote = 0 + cdef Writer writer + cdef int kind = PyUnicode_KIND(val) + cdef const void *data = PyUnicode_DATA(val) + + # If everything in the string is in the safe + # table and all ASCII, we can skip quoting + while idx: + idx -= 1 + ch = PyUnicode_READ(kind, data, idx) + if ch >= 128 or not bit_at(self._safe_table, ch): + must_quote = 1 + break + + if not must_quote: + return val + + _init_writer(&writer, &buffer[0]) + try: + return self._do_quote(val, length, kind, data, &writer) + finally: + _release_writer(&writer) + + cdef str _do_quote( + self, + str val, + Py_ssize_t length, + int kind, + const void *data, + Writer *writer + ): + cdef Py_UCS4 ch + cdef Py_UCS4 d1 + cdef Py_UCS4 d2 + cdef long chl + cdef int changed + cdef bint surrogate_skipped + cdef Py_ssize_t idx = 0 + cdef Py_ssize_t pos1 + cdef Py_ssize_t pos2 + + while idx < length: + ch = PyUnicode_READ(kind, data, idx) + idx += 1 + if ch == '%' and self._requote and idx < length: + # Lone surrogates are dropped (see _skip_surrogates), so look + # through them for the two hex digits of the "%XX" escape; this + # keeps the C quoter consistent with the pure-Python backend, + # which strips surrogates before scanning. + pos1 = _skip_surrogates(kind, data, idx, length) + pos2 = _skip_surrogates(kind, data, pos1 + 1, length) + if pos2 < length: + d1 = PyUnicode_READ(kind, data, pos1) + d2 = PyUnicode_READ(kind, data, pos2) + chl = _restore_ch(d1, d2) + else: + chl = -1 + if chl != -1: + ch = chl + surrogate_skipped = pos1 != idx or pos2 != pos1 + 1 + idx = pos2 + 1 + if ch < 128: + if bit_at(self._protected_table, ch): + if _write_pct(writer, ch, True) < 0: + raise + continue + + if bit_at(self._safe_table, ch): + if _write_char(writer, ch, True) < 0: + raise + continue + + changed = (surrogate_skipped or + _is_lower_hex(d1) or _is_lower_hex(d2)) + if _write_pct(writer, ch, changed) < 0: + raise + continue + else: + ch = '%' + + if self._write(writer, ch) < 0: + raise + + if not writer.changed: + return val + else: + return PyUnicode_DecodeASCII(writer.buf, writer.pos, "strict") + + cdef inline int _write(self, Writer *writer, Py_UCS4 ch): + if self._qs: + if ch == ' ': + return _write_char(writer, '+', True) + + if ch < 128 and bit_at(self._safe_table, ch): + return _write_char(writer, ch, False) + + return _write_utf8(writer, ch) + + +cdef class _Unquoter: + cdef str _ignore + cdef bint _has_ignore + cdef str _unsafe + cdef bytes _unsafe_bytes + cdef Py_ssize_t _unsafe_bytes_len + cdef const unsigned char * _unsafe_bytes_char + cdef bint _qs + cdef bint _plus # to match urllib.parse.unquote_plus + cdef _Quoter _quoter + cdef _Quoter _qs_quoter + + def __init__(self, *, ignore="", unsafe="", qs=False, plus=False): + self._ignore = ignore + self._has_ignore = bool(self._ignore) + self._unsafe = unsafe + # unsafe may only be extended ascii characters (0-255) + self._unsafe_bytes = self._unsafe.encode('ascii') + self._unsafe_bytes_len = len(self._unsafe_bytes) + self._unsafe_bytes_char = self._unsafe_bytes + self._qs = qs + self._plus = plus + self._quoter = _Quoter() + self._qs_quoter = _Quoter(qs=True) + + def __call__(self, val): + if val is None: + return None + if type(val) is not str: + if isinstance(val, str): + # derived from str + val = str(val) + else: + raise TypeError("Argument should be str") + return self._do_unquote(val) + + cdef str _do_unquote(self, str val): + cdef Py_ssize_t length = PyUnicode_GET_LENGTH(val) + if length == 0: + return val + + cdef list ret = [] + cdef char buffer[4] + cdef Py_ssize_t buflen = 0 + cdef Py_ssize_t consumed + cdef str unquoted + cdef Py_UCS4 ch = 0 + cdef long chl = 0 + cdef Py_ssize_t idx = 0 + cdef Py_ssize_t start_pct + cdef int kind = PyUnicode_KIND(val) + cdef const void *data = PyUnicode_DATA(val) + cdef bint changed = 0 + while idx < length: + ch = PyUnicode_READ(kind, data, idx) + idx += 1 + if ch == '%' and idx <= length - 2: + changed = 1 + chl = _restore_ch( + PyUnicode_READ(kind, data, idx), + PyUnicode_READ(kind, data, idx + 1) + ) + if chl != -1: + ch = chl + idx += 2 + assert buflen < 4 + buffer[buflen] = ch + buflen += 1 + try: + unquoted = PyUnicode_DecodeUTF8Stateful(buffer, buflen, + NULL, &consumed) + except UnicodeDecodeError: + start_pct = idx - buflen * 3 + buffer[0] = ch + buflen = 1 + ret.append(val[start_pct : idx - 3]) + try: + unquoted = PyUnicode_DecodeUTF8Stateful(buffer, buflen, + NULL, &consumed) + except UnicodeDecodeError: + buflen = 0 + ret.append(val[idx - 3 : idx]) + continue + if not unquoted: + assert consumed == 0 + continue + assert consumed == buflen + buflen = 0 + if self._qs and unquoted in '+=&;': + ret.append(self._qs_quoter(unquoted)) + elif ( + (self._unsafe_bytes_len and unquoted in self._unsafe) or + (self._has_ignore and unquoted in self._ignore) + ): + ret.append(self._quoter(unquoted)) + else: + ret.append(unquoted) + continue + else: + ch = '%' + + if buflen: + start_pct = idx - 1 - buflen * 3 + ret.append(val[start_pct : idx - 1]) + buflen = 0 + + if ch == '+': + if ( + (not self._qs and not self._plus) or + (self._unsafe_bytes_len and self._is_char_unsafe(ch)) + ): + ret.append('+') + else: + changed = 1 + ret.append(' ') + continue + + if self._unsafe_bytes_len and self._is_char_unsafe(ch): + changed = 1 + ret.append('%') + h = hex(ord(ch)).upper()[2:] + for ch in h: + ret.append(ch) + continue + + ret.append(ch) + + if not changed: + return val + + if buflen: + ret.append(val[length - buflen * 3 : length]) + + return ''.join(ret) + + cdef inline bint _is_char_unsafe(self, Py_UCS4 ch): + for i in range(self._unsafe_bytes_len): + if ch == self._unsafe_bytes_char[i]: + return True + return False diff --git a/py_modules/yarl/_quoting_py.py b/py_modules/yarl/_quoting_py.py new file mode 100644 index 0000000..98197aa --- /dev/null +++ b/py_modules/yarl/_quoting_py.py @@ -0,0 +1,213 @@ +import codecs +import re +from string import ascii_letters, ascii_lowercase, digits +from typing import overload + +BASCII_LOWERCASE = ascii_lowercase.encode("ascii") +BPCT_ALLOWED = {f"%{i:02X}".encode("ascii") for i in range(256)} +GEN_DELIMS = ":/?#[]@" +SUB_DELIMS_WITHOUT_QS = "!$'()*," +SUB_DELIMS = SUB_DELIMS_WITHOUT_QS + "+&=;" +RESERVED = GEN_DELIMS + SUB_DELIMS +UNRESERVED = ascii_letters + digits + "-._~" +ALLOWED = UNRESERVED + SUB_DELIMS_WITHOUT_QS + + +_IS_HEX = re.compile(b"[A-Z0-9][A-Z0-9]") +_IS_HEX_STR = re.compile("[A-Fa-f0-9][A-Fa-f0-9]") + +utf8_decoder = codecs.getincrementaldecoder("utf-8") + + +class _Quoter: + def __init__( + self, + *, + safe: str = "", + protected: str = "", + qs: bool = False, + requote: bool = True, + ) -> None: + self._safe = safe + self._protected = protected + self._qs = qs + self._requote = requote + + @overload + def __call__(self, val: str) -> str: ... + @overload + def __call__(self, val: None) -> None: ... + def __call__(self, val: str | None) -> str | None: + if val is None: + return None + if not isinstance(val, str): + raise TypeError("Argument should be str") + if not val: + return "" + bval = val.encode("utf8", errors="ignore") + ret = bytearray() + pct = bytearray() + safe = self._safe + safe += ALLOWED + if not self._qs: + safe += "+&=;" + safe += self._protected + bsafe = safe.encode("ascii") + idx = 0 + while idx < len(bval): + ch = bval[idx] + idx += 1 + + if pct: + if ch in BASCII_LOWERCASE: + ch = ch - 32 # convert to uppercase + pct.append(ch) + if len(pct) == 3: # pragma: no branch # peephole optimizer + buf = pct[1:] + if not _IS_HEX.match(buf): + ret.extend(b"%25") + pct.clear() + idx -= 2 + continue + try: + unquoted = chr(int(pct[1:].decode("ascii"), base=16)) + except ValueError: + ret.extend(b"%25") + pct.clear() + idx -= 2 + continue + + if unquoted in self._protected: + ret.extend(pct) + elif unquoted in safe: + ret.append(ord(unquoted)) + else: + ret.extend(pct) + pct.clear() + + # special case, if we have only one char after "%" + elif len(pct) == 2 and idx == len(bval): + ret.extend(b"%25") + pct.clear() + idx -= 1 + + continue + + elif ch == ord("%") and self._requote: + pct.clear() + pct.append(ch) + + # special case if "%" is last char + if idx == len(bval): + ret.extend(b"%25") + + continue + + if self._qs and ch == ord(" "): + ret.append(ord("+")) + continue + if ch in bsafe: + ret.append(ch) + continue + + ret.extend((f"%{ch:02X}").encode("ascii")) + + ret2 = ret.decode("ascii") + if ret2 == val: + return val + return ret2 + + +class _Unquoter: + def __init__( + self, + *, + ignore: str = "", + unsafe: str = "", + qs: bool = False, + plus: bool = False, + ) -> None: + self._ignore = ignore + self._unsafe = unsafe + self._qs = qs + self._plus = plus # to match urllib.parse.unquote_plus + self._quoter = _Quoter() + self._qs_quoter = _Quoter(qs=True) + + @overload + def __call__(self, val: str) -> str: ... + @overload + def __call__(self, val: None) -> None: ... + def __call__(self, val: str | None) -> str | None: + if val is None: + return None + if not isinstance(val, str): + raise TypeError("Argument should be str") + if not val: + return "" + decoder = utf8_decoder() + ret = [] + idx = 0 + while idx < len(val): + ch = val[idx] + idx += 1 + if ch == "%" and idx <= len(val) - 2: + pct = val[idx : idx + 2] + if _IS_HEX_STR.fullmatch(pct): + b = bytes([int(pct, base=16)]) + idx += 2 + try: + unquoted = decoder.decode(b) + except UnicodeDecodeError: + start_pct = idx - 3 - len(decoder.buffer) * 3 + ret.append(val[start_pct : idx - 3]) + decoder.reset() + try: + unquoted = decoder.decode(b) + except UnicodeDecodeError: + ret.append(val[idx - 3 : idx]) + continue + if not unquoted: + continue + if self._qs and unquoted in "+=&;": + to_add = self._qs_quoter(unquoted) + if to_add is None: # pragma: no cover + raise RuntimeError("Cannot quote None") + ret.append(to_add) + elif unquoted in self._unsafe or unquoted in self._ignore: + to_add = self._quoter(unquoted) + if to_add is None: # pragma: no cover + raise RuntimeError("Cannot quote None") + ret.append(to_add) + else: + ret.append(unquoted) + continue + + if decoder.buffer: + start_pct = idx - 1 - len(decoder.buffer) * 3 + ret.append(val[start_pct : idx - 1]) + decoder.reset() + + if ch == "+": + if (not self._qs and not self._plus) or ch in self._unsafe: + ret.append("+") + else: + ret.append(" ") + continue + + if ch in self._unsafe: + ret.append("%") + h = hex(ord(ch)).upper()[2:] + for ch in h: + ret.append(ch) + continue + + ret.append(ch) + + if decoder.buffer: + ret.append(val[-len(decoder.buffer) * 3 :]) + + ret2 = "".join(ret) + if ret2 == val: + return val + return ret2 diff --git a/py_modules/yarl/_url.py b/py_modules/yarl/_url.py new file mode 100644 index 0000000..54a8946 --- /dev/null +++ b/py_modules/yarl/_url.py @@ -0,0 +1,1791 @@ +import re +import sys +import warnings +from collections.abc import Mapping, Sequence +from enum import Enum +from functools import _CacheInfo, lru_cache +from importlib.util import find_spec +from ipaddress import ip_address +from typing import ( + TYPE_CHECKING, + Any, + NoReturn, + TypedDict, + TypeVar, + Union, + cast, + overload, +) +from urllib.parse import SplitResult, scheme_chars, uses_relative + +import idna +from multidict import MultiDict, MultiDictProxy, istr +from propcache.api import under_cached_property as cached_property + +from ._parse import ( + USES_AUTHORITY, + SplitURLType, + make_netloc, + query_to_pairs, + split_netloc, + split_url, + unsplit_result, +) +from ._path import normalize_path, normalize_path_segments +from ._query import ( + Query, + QueryVariable, + SimpleQuery, + get_str_query, + get_str_query_from_iterable, + get_str_query_from_sequence_iterable, +) +from ._quoters import ( + FRAGMENT_QUOTER, + FRAGMENT_REQUOTER, + PATH_QUOTER, + PATH_REQUOTER, + PATH_SAFE_UNQUOTER, + PATH_UNQUOTER, + QS_UNQUOTER, + QUERY_QUOTER, + QUERY_REQUOTER, + QUOTER, + REQUOTER, + UNQUOTER, + human_quote, +) + +# Avoid Pydantic import if not used (increases yarl's import time by 3-7x). +HAS_PYDANTIC = find_spec("pydantic_core") is not None +if TYPE_CHECKING: + from pydantic import GetCoreSchemaHandler, GetJsonSchemaHandler + from pydantic.json_schema import JsonSchemaValue + from pydantic_core import CoreSchema + + +DEFAULT_PORTS = {"http": 80, "https": 443, "ws": 80, "wss": 443, "ftp": 21} +USES_RELATIVE = frozenset(uses_relative) +_SCHEME_CHARS = frozenset(scheme_chars) + +# Special schemes https://url.spec.whatwg.org/#special-scheme +# are not allowed to have an empty host https://url.spec.whatwg.org/#url-representation +SCHEME_REQUIRES_HOST = frozenset(("http", "https", "ws", "wss", "ftp")) + + +# reg-name: unreserved / pct-encoded / sub-delims +# this pattern matches anything that is *not* in those classes. and is only used +# on lower-cased ASCII values. +NOT_REG_NAME = re.compile( + r""" + # any character not in the unreserved or sub-delims sets, plus % + # (validated with the additional check for pct-encoded sequences below) + [^a-z0-9\-._~!$&'()*+,;=%] + | + # % only allowed if it is part of a pct-encoded + # sequence of 2 hex digits. + %(?![0-9a-f]{2}) + """, + re.VERBOSE, +) + +# Invisible default-ignorable / format code points that must not appear in a +# host (soft hyphen, zero-width space, word joiner, bidi controls, variation +# selectors, ...). Depending on the code point IDNA either silently deletes it +# (so ``evil.com`` encodes to ``evil.com``) or folds it into a different +# punycode host; either way the parsed host differs from the string an +# application validated. The set is the union of two authoritative sources, +# matching the two encoders _idna_encode dispatches to: +# +# 1. Unicode Default_Ignorable_Code_Point (uts46=True path via the ``idna`` +# package). Ranges taken from the DerivedCoreProperties data file: +# https://www.unicode.org/Public/UCD/latest/ucd/DerivedCoreProperties.txt +# (the E0000..E0FFF block is contiguous under this property). +# 2. RFC 3454 (Stringprep) Table B.1 "commonly mapped to nothing", used by the +# stdlib ``str.encode("idna")`` / IDNA2003 nameprep fallback: +# https://www.rfc-editor.org/rfc/rfc3454#appendix-B.1 +# This is the source of U+1806, which is not Default_Ignorable. +# +# Coverage is pinned to the installed ``idna``/Unicode data by a sweep test +# (test_default_ignorable_covers_idna_stripped in tests/test_url.py) that +# brute-forces every code point through _idna_encode and fails if any point it +# silently deletes is not matched here. +_DEFAULT_IGNORABLE_RE = re.compile( + "[" + "\u00ad" # SOFT HYPHEN + "\u034f" # COMBINING GRAPHEME JOINER + "\u061c" # ARABIC LETTER MARK + "\u115f-\u1160" # HANGUL CHOSEONG/JUNGSEONG FILLER + "\u17b4-\u17b5" # KHMER VOWEL INHERENT AQ/AA + "\u1806" # MONGOLIAN TODO SOFT HYPHEN (nameprep maps to nothing) + "\u180b-\u180f" # MONGOLIAN FVS ONE..FOUR and VOWEL SEPARATOR + "\u200b-\u200f" # ZERO WIDTH SPACE..RIGHT-TO-LEFT MARK + "\u202a-\u202e" # bidi embedding/override controls + "\u2060-\u206f" # WORD JOINER..NOMINAL DIGIT SHAPES + "\u3164" # HANGUL FILLER + "\ufe00-\ufe0f" # VARIATION SELECTOR-1..16 + "\ufeff" # ZERO WIDTH NO-BREAK SPACE (BOM) + "\uffa0" # HALFWIDTH HANGUL FILLER + "\ufff0-\ufff8" # reserved default-ignorables + "\U0001bca0-\U0001bca3" # SHORTHAND FORMAT controls + "\U0001d173-\U0001d17a" # MUSICAL SYMBOL begin/end controls + "\U000e0000-\U000e0fff" # tags and VARIATION SELECTOR SUPPLEMENT + "]" +) + +# Zone IDs are OS-specific text strings with no format defined by the RFCs: +# https://datatracker.ietf.org/doc/html/rfc4007#section-11.2 +# RFC 9844 §6.3 recommends rejecting characters inappropriate for the +# environment; for yarl we reject ASCII control characters (CTL): +# https://datatracker.ietf.org/doc/html/rfc9844#section-6-3 +_ZONE_ID_UNSAFE_RE = re.compile(r"[\x00-\x1f\x7f]") + +_T = TypeVar("_T") + +if sys.version_info >= (3, 11): + from typing import Self +else: + Self = Any + + +class UndefinedType(Enum): + """Singleton type for use with not set sentinel values.""" + + _singleton = 0 + + +UNDEFINED = UndefinedType._singleton + + +class CacheInfo(TypedDict): + """Host encoding cache.""" + + idna_encode: _CacheInfo + idna_decode: _CacheInfo + ip_address: _CacheInfo + host_validate: _CacheInfo + encode_host: _CacheInfo + + +class _InternalURLCache(TypedDict, total=False): + _val: SplitURLType + _origin: "URL" + absolute: bool + hash: int + scheme: str + raw_authority: str + authority: str + raw_user: str | None + user: str | None + raw_password: str | None + password: str | None + raw_host: str | None + host: str | None + host_subcomponent: str | None + host_port_subcomponent: str | None + port: int | None + explicit_port: int | None + raw_path: str + path: str + _parsed_query: list[tuple[str, str]] + query: "MultiDictProxy[str]" + raw_query_string: str + query_string: str + path_qs: str + raw_path_qs: str + raw_fragment: str + fragment: str + raw_parts: tuple[str, ...] + parts: tuple[str, ...] + parent: "URL" + raw_name: str + name: str + raw_suffix: str + suffix: str + raw_suffixes: tuple[str, ...] + suffixes: tuple[str, ...] + + +def rewrite_module(obj: _T) -> _T: + obj.__module__ = "yarl" + return obj + + +def _encode_relative_scheme_colon(path: str) -> str: + """Re-encode a scheme-shaped leading ``:`` in a relative path to ``%3A``.""" + colon_pos = path.find(":") + if colon_pos <= 0: + return path + for c in path[:colon_pos]: + if c not in _SCHEME_CHARS: + return path + return path[:colon_pos] + "%3A" + path[colon_pos + 1 :] + + +@lru_cache +def encode_url(url_str: str) -> "URL": + """Parse unencoded URL.""" + cache: _InternalURLCache = {} + host: str | None + scheme, netloc, path, query, fragment = split_url(url_str) + if not netloc: # netloc + host = "" + else: + if ":" in netloc or "@" in netloc or "[" in netloc: + # Complex netloc + username, password, host, port = split_netloc(netloc) + else: + username = password = port = None + host = netloc + if host is None: + if scheme in SCHEME_REQUIRES_HOST: + msg = ( + "Invalid URL: host is required for " + f"absolute urls with the {scheme} scheme" + ) + raise ValueError(msg) + else: + host = "" + host = _encode_host(host, validate_host=False) + # Remove brackets as host encoder adds back brackets for IPv6 addresses + cache["raw_host"] = host[1:-1] if "[" in host else host + cache["explicit_port"] = port + if password is None and username is None: + # Fast path for URLs without user, password + netloc = host if port is None else f"{host}:{port}" + cache["raw_user"] = None + cache["raw_password"] = None + else: + raw_user = REQUOTER(username) if username else username + raw_password = REQUOTER(password) if password else password + netloc = make_netloc(raw_user, raw_password, host, port) + cache["raw_user"] = raw_user + cache["raw_password"] = raw_password + + if path: + path = PATH_REQUOTER(path) + if netloc and "." in path: + path = normalize_path(path) + elif not scheme and not netloc: + path = _encode_relative_scheme_colon(path) + if query: + query = QUERY_REQUOTER(query) + if fragment: + fragment = FRAGMENT_REQUOTER(fragment) + + cache["scheme"] = scheme + cache["raw_path"] = "/" if not path and netloc else path + cache["raw_query_string"] = query + cache["raw_fragment"] = fragment + + self = object.__new__(URL) + self._scheme = scheme + self._netloc = netloc + self._path = path + self._query = query + self._fragment = fragment + self._cache = cache + return self + + +@lru_cache +def pre_encoded_url(url_str: str) -> "URL": + """Parse pre-encoded URL.""" + self = object.__new__(URL) + val = split_url(url_str) + self._scheme, self._netloc, self._path, self._query, self._fragment = val + self._cache = {} + return self + + +@lru_cache +def build_pre_encoded_url( + scheme: str, + authority: str, + user: str | None, + password: str | None, + host: str, + port: int | None, + path: str, + query_string: str, + fragment: str, +) -> "URL": + """Build a pre-encoded URL from parts.""" + self = object.__new__(URL) + self._scheme = scheme + if authority: + self._netloc = authority + elif host: + if port is not None: + port = None if port == DEFAULT_PORTS.get(scheme) else port + if user is None and password is None: + self._netloc = host if port is None else f"{host}:{port}" + else: + self._netloc = make_netloc(user, password, host, port) + else: + self._netloc = "" + if path and not scheme and not self._netloc and ":" in path: + path = _encode_relative_scheme_colon(path) + self._path = path + self._query = query_string + self._fragment = fragment + self._cache = {} + return self + + +def from_parts_uncached( + scheme: str, netloc: str, path: str, query: str, fragment: str +) -> "URL": + """Create a new URL from parts.""" + self = object.__new__(URL) + self._scheme = scheme + self._netloc = netloc + if path and not scheme and not netloc and ":" in path: + path = _encode_relative_scheme_colon(path) + self._path = path + self._query = query + self._fragment = fragment + self._cache = {} + return self + + +from_parts = lru_cache(from_parts_uncached) + + +@rewrite_module +class URL: + # Don't derive from str + # follow pathlib.Path design + # probably URL will not suffer from pathlib problems: + # it's intended for libraries like aiohttp, + # not to be passed into standard library functions like os.open etc. + + # URL grammar (RFC 3986) + # pct-encoded = "%" HEXDIG HEXDIG + # reserved = gen-delims / sub-delims + # gen-delims = ":" / "/" / "?" / "#" / "[" / "]" / "@" + # sub-delims = "!" / "$" / "&" / "'" / "(" / ")" + # / "*" / "+" / "," / ";" / "=" + # unreserved = ALPHA / DIGIT / "-" / "." / "_" / "~" + # URI = scheme ":" hier-part [ "?" query ] [ "#" fragment ] + # hier-part = "//" authority path-abempty + # / path-absolute + # / path-rootless + # / path-empty + # scheme = ALPHA *( ALPHA / DIGIT / "+" / "-" / "." ) + # authority = [ userinfo "@" ] host [ ":" port ] + # userinfo = *( unreserved / pct-encoded / sub-delims / ":" ) + # host = IP-literal / IPv4address / reg-name + # IP-literal = "[" ( IPv6address / IPvFuture ) "]" + # IPvFuture = "v" 1*HEXDIG "." 1*( unreserved / sub-delims / ":" ) + # IPv6address = 6( h16 ":" ) ls32 + # / "::" 5( h16 ":" ) ls32 + # / [ h16 ] "::" 4( h16 ":" ) ls32 + # / [ *1( h16 ":" ) h16 ] "::" 3( h16 ":" ) ls32 + # / [ *2( h16 ":" ) h16 ] "::" 2( h16 ":" ) ls32 + # / [ *3( h16 ":" ) h16 ] "::" h16 ":" ls32 + # / [ *4( h16 ":" ) h16 ] "::" ls32 + # / [ *5( h16 ":" ) h16 ] "::" h16 + # / [ *6( h16 ":" ) h16 ] "::" + # ls32 = ( h16 ":" h16 ) / IPv4address + # ; least-significant 32 bits of address + # h16 = 1*4HEXDIG + # ; 16 bits of address represented in hexadecimal + # IPv4address = dec-octet "." dec-octet "." dec-octet "." dec-octet + # dec-octet = DIGIT ; 0-9 + # / %x31-39 DIGIT ; 10-99 + # / "1" 2DIGIT ; 100-199 + # / "2" %x30-34 DIGIT ; 200-249 + # / "25" %x30-35 ; 250-255 + # reg-name = *( unreserved / pct-encoded / sub-delims ) + # port = *DIGIT + # path = path-abempty ; begins with "/" or is empty + # / path-absolute ; begins with "/" but not "//" + # / path-noscheme ; begins with a non-colon segment + # / path-rootless ; begins with a segment + # / path-empty ; zero characters + # path-abempty = *( "/" segment ) + # path-absolute = "/" [ segment-nz *( "/" segment ) ] + # path-noscheme = segment-nz-nc *( "/" segment ) + # path-rootless = segment-nz *( "/" segment ) + # path-empty = 0 + # segment = *pchar + # segment-nz = 1*pchar + # segment-nz-nc = 1*( unreserved / pct-encoded / sub-delims / "@" ) + # ; non-zero-length segment without any colon ":" + # pchar = unreserved / pct-encoded / sub-delims / ":" / "@" + # query = *( pchar / "/" / "?" ) + # fragment = *( pchar / "/" / "?" ) + # URI-reference = URI / relative-ref + # relative-ref = relative-part [ "?" query ] [ "#" fragment ] + # relative-part = "//" authority path-abempty + # / path-absolute + # / path-noscheme + # / path-empty + # absolute-URI = scheme ":" hier-part [ "?" query ] + __slots__ = ("_cache", "_scheme", "_netloc", "_path", "_query", "_fragment") + + _cache: _InternalURLCache + _scheme: str + _netloc: str + _path: str + _query: str + _fragment: str + + def __new__( + cls, + val: Union[str, SplitResult, "URL", UndefinedType] = UNDEFINED, + *, + encoded: bool = False, + strict: bool | None = None, + ) -> "URL": + if strict is not None: # pragma: no cover + warnings.warn("strict parameter is ignored") + if type(val) is str: + return pre_encoded_url(val) if encoded else encode_url(val) + if type(val) is cls: + return val + if type(val) is SplitResult: + if not encoded: + raise ValueError("Cannot apply decoding to SplitResult") + return from_parts(*val) + if isinstance(val, str): + return pre_encoded_url(str(val)) if encoded else encode_url(str(val)) + if val is UNDEFINED: + # Special case for UNDEFINED since it might be unpickling and we do + # not want to cache as the `__set_state__` call would mutate the URL + # object in the `pre_encoded_url` or `encoded_url` caches. + self = object.__new__(URL) + self._scheme = self._netloc = self._path = self._query = self._fragment = "" + self._cache = {} + return self + raise TypeError("Constructor parameter should be str") + + @classmethod + def build( + cls, + *, + scheme: str = "", + authority: str = "", + user: str | None = None, + password: str | None = None, + host: str = "", + port: int | None = None, + path: str = "", + query: Query | None = None, + query_string: str = "", + fragment: str = "", + encoded: bool = False, + ) -> "URL": + """Creates and returns a new URL""" + + if authority and (user or password or host or port): + raise ValueError( + 'Can\'t mix "authority" with "user", "password", "host" or "port".' + ) + if port is not None and not isinstance(port, int): + raise TypeError(f"The port is required to be int, got {type(port)!r}.") + if port and not host: + raise ValueError('Can\'t build URL with "port" but without "host".') + if query and query_string: + raise ValueError('Only one of "query" or "query_string" should be passed') + if ( + scheme is None # type: ignore[redundant-expr] + or authority is None # type: ignore[redundant-expr] + or host is None # type: ignore[redundant-expr] + or path is None # type: ignore[redundant-expr] + or query_string is None # type: ignore[redundant-expr] + or fragment is None + ): + raise TypeError( + 'NoneType is illegal for "scheme", "authority", "host", "path", ' + '"query_string", and "fragment" args, use empty string instead.' + ) + + if query: + query_string = get_str_query(query) or "" + + if encoded: + return build_pre_encoded_url( + scheme, + authority, + user, + password, + host, + port, + path, + query_string, + fragment, + ) + + self = object.__new__(URL) + self._scheme = scheme + _host: str | None = None + if authority: + user, password, _host, port = split_netloc(authority) + _host = _encode_host(_host, validate_host=False) if _host else "" + elif host: + _host = _encode_host(host, validate_host=True) + else: + self._netloc = "" + + if _host is not None: + if port is not None: + port = None if port == DEFAULT_PORTS.get(scheme) else port + if user is None and password is None: + self._netloc = _host if port is None else f"{_host}:{port}" + else: + self._netloc = make_netloc(user, password, _host, port, True) + + path = PATH_QUOTER(path) if path else path + if path and self._netloc: + if "." in path: + path = normalize_path(path) + if path[0] != "/": + msg = ( + "Path in a URL with authority should " + "start with a slash ('/') if set" + ) + raise ValueError(msg) + + if path and not self._scheme and not self._netloc and ":" in path: + path = _encode_relative_scheme_colon(path) + self._path = path + if not query and query_string: + query_string = QUERY_QUOTER(query_string) + self._query = query_string + self._fragment = FRAGMENT_QUOTER(fragment) if fragment else fragment + self._cache = {} + return self + + def __init_subclass__(cls) -> NoReturn: + raise TypeError(f"Inheriting a class {cls!r} from URL is forbidden") + + def __str__(self) -> str: + if not self._path and self._netloc and (self._query or self._fragment): + path = "/" + else: + path = self._path + if (port := self.explicit_port) is not None and port == DEFAULT_PORTS.get( + self._scheme + ): + # port normalization - using None for default ports to remove from rendering + # https://datatracker.ietf.org/doc/html/rfc3986.html#section-6.2.3 + host = self.host_subcomponent + netloc = make_netloc(self.raw_user, self.raw_password, host, None) + else: + netloc = self._netloc + return unsplit_result(self._scheme, netloc, path, self._query, self._fragment) + + def __repr__(self) -> str: + return f"{self.__class__.__name__}('{str(self)}')" + + def __bytes__(self) -> bytes: + return str(self).encode("ascii") + + def __eq__(self, other: object) -> bool: + if type(other) is not URL: + return NotImplemented + + path1 = "/" if not self._path and self._netloc else self._path + path2 = "/" if not other._path and other._netloc else other._path + return ( + self._scheme == other._scheme + and self._netloc == other._netloc + and path1 == path2 + and self._query == other._query + and self._fragment == other._fragment + ) + + def __hash__(self) -> int: + if (ret := self._cache.get("hash")) is None: + path = "/" if not self._path and self._netloc else self._path + ret = self._cache["hash"] = hash( + (self._scheme, self._netloc, path, self._query, self._fragment) + ) + return ret + + def __le__(self, other: object) -> bool: + if type(other) is not URL: + return NotImplemented + return self._val <= other._val + + def __lt__(self, other: object) -> bool: + if type(other) is not URL: + return NotImplemented + return self._val < other._val + + def __ge__(self, other: object) -> bool: + if type(other) is not URL: + return NotImplemented + return self._val >= other._val + + def __gt__(self, other: object) -> bool: + if type(other) is not URL: + return NotImplemented + return self._val > other._val + + def __truediv__(self, name: str) -> "URL": + if not isinstance(name, str): + return NotImplemented + return self._make_child((str(name),)) + + def __mod__(self, query: Query) -> "URL": + return self.update_query(query) + + def __bool__(self) -> bool: + return bool(self._netloc or self._path or self._query or self._fragment) + + def __getstate__(self) -> tuple[SplitURLType]: + # Return a plain tuple rather than a ``SplitResult``. Constructing a + # ``SplitResult`` via ``tuple.__new__`` skips its ``__init__`` and on + # Python 3.15+ leaves ``_keep_empty`` unset, which breaks pickling: the + # new ``SplitResult.__getstate__`` indexes a state that ends up as + # ``None`` (gh-1632). ``__setstate__`` already unpacks both shapes, so + # pickles produced by older yarl releases (which embed a real + # ``SplitResult``) still load correctly. + return (self._val,) + + def __setstate__( + self, state: tuple[SplitURLType] | tuple[None, _InternalURLCache] + ) -> None: + if state[0] is None and isinstance(state[1], dict): + # default style pickle + val = state[1]["_val"] + else: + unused: list[object] + val, *unused = state + self._scheme, self._netloc, self._path, self._query, self._fragment = val + self._cache = {} + + def _cache_netloc(self) -> None: + """Cache the netloc parts of the URL.""" + c = self._cache + split_loc = split_netloc(self._netloc) + c["raw_user"], c["raw_password"], c["raw_host"], c["explicit_port"] = split_loc + + def is_absolute(self) -> bool: + """A check for absolute URLs. + + Return True for absolute ones (having scheme or starting + with //), False otherwise. + + Is is preferred to call the .absolute property instead + as it is cached. + """ + return self.absolute + + def is_default_port(self) -> bool: + """A check for default port. + + Return True if port is default for specified scheme, + e.g. 'http://python.org' or 'http://python.org:80', False + otherwise. + + Return False for relative URLs. + + """ + if (explicit := self.explicit_port) is None: + # If the explicit port is None, then the URL must be + # using the default port unless its a relative URL + # which does not have an implicit port / default port + return self._netloc != "" + return explicit == DEFAULT_PORTS.get(self._scheme) + + def origin(self) -> "URL": + """Return an URL with scheme, host and port parts only. + + user, password, path, query and fragment are removed. + + """ + # TODO: add a keyword-only option for keeping user/pass maybe? + return self._origin + + @cached_property + def _val(self) -> SplitURLType: + return (self._scheme, self._netloc, self._path, self._query, self._fragment) + + @cached_property + def _origin(self) -> "URL": + """Return an URL with scheme, host and port parts only. + + user, password, path, query and fragment are removed. + """ + if not (netloc := self._netloc): + raise ValueError("URL should be absolute") + if not (scheme := self._scheme): + raise ValueError("URL should have scheme") + if "@" in netloc: + encoded_host = self.host_subcomponent + netloc = make_netloc(None, None, encoded_host, self.explicit_port) + elif not self._path and not self._query and not self._fragment: + return self + return from_parts(scheme, netloc, "", "", "") + + def relative(self) -> "URL": + """Return a relative part of the URL. + + scheme, user, password, host and port are removed. + + """ + if not self._netloc: + raise ValueError("URL should be absolute") + return from_parts("", "", self._path, self._query, self._fragment) + + @cached_property + def absolute(self) -> bool: + """A check for absolute URLs. + + Return True for absolute ones (having scheme or starting + with //), False otherwise. + + """ + # `netloc`` is an empty string for relative URLs + # Checking `netloc` is faster than checking `hostname` + # because `hostname` is a property that does some extra work + # to parse the host from the `netloc` + return self._netloc != "" + + @cached_property + def scheme(self) -> str: + """Scheme for absolute URLs. + + Empty string for relative URLs or URLs starting with // + + """ + return self._scheme + + @cached_property + def raw_authority(self) -> str: + """Encoded authority part of URL. + + Empty string for relative URLs. + + """ + return self._netloc + + @cached_property + def authority(self) -> str: + """Decoded authority part of URL. + + Empty string for relative URLs. + + """ + return make_netloc(self.user, self.password, self.host, self.port) + + @cached_property + def raw_user(self) -> str | None: + """Encoded user part of URL. + + None if user is missing. + + """ + # not .username + self._cache_netloc() + return self._cache["raw_user"] + + @cached_property + def user(self) -> str | None: + """Decoded user part of URL. + + None if user is missing. + + """ + if (raw_user := self.raw_user) is None: + return None + return UNQUOTER(raw_user) + + @cached_property + def raw_password(self) -> str | None: + """Encoded password part of URL. + + None if password is missing. + + """ + self._cache_netloc() + return self._cache["raw_password"] + + @cached_property + def password(self) -> str | None: + """Decoded password part of URL. + + None if password is missing. + + """ + if (raw_password := self.raw_password) is None: + return None + return UNQUOTER(raw_password) + + @cached_property + def raw_host(self) -> str | None: + """Encoded host part of URL. + + None for relative URLs. + + When working with IPv6 addresses, use the `host_subcomponent` property instead + as it will return the host subcomponent with brackets. + """ + # Use host instead of hostname for sake of shortness + # May add .hostname prop later + self._cache_netloc() + return self._cache["raw_host"] + + @cached_property + def host(self) -> str | None: + """Decoded host part of URL. + + None for relative URLs. + + For IPv6 hosts that carry an RFC 6874 zone identifier, the + ``%25`` zone separator is decoded back to ``%``; the encoded + form is still available via :attr:`raw_host` and + :attr:`host_subcomponent`. + + """ + if (raw := self.raw_host) is None: + return None + if raw and raw[-1].isdigit() or ":" in raw: + # IP addresses are never IDNA encoded. The replace decodes + # every %25 in the raw host, i.e. the RFC 6874 zone + # separator and any %25 that percent-encodes a literal % + # inside the zone identifier. + if "%25" in raw: + return raw.replace("%25", "%") + return raw + return _idna_decode(raw) + + @cached_property + def host_subcomponent(self) -> str | None: + """Return the host subcomponent part of URL. + + None for relative URLs. + + https://datatracker.ietf.org/doc/html/rfc3986#section-3.2.2 + + `IP-literal = "[" ( IPv6address / IPvFuture ) "]"` + + Examples: + - `http://example.com:8080` -> `example.com` + - `http://example.com:80` -> `example.com` + - `https://127.0.0.1:8443` -> `127.0.0.1` + - `https://[::1]:8443` -> `[::1]` + - `http://[::1]` -> `[::1]` + + """ + if (raw := self.raw_host) is None: + return None + return f"[{raw}]" if ":" in raw else raw + + @cached_property + def host_port_subcomponent(self) -> str | None: + """Return the host and port subcomponent part of URL. + + Trailing dots are removed from the host part. + + This value is suitable for use in the Host header of an HTTP request. + + None for relative URLs. + + https://datatracker.ietf.org/doc/html/rfc3986#section-3.2.2 + `IP-literal = "[" ( IPv6address / IPvFuture ) "]"` + https://datatracker.ietf.org/doc/html/rfc3986#section-3.2.3 + port = *DIGIT + + Examples: + - `http://example.com:8080` -> `example.com:8080` + - `http://example.com:80` -> `example.com` + - `http://example.com.:80` -> `example.com` + - `https://127.0.0.1:8443` -> `127.0.0.1:8443` + - `https://[::1]:8443` -> `[::1]:8443` + - `http://[::1]` -> `[::1]` + + """ + if (raw := self.raw_host) is None: + return None + if raw[-1] == ".": + # Remove all trailing dots from the netloc as while + # they are valid FQDNs in DNS, TLS validation fails. + # See https://github.com/aio-libs/aiohttp/issues/3636. + # To avoid string manipulation we only call rstrip if + # the last character is a dot. + raw = raw.rstrip(".") + port = self.explicit_port + if port is None or port == DEFAULT_PORTS.get(self._scheme): + return f"[{raw}]" if ":" in raw else raw + return f"[{raw}]:{port}" if ":" in raw else f"{raw}:{port}" + + @cached_property + def port(self) -> int | None: + """Port part of URL, with scheme-based fallback. + + None for relative URLs or URLs without explicit port and + scheme without default port substitution. + + """ + if (explicit_port := self.explicit_port) is not None: + return explicit_port + return DEFAULT_PORTS.get(self._scheme) + + @cached_property + def explicit_port(self) -> int | None: + """Port part of URL, without scheme-based fallback. + + None for relative URLs or URLs without explicit port. + + """ + self._cache_netloc() + return self._cache["explicit_port"] + + @cached_property + def raw_path(self) -> str: + """Encoded path of URL. + + / for absolute URLs without path part. + + """ + return self._path if self._path or not self._netloc else "/" + + @cached_property + def path(self) -> str: + """Decoded path of URL. + + / for absolute URLs without path part. + + """ + return PATH_UNQUOTER(self._path) if self._path else "/" if self._netloc else "" + + @cached_property + def path_safe(self) -> str: + """Decoded path of URL. + + / for absolute URLs without path part. + + / (%2F) and % (%25) are not decoded + + """ + if self._path: + return PATH_SAFE_UNQUOTER(self._path) + return "/" if self._netloc else "" + + @cached_property + def _parsed_query(self) -> list[tuple[str, str]]: + """Parse query part of URL.""" + return query_to_pairs(self._query) + + @cached_property + def query(self) -> "MultiDictProxy[str]": + """A MultiDictProxy representing parsed query parameters in decoded + representation. + + Empty value if URL has no query part. + + """ + return MultiDictProxy(MultiDict(self._parsed_query)) + + @cached_property + def raw_query_string(self) -> str: + """Encoded query part of URL. + + Empty string if query is missing. + + """ + return self._query + + @cached_property + def query_string(self) -> str: + """Decoded query part of URL. + + Empty string if query is missing. + + """ + return QS_UNQUOTER(self._query) if self._query else "" + + @cached_property + def path_qs(self) -> str: + """Decoded path of URL with query.""" + return self.path if not (q := self.query_string) else f"{self.path}?{q}" + + @cached_property + def raw_path_qs(self) -> str: + """Encoded path of URL with query.""" + if q := self._query: + return f"{self._path}?{q}" if self._path or not self._netloc else f"/?{q}" + return self._path if self._path or not self._netloc else "/" + + @cached_property + def raw_fragment(self) -> str: + """Encoded fragment part of URL. + + Empty string if fragment is missing. + + """ + return self._fragment + + @cached_property + def fragment(self) -> str: + """Decoded fragment part of URL. + + Empty string if fragment is missing. + + """ + return UNQUOTER(self._fragment) if self._fragment else "" + + @cached_property + def raw_parts(self) -> tuple[str, ...]: + """A tuple containing encoded *path* parts. + + ('/',) for absolute URLs if *path* is missing. + + """ + path = self._path + if self._netloc: + return ("/", *path[1:].split("/")) if path else ("/",) + if path and path[0] == "/": + return ("/", *path[1:].split("/")) + return tuple(path.split("/")) + + @cached_property + def parts(self) -> tuple[str, ...]: + """A tuple containing decoded *path* parts. + + ('/',) for absolute URLs if *path* is missing. + + """ + return tuple(UNQUOTER(part) for part in self.raw_parts) + + @cached_property + def parent(self) -> "URL": + """A new URL with last part of path removed and cleaned up query and + fragment. + + """ + path = self._path + if not path or path == "/": + if self._fragment or self._query: + return from_parts(self._scheme, self._netloc, path, "", "") + return self + parts = path.split("/") + return from_parts(self._scheme, self._netloc, "/".join(parts[:-1]), "", "") + + @cached_property + def raw_name(self) -> str: + """The last part of raw_parts.""" + parts = self.raw_parts + if not self._netloc: + return parts[-1] + parts = parts[1:] + return parts[-1] if parts else "" + + @cached_property + def name(self) -> str: + """The last part of parts.""" + return UNQUOTER(self.raw_name) + + @cached_property + def raw_suffix(self) -> str: + name = self.raw_name + i = name.rfind(".") + return name[i:] if 0 < i < len(name) - 1 else "" + + @cached_property + def suffix(self) -> str: + return UNQUOTER(self.raw_suffix) + + @cached_property + def raw_suffixes(self) -> tuple[str, ...]: + name = self.raw_name + if name.endswith("."): + return () + name = name.lstrip(".") + return tuple("." + suffix for suffix in name.split(".")[1:]) + + @cached_property + def suffixes(self) -> tuple[str, ...]: + return tuple(UNQUOTER(suffix) for suffix in self.raw_suffixes) + + def _make_child(self, paths: "Sequence[str]", encoded: bool = False) -> "URL": + """ + add paths to self._path, accounting for absolute vs relative paths, + keep existing, but do not create new, empty segments + """ + parsed: list[str] = [] + needs_normalize: bool = False + for idx, path in enumerate(reversed(paths)): + # empty segment of last is not removed + last = idx == 0 + if path and path[0] == "/": + raise ValueError( + f"Appending path {path!r} starting from slash is forbidden" + ) + # We need to quote the path if it is not already encoded + # This cannot be done at the end because the existing + # path is already quoted and we do not want to double quote + # the existing path. + path = path if encoded else PATH_QUOTER(path) + needs_normalize |= "." in path + segments = path.split("/") + segments.reverse() + # remove trailing empty segment for all but the last path + parsed += segments[1:] if not last and segments[0] == "" else segments + + if (path := self._path) and (old_segments := path.split("/")): + # If the old path ends with a slash, the last segment is an empty string + # and should be removed before adding the new path segments. + old = old_segments[:-1] if old_segments[-1] == "" else old_segments + old.reverse() + parsed += old + + # If the netloc is present, inject a leading slash when adding a + # path to an absolute URL where there was none before. + if (netloc := self._netloc) and parsed and parsed[-1] != "": + parsed.append("") + + parsed.reverse() + if not netloc or not needs_normalize: + return from_parts(self._scheme, netloc, "/".join(parsed), "", "") + + path = "/".join(normalize_path_segments(parsed)) + # If normalizing the path segments removed the leading slash, add it back. + if path and path[0] != "/": + path = f"/{path}" + return from_parts(self._scheme, netloc, path, "", "") + + def with_scheme(self, scheme: str) -> "URL": + """Return a new URL with scheme replaced.""" + # N.B. doesn't cleanup query/fragment + if not isinstance(scheme, str): + raise TypeError("Invalid scheme type") + lower_scheme = scheme.lower() + netloc = self._netloc + if not netloc and lower_scheme in SCHEME_REQUIRES_HOST: + msg = ( + "scheme replacement is not allowed for " + f"relative URLs for the {lower_scheme} scheme" + ) + raise ValueError(msg) + return from_parts(lower_scheme, netloc, self._path, self._query, self._fragment) + + def with_user(self, user: str | None) -> "URL": + """Return a new URL with user replaced. + + Autoencode user if needed. + + Clear user/password if user is None. + + """ + # N.B. doesn't cleanup query/fragment + if user is None: + password = None + elif isinstance(user, str): + user = QUOTER(user) + password = self.raw_password + else: + raise TypeError("Invalid user type") + if not (netloc := self._netloc): + raise ValueError("user replacement is not allowed for relative URLs") + encoded_host = self.host_subcomponent or "" + netloc = make_netloc(user, password, encoded_host, self.explicit_port) + return from_parts(self._scheme, netloc, self._path, self._query, self._fragment) + + def with_password(self, password: str | None) -> "URL": + """Return a new URL with password replaced. + + Autoencode password if needed. + + Clear password if argument is None. + + """ + # N.B. doesn't cleanup query/fragment + if password is None: + pass + elif isinstance(password, str): + password = QUOTER(password) + else: + raise TypeError("Invalid password type") + if not (netloc := self._netloc): + raise ValueError("password replacement is not allowed for relative URLs") + encoded_host = self.host_subcomponent or "" + port = self.explicit_port + netloc = make_netloc(self.raw_user, password, encoded_host, port) + return from_parts(self._scheme, netloc, self._path, self._query, self._fragment) + + def with_host(self, host: str) -> "URL": + """Return a new URL with host replaced. + + Autoencode host if needed. + + Changing host for relative URLs is not allowed, use .join() + instead. + + """ + # N.B. doesn't cleanup query/fragment + if not isinstance(host, str): + raise TypeError("Invalid host type") + if not (netloc := self._netloc): + raise ValueError("host replacement is not allowed for relative URLs") + if not host: + raise ValueError("host removing is not allowed") + encoded_host = _encode_host(host, validate_host=True) if host else "" + port = self.explicit_port + netloc = make_netloc(self.raw_user, self.raw_password, encoded_host, port) + return from_parts(self._scheme, netloc, self._path, self._query, self._fragment) + + def with_port(self, port: int | None) -> "URL": + """Return a new URL with port replaced. + + Clear port to default if None is passed. + + """ + # N.B. doesn't cleanup query/fragment + if port is not None: + if isinstance(port, bool) or not isinstance(port, int): + raise TypeError(f"port should be int or None, got {type(port)}") + if not (0 <= port <= 65535): + raise ValueError(f"port must be between 0 and 65535, got {port}") + if not (netloc := self._netloc): + raise ValueError("port replacement is not allowed for relative URLs") + encoded_host = self.host_subcomponent or "" + netloc = make_netloc(self.raw_user, self.raw_password, encoded_host, port) + return from_parts(self._scheme, netloc, self._path, self._query, self._fragment) + + def with_path( + self, + path: str, + *, + encoded: bool = False, + keep_query: bool = False, + keep_fragment: bool = False, + ) -> "URL": + """Return a new URL with path replaced.""" + netloc = self._netloc + if not encoded: + path = PATH_QUOTER(path) + if netloc: + path = normalize_path(path) if "." in path else path + if path and path[0] != "/": + path = f"/{path}" + query = self._query if keep_query else "" + fragment = self._fragment if keep_fragment else "" + return from_parts(self._scheme, netloc, path, query, fragment) + + @overload + def with_query(self, query: Query) -> "URL": ... + + @overload + def with_query(self, **kwargs: QueryVariable) -> "URL": ... + + def with_query(self, *args: Any, **kwargs: Any) -> "URL": + """Return a new URL with query part replaced. + + Accepts any Mapping (e.g. dict, multidict.MultiDict instances) + or str, autoencode the argument if needed. + + A sequence of (key, value) pairs is supported as well. + + It also can take an arbitrary number of keyword arguments. + + Clear query if None is passed. + + """ + # N.B. doesn't cleanup query/fragment + query = get_str_query(*args, **kwargs) or "" + return from_parts_uncached( + self._scheme, self._netloc, self._path, query, self._fragment + ) + + @overload + def extend_query(self, query: Query) -> "URL": ... + + @overload + def extend_query(self, **kwargs: QueryVariable) -> "URL": ... + + def extend_query(self, *args: Any, **kwargs: Any) -> "URL": + """Return a new URL with query part combined with the existing. + + This method will not remove existing query parameters. + + Example: + >>> url = URL('http://example.com/?a=1&b=2') + >>> url.extend_query(a=3, c=4) + URL('http://example.com/?a=1&b=2&a=3&c=4') + """ + if not (new_query := get_str_query(*args, **kwargs)): + return self + if query := self._query: + # both strings are already encoded so we can use a simple + # string join + query += new_query if query[-1] == "&" else f"&{new_query}" + else: + query = new_query + return from_parts_uncached( + self._scheme, self._netloc, self._path, query, self._fragment + ) + + @overload + def update_query(self, query: Query) -> "URL": ... + + @overload + def update_query(self, **kwargs: QueryVariable) -> "URL": ... + + def update_query(self, *args: Any, **kwargs: Any) -> "URL": + """Return a new URL with query part updated. + + This method will overwrite existing query parameters. + + Example: + >>> url = URL('http://example.com/?a=1&b=2') + >>> url.update_query(a=3, c=4) + URL('http://example.com/?a=3&b=2&c=4') + """ + in_query: ( + str + | Mapping[str, QueryVariable] + | Sequence[tuple[str | istr, SimpleQuery]] + | None + ) + if kwargs: + if args: + msg = "Either kwargs or single query parameter must be present" + raise ValueError(msg) + in_query = kwargs + elif len(args) == 1: + in_query = args[0] + else: + raise ValueError("Either kwargs or single query parameter must be present") + + if in_query is None: + query = "" + elif not in_query: + query = self._query + elif isinstance(in_query, Mapping): + qm: MultiDict[QueryVariable] = MultiDict(self._parsed_query) + qm.update(in_query) + query = get_str_query_from_sequence_iterable(qm.items()) + elif isinstance(in_query, str): + qstr: MultiDict[str] = MultiDict(self._parsed_query) + qstr.update(query_to_pairs(in_query)) + query = get_str_query_from_iterable(qstr.items()) + elif isinstance(in_query, (bytes, bytearray, memoryview)): + msg = "Invalid query type: bytes, bytearray and memoryview are forbidden" + raise TypeError(msg) + elif isinstance(in_query, Sequence): + # We don't expect sequence values if we're given a list of pairs + # already; only mappings like builtin `dict` which can't have the + # same key pointing to multiple values are allowed to use + # `_query_seq_pairs`. + if TYPE_CHECKING: + in_query = cast( + Sequence[tuple[Union[str, istr], SimpleQuery]], in_query + ) + qs: MultiDict[SimpleQuery] = MultiDict(self._parsed_query) + qs.update(in_query) + query = get_str_query_from_iterable(qs.items()) + else: + raise TypeError( + "Invalid query type: only str, mapping or " + "sequence of (key, value) pairs is allowed" + ) + return from_parts_uncached( + self._scheme, self._netloc, self._path, query, self._fragment + ) + + def without_query_params(self, *query_params: str) -> "URL": + """Remove some keys from query part and return new URL.""" + params_to_remove = set(query_params) & self.query.keys() + if not params_to_remove: + return self + return self.with_query( + tuple( + (name, value) + for name, value in self.query.items() + if name not in params_to_remove + ) + ) + + def with_fragment(self, fragment: str | None) -> "URL": + """Return a new URL with fragment replaced. + + Autoencode fragment if needed. + + Clear fragment to default if None is passed. + + """ + # N.B. doesn't cleanup query/fragment + if fragment is None: + raw_fragment = "" + elif not isinstance(fragment, str): + raise TypeError("Invalid fragment type") + else: + raw_fragment = FRAGMENT_QUOTER(fragment) + if self._fragment == raw_fragment: + return self + return from_parts( + self._scheme, self._netloc, self._path, self._query, raw_fragment + ) + + def with_name( + self, + name: str, + *, + keep_query: bool = False, + keep_fragment: bool = False, + ) -> "URL": + """Return a new URL with name (last part of path) replaced. + + Query and fragment parts are cleaned up. + + Name is encoded if needed. + + """ + # N.B. DOES cleanup query/fragment + if not isinstance(name, str): + raise TypeError("Invalid name type") + if "/" in name: + raise ValueError("Slash in name is not allowed") + name = PATH_QUOTER(name) + if name in (".", ".."): + raise ValueError(". and .. values are forbidden") + parts = list(self.raw_parts) + if netloc := self._netloc: + if len(parts) == 1: + parts.append(name) + else: + parts[-1] = name + parts[0] = "" # replace leading '/' + else: + parts[-1] = name + if parts[0] == "/": + parts[0] = "" # replace leading '/' + + query = self._query if keep_query else "" + fragment = self._fragment if keep_fragment else "" + return from_parts(self._scheme, netloc, "/".join(parts), query, fragment) + + def with_suffix( + self, + suffix: str, + *, + keep_query: bool = False, + keep_fragment: bool = False, + ) -> "URL": + """Return a new URL with suffix (file extension of name) replaced. + + Query and fragment parts are cleaned up. + + suffix is encoded if needed. + """ + if not isinstance(suffix, str): + raise TypeError("Invalid suffix type") + if suffix and not suffix[0] == "." or suffix == "." or "/" in suffix: + raise ValueError(f"Invalid suffix {suffix!r}") + name = self.raw_name + if not name: + raise ValueError(f"{self!r} has an empty name") + old_suffix = self.raw_suffix + suffix = PATH_QUOTER(suffix) + name = name + suffix if not old_suffix else name[: -len(old_suffix)] + suffix + if name in (".", ".."): + raise ValueError(". and .. values are forbidden") + parts = list(self.raw_parts) + if netloc := self._netloc: + if len(parts) == 1: + parts.append(name) + else: + parts[-1] = name + parts[0] = "" # replace leading '/' + else: + parts[-1] = name + if parts[0] == "/": + parts[0] = "" # replace leading '/' + + query = self._query if keep_query else "" + fragment = self._fragment if keep_fragment else "" + return from_parts(self._scheme, netloc, "/".join(parts), query, fragment) + + def join(self, url: "URL") -> "URL": + """Join URLs + + Construct a full (“absolute”) URL by combining a “base URL” + (self) with another URL (url). + + Informally, this uses components of the base URL, in + particular the addressing scheme, the network location and + (part of) the path, to provide missing components in the + relative URL. + + """ + if type(url) is not URL: + raise TypeError("url should be URL") + + scheme = url._scheme or self._scheme + if scheme != self._scheme or scheme not in USES_RELATIVE: + return url + + # scheme is in uses_authority as uses_authority is a superset of uses_relative + if (join_netloc := url._netloc) and scheme in USES_AUTHORITY: + return from_parts(scheme, join_netloc, url._path, url._query, url._fragment) + + orig_path = self._path + if join_path := url._path: + if join_path[0] == "/": + path = join_path + elif not orig_path: + path = f"/{join_path}" + elif orig_path[-1] == "/": + path = f"{orig_path}{join_path}" + else: + # … + # and relativizing ".." + # parts[0] is / for absolute urls, + # this join will add a double slash there + path = "/".join([*self.parts[:-1], ""]) + join_path + # which has to be removed + if orig_path[0] == "/": + path = path[1:] + path = normalize_path(path) if "." in path else path + else: + path = orig_path + + return from_parts( + scheme, + self._netloc, + path, + url._query if join_path or url._query else self._query, + url._fragment if join_path or url._fragment else self._fragment, + ) + + def joinpath(self, *other: str, encoded: bool = False) -> "URL": + """Return a new URL with the elements in other appended to the path.""" + return self._make_child(other, encoded=encoded) + + def human_repr(self) -> str: + """Return decoded human readable string for URL representation.""" + user = human_quote(self.user, "#/:?@[]\\") + password = human_quote(self.password, "#/:?@[]\\") + if (host := self.host) and ":" in host: + host = f"[{host}]" + path = human_quote(self.path, "#?") + if TYPE_CHECKING: + assert path is not None + if not self._scheme and not self._netloc: + path = _encode_relative_scheme_colon(path) + query_string = "&".join( + "{}={}".format(human_quote(k, "#&+;="), human_quote(v, "#&+;=")) + for k, v in self.query.items() + ) + fragment = human_quote(self.fragment, "") + if TYPE_CHECKING: + assert fragment is not None + netloc = make_netloc(user, password, host, self.explicit_port) + return unsplit_result(self._scheme, netloc, path, query_string, fragment) + + if HAS_PYDANTIC: + # Borrowed from https://docs.pydantic.dev/latest/concepts/types/#handling-third-party-types + @classmethod + def __get_pydantic_json_schema__( + cls, + core_schema: "CoreSchema", + handler: "GetJsonSchemaHandler", + ) -> "JsonSchemaValue": + field_schema: dict[str, Any] = {} + field_schema.update(type="string", format="uri") + return field_schema + + @classmethod + def __get_pydantic_core_schema__( + cls, + source_type: type[Self] | type[str], + handler: "GetCoreSchemaHandler", + ) -> "CoreSchema": + # Lazy import: pulling in pydantic_core at module load time + # increases yarl's import cost 3-7x for users who don't use + # pydantic. Keep this import function-scoped. + from pydantic_core import core_schema # noqa: PLC0415 + + from_str_schema = core_schema.chain_schema( + [ + core_schema.str_schema(), + core_schema.no_info_plain_validator_function(URL), + ] + ) + + return core_schema.json_or_python_schema( + json_schema=from_str_schema, + python_schema=core_schema.union_schema( + [ + # check if it's an instance first before doing any further work + core_schema.is_instance_schema(URL), + from_str_schema, + ] + ), + serialization=core_schema.plain_serializer_function_ser_schema(str), + ) + + +_DEFAULT_IDNA_SIZE = 256 +_DEFAULT_ENCODE_SIZE = 512 + + +@lru_cache(_DEFAULT_IDNA_SIZE) +def _idna_decode(raw: str) -> str: + try: + return idna.decode(raw.encode("ascii")) + except UnicodeError: # e.g. '::1' + return raw.encode("ascii").decode("idna") + + +@lru_cache(_DEFAULT_IDNA_SIZE) +def _idna_encode(host: str) -> str: + try: + return idna.encode(host, uts46=True).decode("ascii") + except UnicodeError: + return host.encode("idna").decode("ascii") + + +@lru_cache(_DEFAULT_ENCODE_SIZE) +def _encode_host(host: str, validate_host: bool) -> str: + """Encode host part of URL.""" + # If the host ends with a digit or contains a colon, its likely + # an IP address. + if host and (host[-1].isdigit() or ":" in host): + # RFC 6874 spells the IPv6 zone separator as the percent-encoded + # ``%25``; bare ``%`` is still accepted so that hosts constructed + # programmatically (e.g. ``with_host("fe80::1%1")``) keep working. + part = "%25" if "%25" in host else "%" + raw_ip, sep, zone = host.partition(part) + # If it looks like an IP, we check with _ip_compressed_version + # and fall-through if its not an IP address. This is a performance + # optimization to avoid parsing IP addresses as much as possible + # because it is orders of magnitude slower than almost any other + # operation this library does. + # Might be an IP address, check it + # + # IP Addresses can look like: + # https://datatracker.ietf.org/doc/html/rfc3986#section-3.2.2 + # - 127.0.0.1 (last character is a digit) + # - 2001:db8::ff00:42:8329 (contains a colon) + # - 2001:db8::ff00:42:8329%eth0 (contains a colon) + # - [2001:db8::ff00:42:8329] (contains a colon -- brackets should + # have been removed before it gets here) + # Rare IP Address formats are not supported per: + # https://datatracker.ietf.org/doc/html/rfc3986#section-7.4 + # + # IP parsing is slow, so its wrapped in an LRU + try: + ip = ip_address(raw_ip) + except ValueError: + pass + else: + if sep and validate_host and (not zone or _ZONE_ID_UNSAFE_RE.search(zone)): + raise ValueError("Invalid characters in zone identifier") + # These checks should not happen in the + # LRU to keep the cache size small + host = ip.compressed + if ip.version == 6: + return f"[{host}{sep}{zone}]" if sep else f"[{host}]" + return f"{host}{sep}{zone}" if sep else host + + # IDNA encoding is slow, skip it for ASCII-only strings + if host.isascii(): + # Check for invalid characters explicitly; _idna_encode() does this + # for non-ascii host names. + host = host.lower() + if validate_host and (invalid := NOT_REG_NAME.search(host)): + value, pos, extra = invalid.group(), invalid.start(), "" + if value == "@" or (value == ":" and "@" in host[pos:]): + # this looks like an authority string + extra = ( + ", if the value includes a username or password, " + "use 'authority' instead of 'host'" + ) + raise ValueError( + f"Host {host!r} cannot contain {value!r} (at position {pos}){extra}" + ) from None + return host + + # IDNA/UTS-46 mapping silently deletes default-ignorable code points, which + # would turn e.g. ``evil.com`` into ``evil.com``, a different host + # than the string the caller supplied. Reject them on every path (this runs + # regardless of ``validate_host`` since the plain ``URL(str)`` constructor + # encodes with ``validate_host=False``) so the parsed host cannot diverge + # from the input, matching idna/httpx/urllib3. + if invalid := _DEFAULT_IGNORABLE_RE.search(host): + raise ValueError( + f"Host {host!r} cannot contain {invalid.group()!r} " + f"(at position {invalid.start()})" + ) from None + encoded = _idna_encode(host) + # IDNA uses NFKC equivalence, so normalization can expand a non-ascii + # character into an ASCII delimiter (e.g. the fullwidth solidus U+FF0F + # becomes '/'). The ascii branch above rejects such delimiters directly; + # apply the same check to the IDNA output so the builder APIs agree with + # the parser's _check_netloc. + if validate_host and (invalid := NOT_REG_NAME.search(encoded)): + raise ValueError( + f"Host {host!r} cannot contain {invalid.group()!r} " + f"after IDNA normalization to {encoded!r}" + ) from None + return encoded + + +@rewrite_module +def cache_clear() -> None: + """Clear all LRU caches.""" + _idna_encode.cache_clear() + _idna_decode.cache_clear() + _encode_host.cache_clear() + + +@rewrite_module +def cache_info() -> CacheInfo: + """Report cache statistics.""" + return { + "idna_encode": _idna_encode.cache_info(), + "idna_decode": _idna_decode.cache_info(), + "ip_address": _encode_host.cache_info(), + "host_validate": _encode_host.cache_info(), + "encode_host": _encode_host.cache_info(), + } + + +@rewrite_module +def cache_configure( + *, + idna_encode_size: int | None = _DEFAULT_IDNA_SIZE, + idna_decode_size: int | None = _DEFAULT_IDNA_SIZE, + ip_address_size: int | None | UndefinedType = UNDEFINED, + host_validate_size: int | None | UndefinedType = UNDEFINED, + encode_host_size: int | None | UndefinedType = UNDEFINED, +) -> None: + """Configure LRU cache sizes.""" + global _idna_decode, _idna_encode, _encode_host + # ip_address_size, host_validate_size are no longer + # used, but are kept for backwards compatibility. + if ip_address_size is not UNDEFINED or host_validate_size is not UNDEFINED: + warnings.warn( + "cache_configure() no longer accepts the " + "ip_address_size or host_validate_size arguments, " + "they are used to set the encode_host_size instead " + "and will be removed in the future", + DeprecationWarning, + stacklevel=2, + ) + + if encode_host_size is not None: + for size in (ip_address_size, host_validate_size): + if size is None: + encode_host_size = None + elif encode_host_size is UNDEFINED: + if size is not UNDEFINED: + encode_host_size = size + elif size is not UNDEFINED: + if TYPE_CHECKING: + assert isinstance(size, int) + assert isinstance(encode_host_size, int) + encode_host_size = max(size, encode_host_size) + if encode_host_size is UNDEFINED: + encode_host_size = _DEFAULT_ENCODE_SIZE + + _encode_host = lru_cache(encode_host_size)(_encode_host.__wrapped__) + _idna_decode = lru_cache(idna_decode_size)(_idna_decode.__wrapped__) + _idna_encode = lru_cache(idna_encode_size)(_idna_encode.__wrapped__) diff --git a/py_modules/yarl/py.typed b/py_modules/yarl/py.typed new file mode 100644 index 0000000..dcf2c80 --- /dev/null +++ b/py_modules/yarl/py.typed @@ -0,0 +1 @@ +# Placeholder diff --git a/scripts/bundle-deps.sh b/scripts/bundle-deps.sh new file mode 100755 index 0000000..f467ab3 --- /dev/null +++ b/scripts/bundle-deps.sh @@ -0,0 +1,55 @@ +#!/usr/bin/env bash +set -euo pipefail + +PY_MODULES_DIR="$(dirname "$0")/../py_modules" +REQUIREMENTS="$(dirname "$0")/../requirements.txt" +TMPDIR=$(mktemp -d) +trap 'rm -rf "$TMPDIR"' EXIT + +# Detect Python version (default: 3.13) +PYTHON_VERSION="${PYTHON_VERSION:-3.11}" +ABI="cp${PYTHON_VERSION//./}" +PLATFORMS="manylinux2014_x86_64,manylinux_2_17_x86_64,linux_x86_64" + +echo "==> Bundling Python dependencies for Python ${PYTHON_VERSION}" +echo "==> Requirements: ${REQUIREMENTS}" +echo "==> Output: ${PY_MODULES_DIR}" + +# Clean py_modules +rm -rf "${PY_MODULES_DIR:?}" +mkdir -p "${PY_MODULES_DIR}" + +# Download all dependencies (with transitive deps) +echo "==> Downloading wheels..." +pip download \ + --python-version "${PYTHON_VERSION}" \ + --abi "${ABI}" \ + --only-binary=:all: \ + --platform manylinux2014_x86_64 \ + --platform manylinux_2_17_x86_64 \ + --platform linux_x86_64 \ + -r "${REQUIREMENTS}" \ + -d "${TMPDIR}" + +echo "==> Downloaded $(ls "${TMPDIR}"/*.whl 2>/dev/null | wc -l) wheels" + +# Extract into py_modules +echo "==> Extracting wheels..." +for whl in "${TMPDIR}"/*.whl; do + unzip -qo "$whl" -d "${PY_MODULES_DIR}" +done + +# Clean up dist-info and RECORD +echo "==> Cleaning up..." +find "${PY_MODULES_DIR}" -name "*.dist-info" -type d -exec rm -rf {} + 2>/dev/null || true +find "${PY_MODULES_DIR}" -name "*.egg-info" -type d -exec rm -rf {} + 2>/dev/null || true +find "${PY_MODULES_DIR}" -name "RECORD" -delete 2>/dev/null || true +find "${PY_MODULES_DIR}" -name "*.pyc" -delete 2>/dev/null || true + +# Show results +echo "==> Bundled packages:" +ls -d "${PY_MODULES_DIR}"/*/ 2>/dev/null | xargs -I{} basename {} | sort +echo "" +SO_COUNT=$(find "${PY_MODULES_DIR}" -name "*.so" | wc -l) +echo "==> ${SO_COUNT} native extensions (.so files)" +echo "==> Done!"