diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 96c9d2a..0ca5d0d 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -40,6 +40,7 @@ jobs: - "3.8" - "3.9" - "3.10" + - "3.11" steps: - uses: actions/checkout@v3 diff --git a/.github/workflows/periodic-ci.yaml b/.github/workflows/periodic-ci.yaml index a76a81f..d68468b 100644 --- a/.github/workflows/periodic-ci.yaml +++ b/.github/workflows/periodic-ci.yaml @@ -18,6 +18,7 @@ jobs: - "3.8" - "3.9" - "3.10" + - "3.11" steps: - uses: actions/checkout@v3 diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index c69e419..8dfa92b 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,19 +1,19 @@ repos: - repo: https://github.com/pre-commit/pre-commit-hooks - rev: v4.3.0 + rev: v4.4.0 hooks: - id: trailing-whitespace - id: check-yaml - id: check-toml - repo: https://github.com/pycqa/isort - rev: 5.10.1 + rev: 5.11.4 hooks: - id: isort additional_dependencies: [toml] - repo: https://github.com/psf/black - rev: 22.6.0 + rev: 22.12.0 hooks: - id: black @@ -25,12 +25,12 @@ repos: args: [-l, '79', -t, py38] - repo: https://github.com/pycqa/pydocstyle - rev: 6.1.1 + rev: 6.2.2 hooks: - id: pydocstyle - additional_dependencies: [toml] + additional_dependencies: [tomli] - - repo: https://gitlab.com/pycqa/flake8 - rev: 4.0.1 + - repo: https://github.com/pycqa/flake8 + rev: 6.0.0 hooks: - id: flake8 diff --git a/CHANGELOG.md b/CHANGELOG.md index 50c090d..6e07278 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,8 +1,15 @@ # Change log +## Unreleased + +New features: + +- New [HTTPX](https://www.python-httpx.org) support with `kafkit.registry.httpx.RegistryApi`, in addition to the existing aiohttp support. +- Documentation is now built with the new Rubin Observatory user guide theme and Sphinx configuration. + ## 0.2.1 (2022-07-15) -A `py.typed` file is now included to advertise typo annotations in Kafkit. +A `py.typed` file is now included to advertise type annotations in Kafkit. ## 0.2.0 (2022-07-15) diff --git a/Makefile b/Makefile index b121f93..484b05e 100644 --- a/Makefile +++ b/Makefile @@ -5,7 +5,7 @@ help: .PHONY: init init: - pip install -e ".[dev]" + pip install -e ".[aiohttp,httpx,dev]" pip install tox pre-commit pre-commit install rm -rf .tox diff --git a/docs/_rst_epilog.rst b/docs/_rst_epilog.rst new file mode 100644 index 0000000..828e6bf --- /dev/null +++ b/docs/_rst_epilog.rst @@ -0,0 +1,12 @@ + +.. _aiohttp: https://aiohttp.readthedocs.io/en/stable/ +.. _aiokafka: https://aiokafka.readthedocs.io/en/stable/ +.. _Confluent Schema Registry: https://docs.confluent.io/current/schema-registry/docs/index.html +.. _Confluent Wire Format: https://docs.confluent.io/current/schema-registry/docs/serializer-formatter.html#wire-format +.. _HTTPX: https://www.python-httpx.org +.. _mypy: http://www.mypy-lang.org +.. _pre-commit: https://pre-commit.com +.. _pytest: https://docs.pytest.org/en/latest/ +.. _Schema Evolution and Compatibility: https://docs.confluent.io/current/schema-registry/avro.html +.. _Strimzi: https://strimzi.io +.. _tox: https://tox.readthedocs.io/en/latest/ diff --git a/docs/api.rst b/docs/api.rst index 53f0841..085fb18 100644 --- a/docs/api.rst +++ b/docs/api.rst @@ -8,6 +8,9 @@ Kafkit API reference .. automodapi:: kafkit.registry.aiohttp :no-inheritance-diagram: +.. automodapi:: kafkit.registry.httpx + :no-inheritance-diagram: + .. automodapi:: kafkit.registry.manager :no-inheritance-diagram: diff --git a/docs/conf.py b/docs/conf.py index f11a587..891034d 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -1,100 +1,5 @@ -import os +from documenteer.conf.guide import * -import lsst_sphinx_bootstrap_theme - -import kafkit - -# Common links and substitutions ============================================= - -rst_epilog = """ - -.. _aiohttp: https://aiohttp.readthedocs.io/en/stable/ -.. _aiokafka: https://aiokafka.readthedocs.io/en/stable/ -.. _Confluent Schema Registry: https://docs.confluent.io/current/schema-registry/docs/index.html -.. _Confluent Wire Format: https://docs.confluent.io/current/schema-registry/docs/serializer-formatter.html#wire-format -.. _mypy: http://www.mypy-lang.org -.. _pre-commit: https://pre-commit.com -.. _pytest: https://docs.pytest.org/en/latest/ -.. _Schema Evolution and Compatibility: https://docs.confluent.io/current/schema-registry/avro.html -.. _Strimzi: https://strimzi.io -.. _tox: https://tox.readthedocs.io/en/latest/ -""" - -# Extensions ================================================================= - -extensions = [ - "myst_parser", - "sphinx.ext.autodoc", - "sphinx.ext.napoleon", - "sphinx.ext.doctest", - "sphinx.ext.intersphinx", - "sphinx.ext.todo", - "sphinx-prompt", - "sphinx_automodapi.automodapi", - "sphinx_automodapi.smart_resolver", - "documenteer.sphinxext", -] - -# General configuration ====================================================== - -source_suffix = { - ".rst": "restructuredtext", - ".txt": "markdown", - ".md": "markdown", -} - -# The root toctree document. -root_doc = "index" - -# General information about the project. -project = "Kafkit" -copyright = ( - "2019-2022 " - "Association of Universities for Research in Astronomy, Inc. (AURA)" -) -author = "LSST Data Management" - -version = kafkit.__version__ -release = version - -# List of patterns, relative to source directory, that match files and -# directories to ignore when looking for source files. -exclude_patterns = ["_build", "README.rst"] - -# The name of the Pygments (syntax highlighting) style to use. -pygments_style = "sphinx" - -# The reST default role cross-links Python (used for this markup: `text`) -default_role = "py:obj" - -nitpick_ignore = [ - # Ignore missing cross-references for modules that don't provide - # intersphinx. The documentation itself should use double-quotes instead - # of single-quotes to not generate a reference, but automatic references - # are generated from the type signatures and can't be avoided. - ("py:obj", "aiokafka.AIOKafkaProducer.send_and_wait"), -] - -# Intersphinx ================================================================ - -intersphinx_mapping = { - "python": ("https://docs.python.org/3/", None), - "aiohttp": ("https://aiohttp.readthedocs.io/en/stable/", None), - "aiokafka": ("https://aiokafka.readthedocs.io/en/stable/", None), - "fastavro": ("https://fastavro.readthedocs.io/en/latest/", None), -} - -intersphinx_timeout = 10.0 # seconds -intersphinx_cache_limit = 5 # days - -# Linkcheck builder ========================================================== - -linkcheck_retries = 2 - -linkcheck_ignore = [ - r"^https://jira.lsstcorp.org/browse/", - r"^http://registry:8081", -] linkcheck_anchors_ignore = [ r"^!", @@ -104,64 +9,6 @@ r"errors", ] -linkcheck_timeout = 15 - -# HTML builder =============================================================== - -templates_path = [ - "_templates", - lsst_sphinx_bootstrap_theme.get_html_templates_path(), -] - -html_theme = "lsst_sphinx_bootstrap_theme" -html_theme_path = [lsst_sphinx_bootstrap_theme.get_html_theme_path()] - -html_context = {} - -# Theme options are theme-specific and customize the look and feel of a theme -# further. For a list of options available for each theme, see the -# documentation. -html_theme_options = {"logotext": project} - -# The name for this set of Sphinx documents. If None, it defaults to -# " v documentation". -html_title = f"{project} v{version}" - -# A shorter title for the navigation bar. Default is the same as html_title. -html_short_title = project - -# Add any paths that contain custom static files (such as style sheets) here, -# relative to this directory. They are copied after the builtin static files, -# so a file named "default.css" will overwrite the builtin "default.css". -html_static_path = [] - -# If true, links to the reST sources are added to the pages. -html_show_sourcelink = False - -# Do not copy reST source for each page into the build -html_copy_source = False - -# If false, no module index is generated. -html_domain_indices = True - -# If false, no index is generated. -html_use_index = True - -# API Reference ============================================================== - -napoleon_google_docstring = False -napoleon_numpy_docstring = True -napoleon_include_init_with_doc = False -napoleon_include_private_with_doc = False -napoleon_include_special_with_doc = True -napoleon_use_admonition_for_examples = False -napoleon_use_admonition_for_notes = False -napoleon_use_admonition_for_references = False -napoleon_use_ivar = False -napoleon_use_keyword = True -napoleon_use_param = True -napoleon_use_rtype = True - napoleon_type_aliases = { # resolves confusion between sans-io version of impl specific version "RegistryApi": "kafkit.registry.sansio.RegistryApi", @@ -169,57 +16,3 @@ "ClientSession": "aiohttp.ClientSession", "optional": "typing.Optional", } - -autosummary_generate = True - -automodapi_toctreedirnm = "api" -automodsumm_inherited_members = True - -# Docstrings for classes and methods are inherited from parents. -autodoc_inherit_docstrings = True - -# Class documentation should only contain the class docstring and -# ignore the __init__ docstring, account to LSST coding standards. -autoclass_content = "class" - -# Default flags for automodapi directives. Special members are dunder -# methods. -autodoc_default_options = { - "show-inheritance": True, - "special-members": True, -} - -# Render inheritance diagrams in SVG -graphviz_output_format = "svg" - -graphviz_dot_args = [ - "-Nfontsize=10", - "-Nfontname=Helvetica Neue, Helvetica, Arial, sans-serif", - "-Efontsize=10", - "-Efontname=Helvetica Neue, Helvetica, Arial, sans-serif", - "-Gfontsize=10", - "-Gfontname=Helvetica Neue, Helvetica, Arial, sans-serif", -] - -# TODO extension ============================================================= - -todo_include_todos = False - -# My-ST (Markdown) =========================================================== -# https://myst-parser.readthedocs.io/en/latest/syntax/optional.html - -myst_enable_extensions = [ - "amsmath", - "dollarmath", - "colon_fence", - "deflist", - "fieldlist", - "html_admonition", - "html_image", - "linkify", - "replacements", - "smartquotes", - "strikethrough", - "substitution", - "tasklist", -] diff --git a/docs/documenteer.toml b/docs/documenteer.toml new file mode 100644 index 0000000..2ff4792 --- /dev/null +++ b/docs/documenteer.toml @@ -0,0 +1,28 @@ +[project] +title = "Kafkit" +copyright = "2019-2023 Association of Universities for Research in Astronomy, Inc. (AURA)" + +[project.python] +package = "kafkit" + +[sphinx] +rst_epilog_file = "_rst_epilog.rst" +nitpick_ignore = [ + [ + "py:obj", + "aiokafka.AIOKafkaProducer.send_and_wait", + ], + [ + "py:class", + "httpx.AsyncClient", + ], +] + +[sphinx.intersphinx.projects] +python = "https://docs.python.org/3/" +aiohttp = "https://aiohttp.readthedocs.io/en/stable/" +aiokafka = "https://aiokafka.readthedocs.io/en/stable/" +fastavro = "https://fastavro.readthedocs.io/en/latest/" + +[sphinx.linkcheck] +ignore = ['^https://jira.lsstcorp.org/browse/', '^http://registry:8081'] diff --git a/docs/guide/index.rst b/docs/guide/index.rst new file mode 100644 index 0000000..9f3cd68 --- /dev/null +++ b/docs/guide/index.rst @@ -0,0 +1,9 @@ +########## +User guide +########## + +.. toctree:: + :maxdepth: 2 + + recordnameschemamanager-howto + strimzi-ssl-howto diff --git a/docs/recordnameschemamanager-howto.rst b/docs/guide/recordnameschemamanager-howto.rst similarity index 100% rename from docs/recordnameschemamanager-howto.rst rename to docs/guide/recordnameschemamanager-howto.rst diff --git a/docs/strimzi-ssl-howto.rst b/docs/guide/strimzi-ssl-howto.rst similarity index 100% rename from docs/strimzi-ssl-howto.rst rename to docs/guide/strimzi-ssl-howto.rst diff --git a/docs/index.rst b/docs/index.rst index d5fdb44..953711a 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -5,9 +5,9 @@ Kafkit Kafkit helps you write Kafka producers and consumers in Python with asyncio: - Kafkit provides a client for the Confluent Schema Registry's HTTP API. - The `~kafkit.registry.aiohttp.RegistryApi` client includes both high-level methods for managing subjects and schemas in a Registry, and direct low-level access to HTTP methods (GET, POST, PUT, PATCH, and DELETE). + The `~kafkit.registry.httpx.RegistryApi` client includes both high-level methods for managing subjects and schemas in a Registry, and direct low-level access to HTTP methods (GET, POST, PUT, PATCH, and DELETE). The high-level methods use caching so you can use the client as an integral part of your application's schema management. - `~kafkit.registry.aiohttp.RegistryApi` is implemented around aiohttp_, but since the base class is designed with a `sans IO architecture `__, a Registry client can be implemented with any asyncio HTTP library. + The client is implemented for both aiohttp_ and httpx_, but since the base class is designed with a `sans IO architecture `__, a Registry client can be implemented with any asyncio HTTP library. - Kafkit provides Avro message serializers and deserializers that integrate with the `Confluent Schema Registry`_: `~kafkit.registry.Deserializer`, `~kafkit.registry.Serializer`, and `~kafkit.registry.PolySerializer`. @@ -18,42 +18,39 @@ Kafkit helps you write Kafka producers and consumers in Python with asyncio: Installation ============ -Install Kafkit with aiohttp: +Kafkit can be installed with different HTTP clients for convenience -.. code-block:: sh +.. tab-set:: - pip install kafkit[aiohttp] + .. tab-item:: httpx -User guide -========== + .. code-block:: sh -.. toctree:: - :maxdepth: 2 + pip install kafkit[httpx] - recordnameschemamanager-howto - strimzi-ssl-howto + .. tab-item:: aiohttp -API reference -============= + .. code-block:: sh -.. toctree:: + pip install kafkit[aiohttp] + + .. tab-item:: No client - api + .. code-block:: sh -Developer guide -=============== + pip install kafkit + +Kafkit is also available on Conda-Forge at https://github.com/conda-forge/kafkit-feedstock. .. toctree:: - :maxdepth: 2 + :hidden: - dev/index + guide/index + API + changelog + Developer guide Project information =================== Kafkit is developed on GitHub at https://github.com/lsst-sqre/kafkit. - -.. toctree:: - :maxdepth: 1 - - changelog diff --git a/pyproject.toml b/pyproject.toml index 834233a..c382f6e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -3,14 +3,11 @@ name = "kafkit" readme = "README.md" description = "Kafkit helps you write Kafka producers and consumers in Python with asyncio." -license = {text = "MIT"} +license = { text = "MIT" } authors = [ - {name = "Association of Universities for Research in Astronomy, Inc. (AURA)", email = "sqre-admin@lists.lsst.org"}, -] -keywords = [ - "rubin", - "lsst", + { name = "Association of Universities for Research in Astronomy, Inc. (AURA)", email = "sqre-admin@lists.lsst.org" }, ] +keywords = ["rubin", "lsst"] # https://pypi.org/classifiers/ classifiers = [ "Development Status :: 4 - Beta", @@ -26,14 +23,12 @@ classifiers = [ "Typing :: Typed", ] requires-python = ">=3.8" -dependencies = [ - "uritemplate", - "fastavro", -] +dependencies = ["uritemplate", "fastavro"] dynamic = ["version"] [project.optional-dependencies] aiohttp = ["aiohttp"] +httpx = ["httpx"] dev = [ # Testing "coverage[toml]", @@ -43,12 +38,7 @@ dev = [ "mypy", # Documentation "sphinx", - "documenteer", - "lsst-sphinx-bootstrap-theme", - "sphinx-prompt", - "sphinx-automodapi", - "myst-parser", - "markdown-it-py[linkify]", + "documenteer[guide]", ] [project.urls] @@ -56,11 +46,7 @@ Homepage = "https://kafkit.lsst.io" Source = "https://github.com/lsst-sqre/kafkit" [build-system] -requires = [ - "setuptools>=61", - "wheel", - "setuptools_scm[toml]>=6.2" -] +requires = ["setuptools>=61", "wheel", "setuptools_scm[toml]>=6.2"] build-backend = 'setuptools.build_meta' [tool.setuptools.packages.find] @@ -89,7 +75,7 @@ exclude_lines = [ "raise NotImplementedError", "if 0:", "if __name__ == .__main__.:", - "if TYPE_CHECKING:" + "if TYPE_CHECKING:", ] [tool.black] @@ -114,7 +100,7 @@ exclude = ''' # Reference: http://www.pydocstyle.org/en/stable/error_codes.html convention = "numpy" add_select = [ - "D212" # Multi-line docstring summary should start at the first line + "D212", # Multi-line docstring summary should start at the first line ] add-ignore = [ "D105", # Missing docstring in magic method @@ -139,10 +125,7 @@ skip = ["docs/conf.py"] [tool.pytest.ini_options] asyncio_mode = "strict" -python_files = [ - "tests/*.py", - "tests/*/*.py" -] +python_files = ["tests/*.py", "tests/*/*.py"] markers = [ "docker", # marks tests as requiring docker-compose (deselect with '-m "not docker"')" ] diff --git a/src/kafkit/registry/httpx.py b/src/kafkit/registry/httpx.py new file mode 100644 index 0000000..bc1d2c9 --- /dev/null +++ b/src/kafkit/registry/httpx.py @@ -0,0 +1,37 @@ +"""Httpx client for the Confluent Schema Registry. + +This code and architecture is based on https://github.com/brettcannon/gidgethub +See licenses/gidgethub.txt for info. +""" + +from __future__ import annotations + +from typing import Mapping, Tuple + +from httpx import AsyncClient + +from . import sansio + +__all__ = ["RegistryApi"] + + +class RegistryApi(sansio.RegistryApi): + """A Confluent Schema Registry client that uses httpx. + + Parameters + ---------- + http_client + The async httpx client. + """ + + def __init__(self, *, http_client: AsyncClient, url: str) -> None: + self._client = http_client + super().__init__(url=url) + + async def _request( + self, method: str, url: str, headers: Mapping[str, str], body: bytes + ) -> Tuple[int, Mapping[str, str], bytes]: + response = await self._client.request( + method, url, headers=headers, content=body + ) + return response.status_code, response.headers, response.read() diff --git a/src/kafkit/registry/manager.py b/src/kafkit/registry/manager.py index 83fc086..e358a9f 100644 --- a/src/kafkit/registry/manager.py +++ b/src/kafkit/registry/manager.py @@ -67,7 +67,7 @@ class RecordNameSchemaManager: ``TopicNameStrategy``, where subjects are named for the topic with ``-key`` or ``-value`` suffixes. - For more information, see :doc:`/recordnameschemamanager-howto` in + For more information, see :doc:`/guide/recordnameschemamanager-howto` in the user guide. """ diff --git a/src/kafkit/registry/sansio.py b/src/kafkit/registry/sansio.py index 337121b..6824f52 100644 --- a/src/kafkit/registry/sansio.py +++ b/src/kafkit/registry/sansio.py @@ -592,7 +592,7 @@ def __init__( self, url: str = "http://registry:8081", status_code: int = 200, - headers: Mapping[str, str] = None, + headers: Optional[Mapping[str, str]] = None, body: Any = b"", ) -> None: super().__init__(url=url) diff --git a/src/kafkit/registry/serializer.py b/src/kafkit/registry/serializer.py index ebc8efb..b63cb62 100644 --- a/src/kafkit/registry/serializer.py +++ b/src/kafkit/registry/serializer.py @@ -11,7 +11,7 @@ import fastavro if TYPE_CHECKING: - from kafit.registry.sansio import RegistryApi + from kafkit.registry.sansio import RegistryApi __all__ = [ "Serializer", diff --git a/tests/registry_httpx_test.py b/tests/registry_httpx_test.py new file mode 100644 index 0000000..03007b6 --- /dev/null +++ b/tests/registry_httpx_test.py @@ -0,0 +1,37 @@ +"""Tests for the kafkit.registry.httpx module.""" + +from __future__ import annotations + +import os + +import pytest +from httpx import AsyncClient + +from kafkit.registry.httpx import RegistryApi + +schema_a = { + "type": "record", + "name": "a", + "namespace": "kafkit.httpx", + "fields": [ + {"name": "field1", "type": "int"}, + {"name": "field2", "type": "string"}, + ], +} + + +@pytest.mark.docker +@pytest.mark.skipif( + os.getenv("SCHEMA_REGISTRY_URL") is None, + reason="SCHEMA_REGISTRY_URL env var must be configured", +) +@pytest.mark.asyncio +async def test_httpxapi() -> None: + """Test that the httpx RegistryApi can connect to the Schema Registry.""" + registry_url = os.getenv("SCHEMA_REGISTRY_URL") + assert registry_url + + async with AsyncClient() as http_client: + registry = RegistryApi(http_client=http_client, url=registry_url) + schema_a_id = await registry.register_schema(schema_a) + assert isinstance(schema_a_id, int) diff --git a/tox.ini b/tox.ini index cfc2d3a..9d55105 100644 --- a/tox.ini +++ b/tox.ini @@ -9,6 +9,7 @@ deps = extras = dev aiohttp + httpx allowlist_externals = docker-compose setenv =