Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 3 additions & 0 deletions .github/workflows/ci.yml
Original file line number Diff line number Diff line change
Expand Up @@ -66,6 +66,9 @@ jobs:
- name: Run ruff format check
run: uv run ruff format --check src/ tests/

- name: Run ty type check
run: uv run ty check

test:
name: Unit Tests
runs-on: ubuntu-latest
Expand Down
6 changes: 2 additions & 4 deletions .github/workflows/integration.yml
Original file line number Diff line number Diff line change
Expand Up @@ -45,7 +45,5 @@ jobs:
- name: Run integration tests
env:
SA_KEY_B64: ${{ secrets.SA_KEY_B64 }}
PROJECT_ID: ${{ secrets.PROJECT_ID }}
TOKEN_URL: ${{ secrets.TOKEN_URL }}
REGION: ${{ secrets.REGION }}
run: uv run pytest tests/integration/ -v --timeout=700 --no-cov
STACKIT_PROJECT_ID: ${{ secrets.STACKIT_PROJECT_ID }}
run: uv run pytest tests/integration/ -v -s --timeout=700 --no-cov
3 changes: 3 additions & 0 deletions .github/workflows/release.yml
Original file line number Diff line number Diff line change
Expand Up @@ -46,6 +46,9 @@ jobs:
- name: Run ruff format check
run: uv run ruff format --check src/ tests/

- name: Run ty type check
run: uv run ty check

- name: Run pytest with coverage
run: uv run pytest --cov=src/pubsub_sdk_python --cov-branch --cov-report=term-missing --cov-fail-under=85

Expand Down
2 changes: 1 addition & 1 deletion CONTRIBUTING.md
Original file line number Diff line number Diff line change
Expand Up @@ -74,7 +74,7 @@ We maintain a high bar for reliability and test coverage:
```bash
make test
```
- Integration tests interact with STACKIT PubSub APIs and expect configuration variables (`SA_KEY_B64`, `PROJECT_ID`, `TOKEN_URL`, `REGION`). Run them with `uv run pytest tests/integration/ --no-cov`.
- Integration tests interact with STACKIT PubSub APIs and expect configuration variables (`SA_KEY_B64`, `STACKIT_PROJECT_ID`). Run them with `uv run pytest tests/integration/ --no-cov`.

---

Expand Down
11 changes: 7 additions & 4 deletions Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -5,13 +5,16 @@ install:
uv pip install -e .

lint:
uvx ruff@0.16.4 check
uv run ruff check src/ tests/

format:
uvx ruff@0.16.4 format
uv run ruff format src/ tests/

check: lint format
uvx ruff@0.16.4 format --check
typecheck:
uv run ty check

check: lint format typecheck
uv run ruff format --check src/ tests/

test: check install
uv run pytest
Expand Down
1 change: 1 addition & 0 deletions pyproject.toml
Original file line number Diff line number Diff line change
Expand Up @@ -35,6 +35,7 @@ dev = [
"pytest-cov>=5.0.0",
"pytest-timeout>=2.3.0",
"ruff>=0.8.0",
"ty>=0.0.93",
]

[tool.pytest.ini_options]
Expand Down
5 changes: 3 additions & 2 deletions src/pubsub_sdk_python/auth.py
Original file line number Diff line number Diff line change
Expand Up @@ -102,8 +102,9 @@ def _patched_fetch(self: KeyAuth) -> None:
except requests.RequestException as e:
raise requests.RequestException("Initial token fetch failed") from e

KeyAuth._KeyAuth__fetch_token_from_endpoint = _patched_fetch
KeyAuth._KeyAuth__patched = True
key_auth_any: Any = KeyAuth
key_auth_any._KeyAuth__fetch_token_from_endpoint = _patched_fetch
key_auth_any._KeyAuth__patched = True

import jwt

Expand Down
8 changes: 6 additions & 2 deletions src/pubsub_sdk_python/publisher.py
Original file line number Diff line number Diff line change
Expand Up @@ -185,8 +185,12 @@ async def _request(

# If we've exhausted retries
if attempt >= self.retry_config.max_retries:
msg = (
f"All {self.retry_config.max_retries} retry attempts "
f"exhausted for {path}: {exc}"
)
raise PubSubRetryError(
f"All {self.retry_config.max_retries} retry attempts exhausted for {path}",
msg,
causes=[last_exception] if last_exception else None,
) from exc

Expand All @@ -202,7 +206,7 @@ async def _request(
)
await self._safe_sleep(delay)

raise last_exception # type: ignore[misc]
raise PubSubRetryError("Retry loop terminated unexpectedly")

@staticmethod
async def _safe_sleep(delay: float) -> None:
Expand Down
170 changes: 104 additions & 66 deletions src/pubsub_sdk_python/retry.py
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,7 @@
import time
from collections.abc import Awaitable, Callable
from functools import wraps
from typing import Any, TypeVar
from typing import Any, TypeVar, overload

import httpx

Expand Down Expand Up @@ -136,11 +136,24 @@ def _should_retry(
return False


def with_retry(func: Callable[..., T]) -> Callable[..., T]:
@overload
def with_retry(func: Callable[..., T]) -> Callable[..., T]: ...


@overload
def with_retry(
func: None = None, *, config: RetryConfig | None = None
) -> Callable[[Callable[..., T]], Callable[..., T]]: ...


def with_retry(
func: Callable[..., T] | None = None, *, config: RetryConfig | None = None
) -> Callable[..., T] | Callable[[Callable[..., T]], Callable[..., T]]:
"""Decorator that adds retry logic with exponential backoff to a synchronous function.

Args:
func: The synchronous function to wrap.
config: Optional RetryConfig to use.

Returns:
The wrapped function with retry logic.
Expand All @@ -150,46 +163,66 @@ def with_retry(func: Callable[..., T]) -> Callable[..., T]:
def my_api_call():
... # Will retry on transient errors
"""
config = RetryConfig()

@wraps(func)
def wrapper(*args: Any, **kwargs: Any) -> T:
last_exception: Exception | None = None
for attempt in range(config.max_retries + 1):
try:
return func(*args, **kwargs)
except Exception as exc:
last_exception = exc
if not _should_retry(exc, config=config):
raise
if attempt >= config.max_retries:
raise PubSubRetryError(
f"All {config.max_retries} retry attempts exhausted",
causes=[last_exception] if last_exception else None,
) from exc
delay = _calculate_delay(config, attempt)
logger.warning(
"Retry attempt %d/%d for %s after %.2fs: %s",
attempt + 1,
config.max_retries,
func.__name__,
delay,
exc,
)
time.sleep(delay)
# Unreachable but mypy needs it
raise last_exception # type: ignore[misc]

return wrapper


def with_async_retry(func: Callable[..., Awaitable[T]]) -> Callable[..., Awaitable[T]]:
retry_config = config or RetryConfig()

def decorator(fn: Callable[..., T]) -> Callable[..., T]:
@wraps(fn)
def wrapper(*args: Any, **kwargs: Any) -> T:
last_exception: Exception | None = None
for attempt in range(retry_config.max_retries + 1):
try:
return fn(*args, **kwargs)
except Exception as exc:
last_exception = exc
if not _should_retry(exc, config=retry_config):
raise
if attempt >= retry_config.max_retries:
raise PubSubRetryError(
f"All {retry_config.max_retries} retry attempts exhausted: {exc}",
causes=[last_exception] if last_exception else None,
) from exc
delay = _calculate_delay(retry_config, attempt)
logger.warning(
"Retry attempt %d/%d for %s after %.2fs: %s",
attempt + 1,
retry_config.max_retries,
getattr(fn, "__name__", repr(fn)),
delay,
exc,
)
time.sleep(delay)
raise PubSubRetryError("Retry loop terminated unexpectedly")

return wrapper

if func is not None:
return decorator(func)
return decorator


@overload
def with_async_retry(func: Callable[..., Awaitable[T]]) -> Callable[..., Awaitable[T]]: ...


@overload
def with_async_retry(
func: None = None, *, config: RetryConfig | None = None
) -> Callable[[Callable[..., Awaitable[T]]], Callable[..., Awaitable[T]]]: ...


def with_async_retry(
func: Callable[..., Awaitable[T]] | None = None, *, config: RetryConfig | None = None
) -> (
Callable[..., Awaitable[T]]
| Callable[[Callable[..., Awaitable[T]]], Callable[..., Awaitable[T]]]
):
"""Async-friendly retry decorator.

Adds retry logic with exponential backoff to an async function.

Args:
func: The async function to wrap.
config: Optional RetryConfig to use.

Returns:
The wrapped async function with retry logic.
Expand All @@ -199,36 +232,41 @@ def with_async_retry(func: Callable[..., Awaitable[T]]) -> Callable[..., Awaitab
async def my_async_api_call():
... # Will retry on transient errors
"""
config = RetryConfig()

@wraps(func)
async def wrapper(*args: Any, **kwargs: Any) -> T:
last_exception: Exception | None = None
for attempt in range(config.max_retries + 1):
try:
return await func(*args, **kwargs)
except Exception as exc:
last_exception = exc
if not _should_retry(exc, config=config):
raise
if attempt >= config.max_retries:
raise PubSubRetryError(
f"All {config.max_retries} retry attempts exhausted",
causes=[last_exception] if last_exception else None,
) from exc
delay = _calculate_delay(config, attempt)
logger.warning(
"Retry attempt %d/%d for %s after %.2fs: %s",
attempt + 1,
config.max_retries,
func.__name__,
delay,
exc,
)
await asyncio.sleep(delay)
raise last_exception # type: ignore[misc]

return wrapper
retry_config = config or RetryConfig()

def decorator(fn: Callable[..., Awaitable[T]]) -> Callable[..., Awaitable[T]]:
@wraps(fn)
async def wrapper(*args: Any, **kwargs: Any) -> T:
last_exception: Exception | None = None
for attempt in range(retry_config.max_retries + 1):
try:
return await fn(*args, **kwargs)
except Exception as exc:
last_exception = exc
if not _should_retry(exc, config=retry_config):
raise
if attempt >= retry_config.max_retries:
raise PubSubRetryError(
f"All {retry_config.max_retries} retry attempts exhausted: {exc}",
causes=[last_exception] if last_exception else None,
) from exc
delay = _calculate_delay(retry_config, attempt)
logger.warning(
"Retry attempt %d/%d for %s after %.2fs: %s",
attempt + 1,
retry_config.max_retries,
getattr(fn, "__name__", repr(fn)),
delay,
exc,
)
await asyncio.sleep(delay)
raise PubSubRetryError("Retry loop terminated unexpectedly")

return wrapper

if func is not None:
return decorator(func)
return decorator


def get_retry_delay(config: RetryConfig, attempt: int) -> float:
Expand Down
8 changes: 6 additions & 2 deletions src/pubsub_sdk_python/subscriber.py
Original file line number Diff line number Diff line change
Expand Up @@ -207,8 +207,12 @@ async def _request(
raise

if attempt >= self.retry_config.max_retries:
msg = (
f"All {self.retry_config.max_retries} retry attempts "
f"exhausted for {path}: {exc}"
)
raise PubSubRetryError(
f"All {self.retry_config.max_retries} retry attempts exhausted for {path}",
msg,
causes=[last_exception] if last_exception else None,
) from exc

Expand All @@ -224,7 +228,7 @@ async def _request(
)
await self._safe_sleep(delay)

raise last_exception # type: ignore[misc]
raise PubSubRetryError("Retry loop terminated unexpectedly")

@staticmethod
async def _safe_sleep(delay: float) -> None:
Expand Down
Loading
Loading