Skip to content

Commit fbfe06a

Browse files
Cap the aggregate of per-node errors in ClusterClient.find_leader
The per-node snippet cap (_MAX_ERROR_MESSAGE_SNIPPET = 200) already bounds the M axis of error-message size on a hostile peer. The N axis (configured node-store size) is operator- controlled and unbounded — a 500-node store of failing peers produces ~100 KB of error text held in the ClusterError args, in every traceback render, and in every __cause__ walk on a long-lived process's pool retry loops. Cap the joined aggregate at _MAX_AGGREGATE_ERROR_PAYLOAD = 16 KiB (= ~80 nodes' worth of detail at the per-node cap; enough for diagnostic utility on any realistic cluster). Truncation marker mirrors the per-node ``_truncate_error`` phrasing for symmetry. Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
1 parent cc1c21b commit fbfe06a

2 files changed

Lines changed: 75 additions & 1 deletion

File tree

src/dqliteclient/cluster.py

Lines changed: 18 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -51,6 +51,17 @@
5151
# held in memory and serialised into every traceback.
5252
_MAX_ERROR_MESSAGE_SNIPPET: Final[int] = 200
5353

54+
# Cap the aggregate-of-all-per-node-errors payload before raising the
55+
# final ``ClusterError``. The per-node cap above bounds the M axis, but
56+
# the N axis (configured node-store size) is still operator-controlled
57+
# and unbounded — a 500-node store all returning hostile-cap-sized
58+
# messages produces ~100 KB of error text held in the ClusterError's
59+
# args, in every traceback render, and in every ``__cause__`` walk.
60+
# 16 KiB / 200 codepoints/snippet ≈ 80 nodes' worth of detail before
61+
# truncation, which is enough for diagnostic utility on any realistic
62+
# cluster while keeping the exception payload bounded.
63+
_MAX_AGGREGATE_ERROR_PAYLOAD: Final[int] = 16 * 1024
64+
5465
# Use OS-entropy randomness for the per-sweep node shuffle so that the
5566
# stampede-avoidance is not defeated by a downstream call to
5667
# ``random.seed(...)``. Test suites and some libraries seed the global
@@ -389,7 +400,13 @@ async def _find_leader_impl(self, *, trust_server_heartbeat: bool) -> str:
389400
last_exc = e
390401
continue
391402

392-
raise ClusterError(f"Could not find leader. Errors: {'; '.join(errors)}") from last_exc
403+
joined = "; ".join(errors)
404+
if len(joined) > _MAX_AGGREGATE_ERROR_PAYLOAD:
405+
kept = len(joined) - _MAX_AGGREGATE_ERROR_PAYLOAD
406+
joined = (
407+
joined[:_MAX_AGGREGATE_ERROR_PAYLOAD] + f"... [aggregate truncated, {kept} chars]"
408+
)
409+
raise ClusterError(f"Could not find leader. Errors: {joined}") from last_exc
393410

394411
async def _query_leader(
395412
self, address: str, *, trust_server_heartbeat: bool = False
Lines changed: 57 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,57 @@
1+
"""Pin: ``ClusterClient.find_leader``'s aggregate-of-per-node-errors
2+
payload is capped at ``_MAX_AGGREGATE_ERROR_PAYLOAD`` so the final
3+
``ClusterError`` does not grow O(N) in operator-configured node count.
4+
5+
The per-node snippet cap (``_MAX_ERROR_MESSAGE_SNIPPET = 200``)
6+
already bounds M (per-node failure message size). Without the
7+
aggregate cap, a 500-node store of failing peers produced ≥100 KB
8+
held in the ClusterError args, in every traceback render, and in
9+
every ``__cause__`` walk on a long-lived process's pool retry
10+
loops.
11+
"""
12+
13+
from __future__ import annotations
14+
15+
import pytest
16+
17+
from dqliteclient.cluster import (
18+
_MAX_AGGREGATE_ERROR_PAYLOAD,
19+
ClusterClient,
20+
)
21+
from dqliteclient.exceptions import ClusterError, DqliteConnectionError
22+
from dqliteclient.node_store import MemoryNodeStore
23+
24+
25+
@pytest.mark.asyncio
26+
async def test_find_leader_aggregate_error_payload_is_capped(
27+
monkeypatch: pytest.MonkeyPatch,
28+
) -> None:
29+
addrs = [f"10.0.{i // 256}.{i % 256}:9001" for i in range(500)]
30+
store = MemoryNodeStore(addrs)
31+
client = ClusterClient(store, timeout=0.01)
32+
33+
huge = "x" * 50_000 # per-node hostile message size
34+
35+
async def _fake_query(self: ClusterClient, address: str, **kwargs: object) -> str:
36+
raise DqliteConnectionError(huge)
37+
38+
monkeypatch.setattr(ClusterClient, "_query_leader", _fake_query)
39+
40+
with pytest.raises(ClusterError) as exc_info:
41+
await client.find_leader()
42+
43+
aggregate = str(exc_info.value)
44+
# The error string is "Could not find leader. Errors: <joined>"
45+
# plus a small truncation marker. Bound the test loosely on the
46+
# aggregate cap plus a small fixed overhead (prefix +
47+
# truncation marker).
48+
assert len(aggregate) <= _MAX_AGGREGATE_ERROR_PAYLOAD + 256, (
49+
f"Aggregate error payload {len(aggregate)} exceeds "
50+
f"_MAX_AGGREGATE_ERROR_PAYLOAD {_MAX_AGGREGATE_ERROR_PAYLOAD} "
51+
f"+ overhead. The N axis (configured node-store size) is "
52+
f"operator-controlled and unbounded — without the aggregate "
53+
f"cap, a 500-node store of failing peers produced >100 KB."
54+
)
55+
# Sanity: the truncation marker should be present (the test
56+
# configuration is designed to exceed the cap).
57+
assert "[aggregate truncated" in aggregate

0 commit comments

Comments
 (0)