From f182e2f5438097ed344ac16f4eecc77570420c0d Mon Sep 17 00:00:00 2001 From: roxanne-o Date: Tue, 24 Feb 2026 23:52:52 +0000 Subject: [PATCH] Fix RemoteDisconnected errors from stale pooled connections When the SDK reuses an HTTP connection that has been idle for ~60s, the server-side infrastructure (ALB / nginx) may have already closed it. urllib3 then raises RemoteDisconnected which was not retried or caught. - Add a default urllib3 Retry policy (3 retries with backoff) so stale connections are transparently re-established at the transport layer. - Catch MaxRetryError and ProtocolError in rest.py so exhausted retries surface as ApiException instead of raw urllib3 errors. --- generated/groundlight_openapi_client/rest.py | 6 ++++++ src/groundlight/client.py | 20 +++++++++++++++++--- 2 files changed, 23 insertions(+), 3 deletions(-) diff --git a/generated/groundlight_openapi_client/rest.py b/generated/groundlight_openapi_client/rest.py index 16d8ca86..cd8d4b2c 100644 --- a/generated/groundlight_openapi_client/rest.py +++ b/generated/groundlight_openapi_client/rest.py @@ -218,6 +218,12 @@ def request( except urllib3.exceptions.SSLError as e: msg = "{0}\n{1}".format(type(e).__name__, str(e)) raise ApiException(status=0, reason=msg) + except urllib3.exceptions.MaxRetryError as e: + msg = "{0}\n{1}".format(type(e).__name__, str(e)) + raise ApiException(status=0, reason=msg) + except urllib3.exceptions.ProtocolError as e: + msg = "{0}\n{1}".format(type(e).__name__, str(e)) + raise ApiException(status=0, reason=msg) if _preload_content: r = RESTResponse(r) diff --git a/src/groundlight/client.py b/src/groundlight/client.py index 669d3a7c..345f21e5 100644 --- a/src/groundlight/client.py +++ b/src/groundlight/client.py @@ -59,6 +59,19 @@ # It used to take >8 min to timeout to a bad IP address DEFAULT_REQUEST_TIMEOUT = 10 # seconds +# Default urllib3 transport-level retry policy. Handles stale/dead pooled +# connections (RemoteDisconnected) that occur when the server (ALB / nginx) +# closes an idle keep-alive connection before the client reuses it. +DEFAULT_HTTP_TRANSPORT_RETRY = Retry( + total=3, + connect=3, + read=3, + redirect=3, + backoff_factor=0.2, + allowed_methods=None, # retry all HTTP methods including POST + raise_on_status=False, # let the SDK's own status-code handling run +) + class GroundlightClientError(Exception): pass @@ -156,9 +169,10 @@ def __init__( # Specify the endpoint self.endpoint = sanitize_endpoint_url(endpoint) self.configuration = Configuration(host=self.endpoint) - if http_transport_retries is not None: - # Once we upgrade openapitools to ^7.7.0, retries can be passed into the constructor of Configuration above. - self.configuration.retries = http_transport_retries + # Once we upgrade openapitools to ^7.7.0, retries can be passed into the constructor of Configuration above. + self.configuration.retries = ( + http_transport_retries if http_transport_retries is not None else DEFAULT_HTTP_TRANSPORT_RETRY + ) if not api_token: try: