From 7fffb5a721e353718dcd672e76c95bd01568a065 Mon Sep 17 00:00:00 2001 From: Torsten Kilias Date: Fri, 26 Jun 2026 08:20:59 +0200 Subject: [PATCH 01/22] Support configurable database node counts --- doc/changes/unreleased.md | 2 + exasol/saas/client/api_access.py | 42 ++++++++++++++---- test/integration/test_databases.py | 16 +++++++ test/unit/test_api_access.py | 70 +++++++++++++++++++++++++++++- 4 files changed, 121 insertions(+), 9 deletions(-) diff --git a/doc/changes/unreleased.md b/doc/changes/unreleased.md index 332dfd9..2c86bd8 100644 --- a/doc/changes/unreleased.md +++ b/doc/changes/unreleased.md @@ -15,6 +15,8 @@ Besides there are many formal and syntactic changes ## Summary +* #147: Added `num_nodes` support to the handwritten database creation helpers. + ## Refactorings * #160: Updated PTB to version 7.0.0 diff --git a/exasol/saas/client/api_access.py b/exasol/saas/client/api_access.py index 2c0d8a4..9baec02 100644 --- a/exasol/saas/client/api_access.py +++ b/exasol/saas/client/api_access.py @@ -40,6 +40,7 @@ create_database, delete_database, get_database, + get_database_settings, list_databases, ) from exasol.saas.client.openapi.api.security import ( @@ -261,6 +262,7 @@ def create_database( cluster_size: str = "XS", region: str = "eu-central-1", idle_time: timedelta | None = None, + num_nodes: int | None = None, ) -> ExasolDatabase | None: def minutes(x: timedelta) -> int: return x.seconds // 60 @@ -275,16 +277,20 @@ def minutes(x: timedelta) -> int: ), ) LOG.info("Creating database %s", name) + database_spec = openapi.models.CreateDatabase( + name=name, + initial_cluster=cluster_spec, + provider="aws", + region=region, + stream_type="innovation-release", + ) + if num_nodes is not None: + database_spec.num_nodes = num_nodes + resp = create_database.sync( self._account_id, client=self._client, - body=openapi.models.CreateDatabase( - name=name, - initial_cluster=cluster_spec, - provider="aws", - region=region, - stream_type="innovation-release", - ), + body=database_spec, ) database = ensure_type( ExasolDatabase, resp, f"Failed to create database {name}" @@ -378,10 +384,15 @@ def database( keep: bool = False, ignore_delete_failure: bool = False, idle_time: timedelta | None = None, + num_nodes: int | None = None, ): db = None try: - db = self.create_database(name, idle_time=idle_time) + db = self.create_database( + name, + idle_time=idle_time, + num_nodes=num_nodes, + ) yield db finally: db_repr = f"{db.name} with ID {db.id}" if db else None @@ -406,6 +417,21 @@ def get_database( ExasolDatabase, resp, f"Failed to get database {database_id}" ) + def get_database_settings( + self, + database_id: str, + ) -> openapi.models.DatabaseSettings | None: + resp = get_database_settings.sync( + self._account_id, + database_id, + client=self._client, + ) + return ensure_type( + openapi.models.DatabaseSettings, + resp, + f"Failed to get settings of database {database_id}", + ) + def wait_until_running( self, database_id: str, diff --git a/test/integration/test_databases.py b/test/integration/test_databases.py index f338799..ba24432 100644 --- a/test/integration/test_databases.py +++ b/test/integration/test_databases.py @@ -69,3 +69,19 @@ def get_connection(db: ExasolDatabase): api_access.delete_database(db.id) api_access.wait_until_deleted(db.id) assert db.id not in api_access.list_database_ids() + + +def test_create_database_with_two_nodes(api_access, local_name): + """ + This integration test verifies that the handwritten helper forwards + `num_nodes` and the created database reports that setting back. + """ + with api_access.database( + local_name, + ignore_delete_failure=True, + num_nodes=2, + ) as db: + settings = api_access.get_database_settings(db.id) + + assert settings is not None + assert settings.num_nodes == 2 diff --git a/test/unit/test_api_access.py b/test/unit/test_api_access.py index 0aa4164..4d8c3a6 100644 --- a/test/unit/test_api_access.py +++ b/test/unit/test_api_access.py @@ -1,4 +1,7 @@ -from datetime import timedelta +from datetime import ( + datetime, + timedelta, +) from test.util import not_raises from unittest.mock import Mock @@ -9,7 +12,13 @@ OpenApiAccess, timestamp_name, ) +from exasol.saas.client.openapi.models.exasol_database import ExasolDatabase +from exasol.saas.client.openapi.models.exasol_database_clusters import ( + ExasolDatabaseClusters, +) from exasol.saas.client.openapi.models.api_error import ApiError +from exasol.saas.client.openapi.models.status import Status +from exasol.saas.client.openapi.types import UNSET def response(status_code: int, message: str, spec=None): @@ -39,6 +48,27 @@ def delete_mock(monkeypatch, side_effect) -> Mock: return mock +def create_database_mock(monkeypatch, side_effect) -> Mock: + from exasol.saas.client.api_access import create_database as api + + mock = Mock(side_effect=side_effect) + monkeypatch.setattr(api, "sync", mock) + return mock + + +def database_response(name: str = "db") -> ExasolDatabase: + return ExasolDatabase( + status=Status.CREATING, + id="db-id", + name=name, + clusters=ExasolDatabaseClusters(total=1, running=0), + provider="aws", + region="eu-central-1", + created_at=datetime(2026, 1, 1), + created_by="tester", + ) + + @pytest.fixture def retry_timings() -> dict[str, timedelta]: """ @@ -123,3 +153,41 @@ def test_timestamp_name() -> None: assert len(set(suffixes)) == 3 # the provided tag should follow the hacky timestamp. assert all(tag == "TEST" for tag in tags) + + +@pytest.mark.parametrize( + "num_nodes, expected_num_nodes", + [ + pytest.param(None, UNSET, id="uses_backend_default"), + pytest.param(2, 2, id="forwards_explicit_value"), + ], +) +def test_create_database_num_nodes( + api_mock, monkeypatch, num_nodes, expected_num_nodes +) -> None: + create = create_database_mock( + monkeypatch, + [database_response("db-with-nodes")], + ) + + result = api_mock.create_database("db-with-nodes", num_nodes=num_nodes) + + assert result is not None + assert create.called + body = create.call_args.kwargs["body"] + assert body.num_nodes == expected_num_nodes + + +def test_database_context_forwards_num_nodes(api_mock, monkeypatch) -> None: + create = Mock(return_value=database_response("db-with-context")) + delete = Mock() + monkeypatch.setattr(api_mock, "create_database", create) + monkeypatch.setattr(api_mock, "delete_database", delete) + + with api_mock.database("db-with-context", num_nodes=2) as db: + assert db is not None + assert db.name == "db-with-context" + + assert create.call_args.args == ("db-with-context",) + assert create.call_args.kwargs == {"idle_time": None, "num_nodes": 2} + delete.assert_called_once_with("db-id", False) From a8c2d739c79c0ea953147f9f3d1bbba243d6ecdf Mon Sep 17 00:00:00 2001 From: Torsten Kilias Date: Fri, 26 Jun 2026 09:02:55 +0200 Subject: [PATCH 02/22] Format API access tests --- test/unit/test_api_access.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/unit/test_api_access.py b/test/unit/test_api_access.py index 4d8c3a6..acafa03 100644 --- a/test/unit/test_api_access.py +++ b/test/unit/test_api_access.py @@ -12,11 +12,11 @@ OpenApiAccess, timestamp_name, ) +from exasol.saas.client.openapi.models.api_error import ApiError from exasol.saas.client.openapi.models.exasol_database import ExasolDatabase from exasol.saas.client.openapi.models.exasol_database_clusters import ( ExasolDatabaseClusters, ) -from exasol.saas.client.openapi.models.api_error import ApiError from exasol.saas.client.openapi.models.status import Status from exasol.saas.client.openapi.types import UNSET From 496c575240f7e95dffab94d521f39c3c094fb113 Mon Sep 17 00:00:00 2001 From: Torsten Kilias Date: Fri, 26 Jun 2026 22:34:47 +0200 Subject: [PATCH 03/22] Stabilize slow integration checks --- exasol/saas/client/api_access.py | 62 +++++++++++++--- test/integration/test_allowed_ip.py | 2 + test/unit/test_api_access.py | 110 ++++++++++++++++++++++++++++ 3 files changed, 163 insertions(+), 11 deletions(-) diff --git a/exasol/saas/client/api_access.py b/exasol/saas/client/api_access.py index 9baec02..78459fe 100644 --- a/exasol/saas/client/api_access.py +++ b/exasol/saas/client/api_access.py @@ -308,8 +308,8 @@ def _ignore_failures(self, ignore: bool = False): def wait_until_deleted( self, database_id: str, - timeout: timedelta = timedelta(seconds=1), - interval: timedelta = timedelta(minutes=1), + timeout: timedelta = timedelta(minutes=5), + interval: timedelta = timedelta(seconds=10), ): @interval_retry(interval, timeout) def verify_not_listed() -> bool: @@ -421,16 +421,28 @@ def get_database_settings( self, database_id: str, ) -> openapi.models.DatabaseSettings | None: - resp = get_database_settings.sync( - self._account_id, - database_id, - client=self._client, - ) - return ensure_type( - openapi.models.DatabaseSettings, - resp, - f"Failed to get settings of database {database_id}", + def is_retry(resp: ApiError) -> bool: + return resp.status == 404 and "User/Database not found" in resp.message + + @interval_retry( + interval=timedelta(seconds=5), + timeout=timedelta(minutes=2), ) + def retrieve_settings() -> openapi.models.DatabaseSettings: + resp = get_database_settings.sync( + self._account_id, + database_id, + client=self._client, + ) + if isinstance(resp, ApiError) and is_retry(resp): + raise TryAgain + return ensure_type( + openapi.models.DatabaseSettings, + resp, + f"Failed to get settings of database {database_id}", + ) + + return retrieve_settings() def wait_until_running( self, @@ -494,6 +506,34 @@ def list_allowed_ip_ids(self) -> Iterable[str]: ips = ensure_type(list, resp, "Failed to retrieve the list of allowed ips") return (x.id for x in ips) + def wait_until_allowed_ip_listed( + self, + allowed_ip_id: str, + timeout: timedelta = timedelta(minutes=1), + interval: timedelta = timedelta(seconds=5), + ) -> None: + @interval_retry(interval, timeout) + def verify_listed() -> bool: + if allowed_ip_id not in self.list_allowed_ip_ids(): + raise TryAgain + return True + + verify_listed() + + def wait_until_allowed_ip_deleted( + self, + allowed_ip_id: str, + timeout: timedelta = timedelta(minutes=1), + interval: timedelta = timedelta(seconds=5), + ) -> None: + @interval_retry(interval, timeout) + def verify_deleted() -> bool: + if allowed_ip_id in self.list_allowed_ip_ids(): + raise TryAgain + return True + + verify_deleted() + def add_allowed_ip( self, cidr_ip: str = "0.0.0.0/0", diff --git a/test/integration/test_allowed_ip.py b/test/integration/test_allowed_ip.py index 1aa5472..5ab5e9e 100644 --- a/test/integration/test_allowed_ip.py +++ b/test/integration/test_allowed_ip.py @@ -1,9 +1,11 @@ def test_lifecycle(api_access): testee = api_access with testee.allowed_ip(ignore_delete_failure=True) as ip: + testee.wait_until_allowed_ip_listed(ip.id) # verify allowed ip is listed assert ip.id in testee.list_allowed_ip_ids() # delete allowed ip and verify it is not listed anymore testee.delete_allowed_ip(ip.id) + testee.wait_until_allowed_ip_deleted(ip.id) assert ip.id not in testee.list_allowed_ip_ids() diff --git a/test/unit/test_api_access.py b/test/unit/test_api_access.py index acafa03..d574c18 100644 --- a/test/unit/test_api_access.py +++ b/test/unit/test_api_access.py @@ -6,13 +6,16 @@ from unittest.mock import Mock import pytest +from tenacity import TryAgain from exasol.saas.client.api_access import ( DatabaseDeleteError, OpenApiAccess, + OpenApiError, timestamp_name, ) from exasol.saas.client.openapi.models.api_error import ApiError +from exasol.saas.client.openapi.models.database_settings import DatabaseSettings from exasol.saas.client.openapi.models.exasol_database import ExasolDatabase from exasol.saas.client.openapi.models.exasol_database_clusters import ( ExasolDatabaseClusters, @@ -56,6 +59,22 @@ def create_database_mock(monkeypatch, side_effect) -> Mock: return mock +def get_database_settings_mock(monkeypatch, side_effect) -> Mock: + from exasol.saas.client.api_access import get_database_settings as api + + mock = Mock(side_effect=side_effect) + monkeypatch.setattr(api, "sync", mock) + return mock + + +def list_allowed_ips_mock(monkeypatch, side_effect) -> Mock: + from exasol.saas.client.api_access import list_allowed_i_ps as api + + mock = Mock(side_effect=side_effect) + monkeypatch.setattr(api, "sync", mock) + return mock + + def database_response(name: str = "db") -> ExasolDatabase: return ExasolDatabase( status=Status.CREATING, @@ -69,6 +88,17 @@ def database_response(name: str = "db") -> ExasolDatabase: ) +def database_settings_response(num_nodes: int = 2) -> DatabaseSettings: + return DatabaseSettings( + offload_enabled=False, + auto_updates_enabled=True, + auto_updates_hard_disabled=False, + num_nodes=num_nodes, + stream_type="innovation-release", + stream_description="Innovation", + ) + + @pytest.fixture def retry_timings() -> dict[str, timedelta]: """ @@ -191,3 +221,83 @@ def test_database_context_forwards_num_nodes(api_mock, monkeypatch) -> None: assert create.call_args.args == ("db-with-context",) assert create.call_args.kwargs == {"idle_time": None, "num_nodes": 2} delete.assert_called_once_with("db-id", False) + + +def test_get_database_settings_retries_transient_not_found( + api_mock, monkeypatch +) -> None: + def immediate_retry(*_args, **_kwargs): + def decorate(func): + def wrapped(): + for _ in range(5): + try: + return func() + except TryAgain: + pass + except Exception: + raise + return func() + + return wrapped + + return decorate + + monkeypatch.setattr( + "exasol.saas.client.api_access.interval_retry", + immediate_retry, + ) + get_settings = get_database_settings_mock( + monkeypatch, + [ + api_error(404, "User/Database not found"), + database_settings_response(), + ], + ) + + result = api_mock.get_database_settings("db-id") + + assert result is not None + assert result.num_nodes == 2 + assert get_settings.call_count == 2 + + +def test_get_database_settings_raises_non_retryable_error( + api_mock, monkeypatch +) -> None: + monkeypatch.setattr( + "exasol.saas.client.api_access.interval_retry", + lambda *_args, **_kwargs: (lambda func: func), + ) + get_database_settings_mock( + monkeypatch, + [api_error(500, "boom")], + ) + + with pytest.raises(OpenApiError, match="Failed to get settings of database db-id"): + api_mock.get_database_settings("db-id") + + +def test_wait_until_allowed_ip_listed_retries(api_mock, monkeypatch) -> None: + list_allowed_ips_mock( + monkeypatch, + [[], [Mock(id="ip-1")]], + ) + + api_mock.wait_until_allowed_ip_listed( + "ip-1", + timeout=timedelta(seconds=1), + interval=timedelta(milliseconds=10), + ) + + +def test_wait_until_allowed_ip_deleted_retries(api_mock, monkeypatch) -> None: + list_allowed_ips_mock( + monkeypatch, + [[Mock(id="ip-1")], []], + ) + + api_mock.wait_until_allowed_ip_deleted( + "ip-1", + timeout=timedelta(seconds=1), + interval=timedelta(milliseconds=10), + ) From 11ab511e2697fb4ed56af2e5e0e5dd672cc647bf Mon Sep 17 00:00:00 2001 From: Torsten Kilias Date: Sat, 27 Jun 2026 00:54:35 +0200 Subject: [PATCH 04/22] Harden generated API error parsing --- exasol/saas/client/api_access.py | 4 +- .../saas/client/openapi/models/api_error.py | 16 +- noxfile.py | 2 + openapi_templates/model.py.jinja | 265 ++++++++++++++++++ test/unit/test_api_access.py | 58 ++++ 5 files changed, 336 insertions(+), 9 deletions(-) create mode 100644 openapi_templates/model.py.jinja diff --git a/exasol/saas/client/api_access.py b/exasol/saas/client/api_access.py index 78459fe..6eb93d8 100644 --- a/exasol/saas/client/api_access.py +++ b/exasol/saas/client/api_access.py @@ -375,7 +375,9 @@ def list_database_ids(self) -> Iterable[str]: resp = list_databases.sync(self._account_id, client=self._client) or [] # actually list[ExasolDatabase] dbs = ensure_type(list, resp, "Failed to list databases") - return (db.id for db in dbs) + return ( + db.id for db in dbs if db.deleted_at is UNSET and db.deleted_by is UNSET + ) @contextmanager def database( diff --git a/exasol/saas/client/openapi/models/api_error.py b/exasol/saas/client/openapi/models/api_error.py index 9159e04..5a595c6 100644 --- a/exasol/saas/client/openapi/models/api_error.py +++ b/exasol/saas/client/openapi/models/api_error.py @@ -82,21 +82,21 @@ def to_dict(self) -> dict[str, Any]: @classmethod def from_dict(cls: type[T], src_dict: Mapping[str, Any]) -> T: d = dict(src_dict) - status = d.pop("status") + status = d.pop("status", 0) - message = d.pop("message") + message = d.pop("message", "Unknown API error") - request_id = d.pop("requestId") + request_id = d.pop("requestId", "") - path = d.pop("path") + path = d.pop("path", "") - method = d.pop("method") + method = d.pop("method", "") - log_id = d.pop("logId") + log_id = d.pop("logId", "") - handler = d.pop("handler") + handler = d.pop("handler", "") - timestamp = d.pop("timestamp") + timestamp = d.pop("timestamp", "") causes = d.pop("causes", UNSET) diff --git a/noxfile.py b/noxfile.py index 1cd1f82..17822bb 100644 --- a/noxfile.py +++ b/noxfile.py @@ -100,6 +100,8 @@ def generate_api(session: Session): "--overwrite", "--config", "openapi_config.yml", + "--custom-template-path", + "openapi_templates", "--output-path", "tmp", silent=local_build, diff --git a/openapi_templates/model.py.jinja b/openapi_templates/model.py.jinja new file mode 100644 index 0000000..79d5b79 --- /dev/null +++ b/openapi_templates/model.py.jinja @@ -0,0 +1,265 @@ +from __future__ import annotations + +from collections.abc import Mapping +from typing import Any, TypeVar, BinaryIO, TextIO, TYPE_CHECKING, Generator + +from attrs import define as _attrs_define +from attrs import field as _attrs_field +{% if model.is_multipart_body %} +import json +from .. import types +{% endif %} + +from ..types import UNSET, Unset + +{% for relative in model.relative_imports | sort %} +{{ relative }} +{% endfor %} + +{% for lazy_import in model.lazy_imports | sort %} +{% if loop.first %} +if TYPE_CHECKING: +{% endif %} + {{ lazy_import }} +{% endfor %} + + +{% if model.additional_properties %} +{% set additional_property_type = 'Any' if model.additional_properties == True else model.additional_properties.get_type_string() %} +{% endif %} + +{% set class_name = model.class_info.name %} +{% set module_name = model.class_info.module_name %} + +{% from "helpers.jinja" import safe_docstring %} + +T = TypeVar("T", bound="{{ class_name }}") + +{% macro class_docstring_content(model) %} + {% if model.title %}{{ model.title | wordwrap(116) }} + + {% endif -%} + {%- if model.description %}{{ model.description | wordwrap(116) }} + + {% endif %} + {% if not model.title and not model.description %} + {# Leave extra space so that a section doesn't start on the first line #} + + {% endif %} + {% if model.example %} + Example: + {{ model.example | string | wordwrap(112) | indent(12) }} + + {% endif %} + {% if (not config.docstrings_on_attributes) and (model.required_properties or model.optional_properties) %} + Attributes: + {% for property in model.required_properties + model.optional_properties %} + {{ property.to_docstring() | wordwrap(112) | indent(12) }} + {% endfor %}{% endif %} +{% endmacro %} + +{% macro declare_property(property) %} +{%- if config.docstrings_on_attributes and property.description -%} +{{ property.to_string() }} +{{ safe_docstring(property.description, omit_if_empty=True) | wordwrap(112) }} +{%- else -%} +{{ property.to_string() }} +{%- endif -%} +{% endmacro %} + +@_attrs_define +class {{ class_name }}: + {{ safe_docstring(class_docstring_content(model), omit_if_empty=config.docstrings_on_attributes) | indent(4) }} + + {% for property in model.required_properties + model.optional_properties %} + {% if property.default is none and property.required %} + {{ declare_property(property) | indent(4) }} + {% endif %} + {% endfor %} + {% for property in model.required_properties + model.optional_properties %} + {% if property.default is not none or not property.required %} + {{ declare_property(property) | indent(4) }} + {% endif %} + {% endfor %} + {% if model.additional_properties %} + additional_properties: dict[str, {{ additional_property_type }}] = _attrs_field(init=False, factory=dict) + {% endif %} + +{% macro _transform_property(property, content) %} +{% import "property_templates/" + property.template as prop_template %} +{%- if prop_template.transform -%} +{{ prop_template.transform(property=property, source=content, destination=property.python_name) }} +{%- else -%} +{{ property.python_name }} = {{ content }} +{%- endif -%} +{% endmacro %} + +{% macro multipart(property, source, destination) %} +{% import "property_templates/" + property.template as prop_template %} +{% if not property.required %} +if not isinstance({{source}}, Unset): + {{ prop_template.multipart(property, source, destination) | indent(4) }} +{% else %} +{{ prop_template.multipart(property, source, destination) }} +{% endif %} +{% endmacro %} + +{% macro _prepare_field_dict() %} +field_dict: dict[str, Any] = {} +{% if model.additional_properties %} +{% import "property_templates/" + model.additional_properties.template as prop_template %} +{% if prop_template.transform %} +for prop_name, prop in self.additional_properties.items(): + {{ prop_template.transform(model.additional_properties, "prop", "field_dict[prop_name]", declare_type=false) | indent(4) }} +{% else %} +field_dict.update(self.additional_properties) +{%- endif -%} +{%- endif -%} +{% endmacro %} + +{% macro _to_dict() %} +{% for property in model.required_properties + model.optional_properties -%} +{{ _transform_property(property, "self." + property.python_name) }} + +{% endfor %} + +{{ _prepare_field_dict() }} +{% if model.required_properties | length > 0 or model.optional_properties | length > 0 %} +field_dict.update({ + {% for property in model.required_properties + model.optional_properties %} + {% if property.required %} + "{{ property.name }}": {{ property.python_name }}, + {% endif %} + {% endfor %} +}) +{% endif %} +{% for property in model.optional_properties %} +{% if not property.required %} +if {{ property.python_name }} is not UNSET: + field_dict["{{ property.name }}"] = {{ property.python_name }} +{% endif %} +{% endfor %} + +return field_dict +{% endmacro %} + + def to_dict(self) -> dict[str, Any]: + {% for lazy_import in model.lazy_imports | sort %} + {{ lazy_import }} + {% endfor %} + {{ _to_dict() | indent(8) }} + +{% if model.is_multipart_body %} + def to_multipart(self) -> types.RequestFiles: + {% for lazy_import in model.lazy_imports | sort %} + {{ lazy_import }} + {% endfor %} + files: types.RequestFiles = [] + + {% for property in model.required_properties + model.optional_properties %} + {% set destination = "\"" + property.name + "\"" %} + {{ multipart(property, "self." + property.python_name, destination) | indent(8) }} + + {% endfor %} + + {% if model.additional_properties %} + for prop_name, prop in self.additional_properties.items(): + {{ multipart(model.additional_properties, "prop", "prop_name") | indent(4) }} + {% endif %} + + return files + +{% endif %} + + @classmethod + def from_dict(cls: type[T], src_dict: Mapping[str, Any]) -> T: + {% for lazy_import in model.lazy_imports | sort %} + {{ lazy_import }} + {% endfor %} +{% if (model.required_properties or model.optional_properties or model.additional_properties) %} + d = dict(src_dict) +{% for property in model.required_properties + model.optional_properties %} + {% if property.required %} + {% if class_name == "ApiError" %} + {% if property.name == "status" %} + {% set property_source = 'd.pop("status", 0)' %} + {% elif property.name == "message" %} + {% set property_source = 'd.pop("message", "Unknown API error")' %} + {% elif property.name == "requestId" %} + {% set property_source = 'd.pop("requestId", "")' %} + {% elif property.name == "path" %} + {% set property_source = 'd.pop("path", "")' %} + {% elif property.name == "method" %} + {% set property_source = 'd.pop("method", "")' %} + {% elif property.name == "logId" %} + {% set property_source = 'd.pop("logId", "")' %} + {% elif property.name == "handler" %} + {% set property_source = 'd.pop("handler", "")' %} + {% elif property.name == "timestamp" %} + {% set property_source = 'd.pop("timestamp", "")' %} + {% else %} + {% set property_source = 'd.pop("' + property.name + '")' %} + {% endif %} + {% else %} + {% set property_source = 'd.pop("' + property.name + '")' %} + {% endif %} + {% else %} + {% set property_source = 'd.pop("' + property.name + '", UNSET)' %} + {% endif %} + {% import "property_templates/" + property.template as prop_template %} + {% if prop_template.construct %} + {{ prop_template.construct(property, property_source) | indent(8) }} + {% else %} + {{ property.python_name }} = {{ property_source }} + {% endif %} + +{% endfor %} +{% endif %} + {{ module_name }} = cls( +{% for property in model.required_properties + model.optional_properties %} + {{ property.python_name }}={{ property.python_name }}, +{% endfor %} + ) + +{% if model.additional_properties %} + {% if model.additional_properties.template %}{# Can be a bool instead of an object #} + {% import "property_templates/" + model.additional_properties.template as prop_template %} + +{% if model.additional_properties.lazy_imports %} + {% for lazy_import in model.additional_properties.lazy_imports | sort %} + {{ lazy_import }} + {% endfor %} +{% endif %} + {% else %} + {% set prop_template = None %} + {% endif %} + {% if prop_template and prop_template.construct %} + additional_properties = {} + for prop_name, prop_dict in d.items(): + {{ prop_template.construct(model.additional_properties, "prop_dict") | indent(12) }} + additional_properties[prop_name] = {{ model.additional_properties.python_name }} + + {{ module_name }}.additional_properties = additional_properties + {% else %} + {{ module_name }}.additional_properties = d + {% endif %} +{% endif %} + return {{ module_name }} + + {% if model.additional_properties %} + @property + def additional_keys(self) -> list[str]: + return list(self.additional_properties.keys()) + + def __getitem__(self, key: str) -> {{ additional_property_type }}: + return self.additional_properties[key] + + def __setitem__(self, key: str, value: {{ additional_property_type }}) -> None: + self.additional_properties[key] = value + + def __delitem__(self, key: str) -> None: + del self.additional_properties[key] + + def __contains__(self, key: str) -> bool: + return key in self.additional_properties + {% endif %} diff --git a/test/unit/test_api_access.py b/test/unit/test_api_access.py index d574c18..17b9f2d 100644 --- a/test/unit/test_api_access.py +++ b/test/unit/test_api_access.py @@ -12,6 +12,7 @@ DatabaseDeleteError, OpenApiAccess, OpenApiError, + ensure_type, timestamp_name, ) from exasol.saas.client.openapi.models.api_error import ApiError @@ -301,3 +302,60 @@ def test_wait_until_allowed_ip_deleted_retries(api_mock, monkeypatch) -> None: timeout=timedelta(seconds=1), interval=timedelta(milliseconds=10), ) + + +def test_api_error_from_dict_tolerates_missing_fields() -> None: + error = ApiError.from_dict( + { + "status": 500, + "message": "boom", + } + ) + + assert error.status == 500 + assert error.message == "boom" + assert error.request_id == "" + assert error.path == "" + assert error.method == "" + assert error.log_id == "" + assert error.handler == "" + assert error.timestamp == "" + assert error.causes is UNSET + + +def test_ensure_type_raises_open_api_error_for_malformed_error_payload() -> None: + malformed_error = ApiError.from_dict({"message": "backend failed"}) + + with pytest.raises( + OpenApiError, + match="Failed to do something: backend failed\\.", + ): + ensure_type(DatabaseSettings, malformed_error, "Failed to do something") + + +def test_list_database_ids_skips_deleted_databases(api_mock, monkeypatch) -> None: + from exasol.saas.client.api_access import list_databases as api + + monkeypatch.setattr( + api, + "sync", + Mock( + return_value=[ + database_response("active-db"), + ExasolDatabase( + status=Status.DELETING, + id="deleted-db-id", + name="deleted-db", + clusters=ExasolDatabaseClusters(total=1, running=0), + provider="aws", + region="eu-central-1", + created_at=datetime(2026, 1, 1), + created_by="tester", + deleted_at=datetime(2026, 1, 2), + deleted_by="tester", + ), + ] + ), + ) + + assert list(api_mock.list_database_ids()) == ["db-id"] From db92e62442fbe19809429b805ee868fdedd0d5d5 Mon Sep 17 00:00:00 2001 From: Torsten Kilias Date: Sat, 27 Jun 2026 01:43:19 +0200 Subject: [PATCH 05/22] Stabilize deletion polling in integration helpers --- exasol/saas/client/api_access.py | 44 ++++++- test/unit/test_api_access.py | 192 ++++++++++++++++++++++++++++--- 2 files changed, 214 insertions(+), 22 deletions(-) diff --git a/exasol/saas/client/api_access.py b/exasol/saas/client/api_access.py index 6eb93d8..82640f7 100644 --- a/exasol/saas/client/api_access.py +++ b/exasol/saas/client/api_access.py @@ -135,6 +135,10 @@ class InternalError(Exception): """ +def _is_not_found(resp: ApiError, entity: str = "User/Database") -> bool: + return resp.status == 404 and f"{entity} not found" in resp.message + + def create_saas_client( host: str, pat: str, @@ -311,14 +315,42 @@ def wait_until_deleted( timeout: timedelta = timedelta(minutes=5), interval: timedelta = timedelta(seconds=10), ): + terminal = {Status.DELETED} + in_progress = {Status.DELETING, Status.TODELETE} + @interval_retry(interval, timeout) - def verify_not_listed() -> bool: + def verify_deleted() -> bool: + resp = get_database.sync( + self._account_id, + database_id, + client=self._client, + ) + if isinstance(resp, ApiError): + if _is_not_found(resp): + return True + raise OpenApiError( + f"Failed to get database {database_id}", + resp, + ) + + if resp.deleted_at is not UNSET or resp.deleted_by is not UNSET: + return True + + if resp.status in terminal: + return True + + if resp.status in in_progress: + LOG.info("- Database deletion status: %s ...", resp.status) + raise TryAgain + if database_id in self.list_database_ids(): + LOG.info("- Database deletion status: %s ...", resp.status) raise TryAgain + return True try: - return verify_not_listed() + return verify_deleted() except (TryAgain, RetryError) as ex: raise DatabaseDeleteTimeout from ex @@ -424,7 +456,7 @@ def get_database_settings( database_id: str, ) -> openapi.models.DatabaseSettings | None: def is_retry(resp: ApiError) -> bool: - return resp.status == 404 and "User/Database not found" in resp.message + return _is_not_found(resp) @interval_retry( interval=timedelta(seconds=5), @@ -506,7 +538,11 @@ def list_allowed_ip_ids(self) -> Iterable[str]: resp = list_allowed_i_ps.sync(self._account_id, client=self._client) or [] # actually list[openapi.models.AllowedIP] ips = ensure_type(list, resp, "Failed to retrieve the list of allowed ips") - return (x.id for x in ips) + return ( + ip.id + for ip in ips + if ip.deleted_at is UNSET and ip.deleted_by is UNSET + ) def wait_until_allowed_ip_listed( self, diff --git a/test/unit/test_api_access.py b/test/unit/test_api_access.py index 17b9f2d..29f0ae8 100644 --- a/test/unit/test_api_access.py +++ b/test/unit/test_api_access.py @@ -9,12 +9,14 @@ from tenacity import TryAgain from exasol.saas.client.api_access import ( + DatabaseDeleteTimeout, DatabaseDeleteError, OpenApiAccess, OpenApiError, ensure_type, timestamp_name, ) +from exasol.saas.client.openapi.models.allowed_ip import AllowedIP from exasol.saas.client.openapi.models.api_error import ApiError from exasol.saas.client.openapi.models.database_settings import DatabaseSettings from exasol.saas.client.openapi.models.exasol_database import ExasolDatabase @@ -76,6 +78,14 @@ def list_allowed_ips_mock(monkeypatch, side_effect) -> Mock: return mock +def get_database_mock(monkeypatch, side_effect) -> Mock: + from exasol.saas.client.api_access import get_database as api + + mock = Mock(side_effect=side_effect) + monkeypatch.setattr(api, "sync", mock) + return mock + + def database_response(name: str = "db") -> ExasolDatabase: return ExasolDatabase( status=Status.CREATING, @@ -89,6 +99,23 @@ def database_response(name: str = "db") -> ExasolDatabase: ) +def allowed_ip_response( + id: str = "ip-1", + *, + deleted_at=UNSET, + deleted_by=UNSET, +) -> AllowedIP: + return AllowedIP( + id=id, + name="test-ip", + cidr_ip="0.0.0.0/0", + created_at=datetime(2026, 1, 1), + created_by="tester", + deleted_at=deleted_at, + deleted_by=deleted_by, + ) + + def database_settings_response(num_nodes: int = 2) -> DatabaseSettings: return DatabaseSettings( offload_enabled=False, @@ -227,22 +254,6 @@ def test_database_context_forwards_num_nodes(api_mock, monkeypatch) -> None: def test_get_database_settings_retries_transient_not_found( api_mock, monkeypatch ) -> None: - def immediate_retry(*_args, **_kwargs): - def decorate(func): - def wrapped(): - for _ in range(5): - try: - return func() - except TryAgain: - pass - except Exception: - raise - return func() - - return wrapped - - return decorate - monkeypatch.setattr( "exasol.saas.client.api_access.interval_retry", immediate_retry, @@ -281,7 +292,7 @@ def test_get_database_settings_raises_non_retryable_error( def test_wait_until_allowed_ip_listed_retries(api_mock, monkeypatch) -> None: list_allowed_ips_mock( monkeypatch, - [[], [Mock(id="ip-1")]], + [[], [allowed_ip_response("ip-1")]], ) api_mock.wait_until_allowed_ip_listed( @@ -294,7 +305,49 @@ def test_wait_until_allowed_ip_listed_retries(api_mock, monkeypatch) -> None: def test_wait_until_allowed_ip_deleted_retries(api_mock, monkeypatch) -> None: list_allowed_ips_mock( monkeypatch, - [[Mock(id="ip-1")], []], + [[allowed_ip_response("ip-1")], []], + ) + + api_mock.wait_until_allowed_ip_deleted( + "ip-1", + timeout=timedelta(seconds=1), + interval=timedelta(milliseconds=10), + ) + + +def test_list_allowed_ip_ids_skips_deleted_entries(api_mock, monkeypatch) -> None: + list_allowed_ips_mock( + monkeypatch, + [ + [ + allowed_ip_response("ip-1"), + allowed_ip_response( + "ip-2", + deleted_at=datetime(2026, 1, 2), + deleted_by="tester", + ), + ] + ], + ) + + assert list(api_mock.list_allowed_ip_ids()) == ["ip-1"] + + +def test_wait_until_allowed_ip_deleted_ignores_soft_deleted_entries( + api_mock, monkeypatch +) -> None: + list_allowed_ips_mock( + monkeypatch, + [ + [allowed_ip_response("ip-1")], + [ + allowed_ip_response( + "ip-1", + deleted_at=datetime(2026, 1, 2), + deleted_by="tester", + ) + ], + ], ) api_mock.wait_until_allowed_ip_deleted( @@ -359,3 +412,106 @@ def test_list_database_ids_skips_deleted_databases(api_mock, monkeypatch) -> Non ) assert list(api_mock.list_database_ids()) == ["db-id"] + + +def immediate_retry(*_args, **_kwargs): + def decorate(func): + def wrapped(): + for _ in range(5): + try: + return func() + except TryAgain: + pass + except Exception: + raise + return func() + + return wrapped + + return decorate + + +def test_wait_until_deleted_uses_get_database_until_not_found( + api_mock, monkeypatch +) -> None: + monkeypatch.setattr( + "exasol.saas.client.api_access.interval_retry", + immediate_retry, + ) + get_database_mock( + monkeypatch, + [ + database_response("db-active"), + ExasolDatabase( + status=Status.TODELETE, + id="db-id", + name="db-active", + clusters=ExasolDatabaseClusters(total=1, running=0), + provider="aws", + region="eu-central-1", + created_at=datetime(2026, 1, 1), + created_by="tester", + ), + api_error(404, "User/Database not found"), + ], + ) + list_databases_mock = Mock( + side_effect=[ + [database_response("db-active")], + [], + ] + ) + from exasol.saas.client.api_access import list_databases as list_api + + monkeypatch.setattr(list_api, "sync", list_databases_mock) + + api_mock.wait_until_deleted("db-id") + + +def test_wait_until_deleted_accepts_soft_deleted_database( + api_mock, monkeypatch +) -> None: + monkeypatch.setattr( + "exasol.saas.client.api_access.interval_retry", + immediate_retry, + ) + get_database_mock( + monkeypatch, + [ + ExasolDatabase( + status=Status.DELETED, + id="db-id", + name="db-active", + clusters=ExasolDatabaseClusters(total=1, running=0), + provider="aws", + region="eu-central-1", + created_at=datetime(2026, 1, 1), + created_by="tester", + deleted_at=datetime(2026, 1, 2), + deleted_by="tester", + ) + ], + ) + + api_mock.wait_until_deleted("db-id") + + +def test_wait_until_deleted_times_out_for_active_database( + api_mock, monkeypatch +) -> None: + monkeypatch.setattr( + "exasol.saas.client.api_access.interval_retry", + lambda *_args, **_kwargs: (lambda func: func), + ) + monkeypatch.setattr( + api_mock, + "list_database_ids", + Mock(return_value=iter(["db-id"])), + ) + get_database_mock( + monkeypatch, + [database_response("db-active")], + ) + + with pytest.raises(DatabaseDeleteTimeout): + api_mock.wait_until_deleted("db-id") From e23ad333cb320799a325d5f91abdf52d4015ecda Mon Sep 17 00:00:00 2001 From: Torsten Kilias Date: Sat, 27 Jun 2026 02:34:46 +0200 Subject: [PATCH 06/22] Parallelize slow checks with dynamic matrix --- .github/workflows/matrix-slow.yml | 37 +++++++++++++++++++++++++++++++ .github/workflows/slow-checks.yml | 19 +++++++++++++--- noxfile.py | 19 ++++++++++++++++ 3 files changed, 72 insertions(+), 3 deletions(-) create mode 100644 .github/workflows/matrix-slow.yml diff --git a/.github/workflows/matrix-slow.yml b/.github/workflows/matrix-slow.yml new file mode 100644 index 0000000..734f9b1 --- /dev/null +++ b/.github/workflows/matrix-slow.yml @@ -0,0 +1,37 @@ +name: Build Matrix (Slow Tests) + +on: + workflow_call: + outputs: + matrix: + description: "Generates the slow test build matrix" + value: ${{ jobs.set-matrix-slow.outputs.matrix }} + +jobs: + set-matrix-slow: + runs-on: "ubuntu-24.04" + permissions: + contents: read + steps: + - name: Check out Repository + id: check-out-repository + uses: actions/checkout@v6 + + - name: Set up Python & Poetry Environment + id: set-up-python-and-poetry-environment + uses: exasol/python-toolbox/.github/actions/python-environment@v7 + with: + python-version: "3.10" + poetry-version: "2.3.0" + + - name: Generate Matrix + id: generate-matrix + run: poetry run -- nox -s matrix:slow + + - name: Set Matrix + id: set-matrix + run: | + echo "matrix=$(poetry run -- nox -s matrix:slow)" >> $GITHUB_OUTPUT + + outputs: + matrix: ${{ steps.set-matrix.outputs.matrix }} diff --git a/.github/workflows/slow-checks.yml b/.github/workflows/slow-checks.yml index 1e20c46..e854887 100644 --- a/.github/workflows/slow-checks.yml +++ b/.github/workflows/slow-checks.yml @@ -4,12 +4,22 @@ on: workflow_call: jobs: + build-slow-matrix: + name: Build Slow Matrix + uses: ./.github/workflows/matrix-slow.yml + permissions: + contents: read run-integration-tests: - name: Run Integration Tests + name: Run Integration Tests (${{ matrix.test-name }}) + needs: + - build-slow-matrix runs-on: "ubuntu-24.04" permissions: contents: read + strategy: + fail-fast: false + matrix: ${{ fromJSON(needs.build-slow-matrix.outputs.matrix) }} env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} steps: @@ -26,7 +36,10 @@ jobs: - name: Run Integration Tests id: run-integration-tests - run: poetry run -- nox -s test:integration -- --coverage + run: | + poetry run coverage run \ + --rcfile=pyproject.toml \ + -m pytest -v "${{ matrix.test-path }}" env: SAAS_HOST: ${{ secrets.INTEGRATION_TEAM_SAAS_STAGING_HOST }} SAAS_ACCOUNT_ID: ${{ secrets.INTEGRATION_TEAM_SAAS_STAGING_ACCOUNT_ID }} @@ -38,6 +51,6 @@ jobs: id: upload-artifacts uses: actions/upload-artifact@v7 with: - name: coverage-python3.10-slow + name: coverage-python3.10-slow-${{ matrix.test-name }} path: .coverage include-hidden-files: true diff --git a/noxfile.py b/noxfile.py index 17822bb..da0fa24 100644 --- a/noxfile.py +++ b/noxfile.py @@ -26,6 +26,19 @@ DEST_DIR = "exasol/saas/client/openapi" +def _slow_test_matrix() -> dict[str, list[dict[str, str]]]: + files = sorted(Path("test/integration").glob("test_*.py")) + return { + "include": [ + { + "test-name": path.stem.removeprefix("test_").replace("_", "-"), + "test-path": str(path), + } + for path in files + ] + } + + def _download_openapi_json() -> Path: url = f"{SAAS_HOST}/openapi.json" response = requests.get(url) @@ -127,3 +140,9 @@ def check_api_outdated(session: Session): """ generate_api(session) session.run("git", "diff", "--exit-code", DEST_DIR) + + +@nox.session(name="matrix:slow", python=False) +def slow_matrix(session: Session): + """Output the build matrix for slow integration tests as JSON.""" + print(json.dumps(_slow_test_matrix())) From 103c4ffc089886c809b71d440da2f5006645fb60 Mon Sep 17 00:00:00 2001 From: Torsten Kilias Date: Sat, 27 Jun 2026 02:42:59 +0200 Subject: [PATCH 07/22] Handle transient None responses while polling deletions --- exasol/saas/client/api_access.py | 8 +++++--- test/unit/test_api_access.py | 22 +++++++++++++++++++++- 2 files changed, 26 insertions(+), 4 deletions(-) diff --git a/exasol/saas/client/api_access.py b/exasol/saas/client/api_access.py index 82640f7..ea4e41a 100644 --- a/exasol/saas/client/api_access.py +++ b/exasol/saas/client/api_access.py @@ -333,6 +333,10 @@ def verify_deleted() -> bool: resp, ) + if resp is None: + LOG.info("- Database deletion status: unavailable ...") + raise TryAgain + if resp.deleted_at is not UNSET or resp.deleted_by is not UNSET: return True @@ -539,9 +543,7 @@ def list_allowed_ip_ids(self) -> Iterable[str]: # actually list[openapi.models.AllowedIP] ips = ensure_type(list, resp, "Failed to retrieve the list of allowed ips") return ( - ip.id - for ip in ips - if ip.deleted_at is UNSET and ip.deleted_by is UNSET + ip.id for ip in ips if ip.deleted_at is UNSET and ip.deleted_by is UNSET ) def wait_until_allowed_ip_listed( diff --git a/test/unit/test_api_access.py b/test/unit/test_api_access.py index 29f0ae8..7eec273 100644 --- a/test/unit/test_api_access.py +++ b/test/unit/test_api_access.py @@ -9,8 +9,8 @@ from tenacity import TryAgain from exasol.saas.client.api_access import ( - DatabaseDeleteTimeout, DatabaseDeleteError, + DatabaseDeleteTimeout, OpenApiAccess, OpenApiError, ensure_type, @@ -496,6 +496,26 @@ def test_wait_until_deleted_accepts_soft_deleted_database( api_mock.wait_until_deleted("db-id") +def test_wait_until_deleted_retries_when_get_database_returns_none( + api_mock, monkeypatch +) -> None: + monkeypatch.setattr( + "exasol.saas.client.api_access.interval_retry", + immediate_retry, + ) + get_database = get_database_mock( + monkeypatch, + [ + None, + api_error(404, "User/Database not found"), + ], + ) + + api_mock.wait_until_deleted("db-id") + + assert get_database.call_count == 2 + + def test_wait_until_deleted_times_out_for_active_database( api_mock, monkeypatch ) -> None: From 79865dcfd88c1d8b1ed22c0d73a5a569b1641f61 Mon Sep 17 00:00:00 2001 From: Torsten Kilias Date: Sat, 27 Jun 2026 03:07:54 +0200 Subject: [PATCH 08/22] Increase delete polling timeouts --- exasol/saas/client/api_access.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/exasol/saas/client/api_access.py b/exasol/saas/client/api_access.py index ea4e41a..b18fa70 100644 --- a/exasol/saas/client/api_access.py +++ b/exasol/saas/client/api_access.py @@ -312,7 +312,7 @@ def _ignore_failures(self, ignore: bool = False): def wait_until_deleted( self, database_id: str, - timeout: timedelta = timedelta(minutes=5), + timeout: timedelta = timedelta(minutes=10), interval: timedelta = timedelta(seconds=10), ): terminal = {Status.DELETED} @@ -362,7 +362,7 @@ def delete_database( self, database_id: str, ignore_failures: bool = False, - timeout: timedelta = timedelta(minutes=30), + timeout: timedelta = timedelta(minutes=45), min_interval: timedelta = timedelta(seconds=1), max_interval: timedelta = timedelta(minutes=2), ) -> None: @@ -563,7 +563,7 @@ def verify_listed() -> bool: def wait_until_allowed_ip_deleted( self, allowed_ip_id: str, - timeout: timedelta = timedelta(minutes=1), + timeout: timedelta = timedelta(minutes=10), interval: timedelta = timedelta(seconds=5), ) -> None: @interval_retry(interval, timeout) From 6fb959e4d9afdf5c94422cb2cb156da314ecdd87 Mon Sep 17 00:00:00 2001 From: Torsten Kilias Date: Sat, 27 Jun 2026 10:34:42 +0200 Subject: [PATCH 09/22] Add debug logging for API helper responses --- .github/workflows/slow-checks.yml | 2 +- exasol/saas/client/api_access.py | 148 ++++++++++++++++++++++++++++-- test/unit/test_api_access.py | 55 +++++++++++ 3 files changed, 195 insertions(+), 10 deletions(-) diff --git a/.github/workflows/slow-checks.yml b/.github/workflows/slow-checks.yml index e854887..ff33e3e 100644 --- a/.github/workflows/slow-checks.yml +++ b/.github/workflows/slow-checks.yml @@ -45,7 +45,7 @@ jobs: SAAS_ACCOUNT_ID: ${{ secrets.INTEGRATION_TEAM_SAAS_STAGING_ACCOUNT_ID }} SAAS_PAT: ${{ secrets.INTEGRATION_TEAM_SAAS_STAGING_PAT }} PROJECT_SHORT_TAG: SAPIPY - PYTEST_ADDOPTS: '-o log_cli=true -o log_cli_level=INFO' + PYTEST_ADDOPTS: '-o log_cli=true -o log_cli_level=DEBUG' - name: Upload Artifacts id: upload-artifacts diff --git a/exasol/saas/client/api_access.py b/exasol/saas/client/api_access.py index b18fa70..2ff7e7a 100644 --- a/exasol/saas/client/api_access.py +++ b/exasol/saas/client/api_access.py @@ -56,7 +56,6 @@ from exasol.saas.client.openapi.types import UNSET LOG = logging.getLogger(__name__) -LOG.setLevel(logging.INFO) logging.getLogger("httpx").setLevel(logging.WARNING) @@ -129,6 +128,30 @@ def ensure_type( raise OpenApiError(message, api_error) +def _serialize_api_output(response: Any) -> Any: + if isinstance(response, list): + return [_serialize_api_output(item) for item in response] + + to_dict = getattr(response, "to_dict", None) + if callable(to_dict): + return to_dict() + + return response + + +def _log_api_output(operation: str, response: Any, **context: Any) -> None: + if not LOG.isEnabledFor(logging.DEBUG): + return + + suffix = f" {context}" if context else "" + LOG.debug( + "%s response%s: %s", + operation, + suffix, + _serialize_api_output(response), + ) + + class InternalError(Exception): """ Internal error during delete with retry. @@ -160,6 +183,12 @@ def _get_database_id( Finds the database id, given the database name. """ dbs = list_databases.sync(account_id, client=client) + _log_api_output( + "list_databases.sync", + dbs, + account_id=account_id, + database_name=database_name, + ) dbs = list( filter( lambda db: (db.name == database_name) # type: ignore @@ -229,12 +258,25 @@ def get_connection_params( account_id, client, database_name=database_name ) clusters = list_clusters.sync(account_id, database_id, client=client) + _log_api_output( + "list_clusters.sync", + clusters, + account_id=account_id, + database_id=database_id, + ) cluster_id = next( filter(lambda cl: cl.main_cluster, clusters) # type: ignore ).id resp = get_cluster_connection.sync( account_id, database_id, cluster_id, client=client ) + _log_api_output( + "get_cluster_connection.sync", + resp, + account_id=account_id, + database_id=database_id, + cluster_id=cluster_id, + ) connection = ensure_type( openapi.models.ClusterConnection, resp, @@ -296,6 +338,12 @@ def minutes(x: timedelta) -> int: client=self._client, body=database_spec, ) + _log_api_output( + "create_database.sync", + resp, + account_id=self._account_id, + database_name=name, + ) database = ensure_type( ExasolDatabase, resp, f"Failed to create database {name}" ) @@ -325,6 +373,12 @@ def verify_deleted() -> bool: database_id, client=self._client, ) + _log_api_output( + "get_database.sync", + resp, + account_id=self._account_id, + database_id=database_id, + ) if isinstance(resp, ApiError): if _is_not_found(resp): return True @@ -347,7 +401,13 @@ def verify_deleted() -> bool: LOG.info("- Database deletion status: %s ...", resp.status) raise TryAgain - if database_id in self.list_database_ids(): + visible_database_ids = list(self.list_database_ids()) + LOG.debug( + "wait_until_deleted visible database IDs {'database_id': %s}: %s", + database_id, + visible_database_ids, + ) + if database_id in visible_database_ids: LOG.info("- Database deletion status: %s ...", resp.status) raise TryAgain @@ -388,6 +448,12 @@ def delete_with_retry() -> None: database_id, client=self._client, ) + _log_api_output( + "delete_database.sync", + resp, + account_id=self._account_id, + database_id=database_id, + ) if not isinstance(resp, ApiError): # success return @@ -409,11 +475,18 @@ def delete_with_retry() -> None: def list_database_ids(self) -> Iterable[str]: resp = list_databases.sync(self._account_id, client=self._client) or [] + _log_api_output( + "list_databases.sync", + resp, + account_id=self._account_id, + ) # actually list[ExasolDatabase] dbs = ensure_type(list, resp, "Failed to list databases") - return ( + active_database_ids = [ db.id for db in dbs if db.deleted_at is UNSET and db.deleted_by is UNSET - ) + ] + LOG.debug("list_database_ids visible IDs: %s", active_database_ids) + return iter(active_database_ids) @contextmanager def database( @@ -451,6 +524,12 @@ def get_database( database_id, client=self._client, ) + _log_api_output( + "get_database.sync", + resp, + account_id=self._account_id, + database_id=database_id, + ) return ensure_type( ExasolDatabase, resp, f"Failed to get database {database_id}" ) @@ -472,6 +551,12 @@ def retrieve_settings() -> openapi.models.DatabaseSettings: database_id, client=self._client, ) + _log_api_output( + "get_database_settings.sync", + resp, + account_id=self._account_id, + database_id=database_id, + ) if isinstance(resp, ApiError) and is_retry(resp): raise TryAgain return ensure_type( @@ -515,6 +600,12 @@ def clusters( ) or [] ) + _log_api_output( + "list_clusters.sync", + resp, + account_id=self._account_id, + database_id=database_id, + ) # actually list[openapi.models.Cluster] return ensure_type( list, resp, f"Failed to list clusters of database {database_id}" @@ -531,6 +622,13 @@ def get_connection( cluster_id, client=self._client, ) + _log_api_output( + "get_cluster_connection.sync", + resp, + account_id=self._account_id, + database_id=database_id, + cluster_id=cluster_id, + ) return ensure_type( openapi.models.ClusterConnection, resp, @@ -540,11 +638,18 @@ def get_connection( def list_allowed_ip_ids(self) -> Iterable[str]: resp = list_allowed_i_ps.sync(self._account_id, client=self._client) or [] + _log_api_output( + "list_allowed_i_ps.sync", + resp, + account_id=self._account_id, + ) # actually list[openapi.models.AllowedIP] ips = ensure_type(list, resp, "Failed to retrieve the list of allowed ips") - return ( + visible_allowed_ip_ids = [ ip.id for ip in ips if ip.deleted_at is UNSET and ip.deleted_by is UNSET - ) + ] + LOG.debug("list_allowed_ip_ids visible IDs: %s", visible_allowed_ip_ids) + return iter(visible_allowed_ip_ids) def wait_until_allowed_ip_listed( self, @@ -554,7 +659,13 @@ def wait_until_allowed_ip_listed( ) -> None: @interval_retry(interval, timeout) def verify_listed() -> bool: - if allowed_ip_id not in self.list_allowed_ip_ids(): + visible_allowed_ip_ids = list(self.list_allowed_ip_ids()) + LOG.debug( + "wait_until_allowed_ip_listed visible IDs {'allowed_ip_id': %s}: %s", + allowed_ip_id, + visible_allowed_ip_ids, + ) + if allowed_ip_id not in visible_allowed_ip_ids: raise TryAgain return True @@ -568,7 +679,13 @@ def wait_until_allowed_ip_deleted( ) -> None: @interval_retry(interval, timeout) def verify_deleted() -> bool: - if allowed_ip_id in self.list_allowed_ip_ids(): + visible_allowed_ip_ids = list(self.list_allowed_ip_ids()) + LOG.debug( + "wait_until_allowed_ip_deleted visible IDs {'allowed_ip_id': %s}: %s", + allowed_ip_id, + visible_allowed_ip_ids, + ) + if allowed_ip_id in visible_allowed_ip_ids: raise TryAgain return True @@ -593,6 +710,12 @@ def add_allowed_ip( client=self._client, body=rule, ) + _log_api_output( + "add_allowed_ip.sync", + resp, + account_id=self._account_id, + cidr_ip=cidr_ip, + ) return ensure_type( openapi.models.AllowedIP, resp, @@ -601,7 +724,14 @@ def add_allowed_ip( def delete_allowed_ip(self, id: str, ignore_failures=False) -> Any | None: with self._ignore_failures(ignore_failures) as client: - return delete_allowed_ip.sync(self._account_id, id, client=client) + resp = delete_allowed_ip.sync(self._account_id, id, client=client) + _log_api_output( + "delete_allowed_ip.sync", + resp, + account_id=self._account_id, + allowed_ip_id=id, + ) + return resp @contextmanager def allowed_ip( diff --git a/test/unit/test_api_access.py b/test/unit/test_api_access.py index 7eec273..8bb72b0 100644 --- a/test/unit/test_api_access.py +++ b/test/unit/test_api_access.py @@ -1,3 +1,4 @@ +import logging from datetime import ( datetime, timedelta, @@ -13,6 +14,7 @@ DatabaseDeleteTimeout, OpenApiAccess, OpenApiError, + _log_api_output, ensure_type, timestamp_name, ) @@ -289,6 +291,25 @@ def test_get_database_settings_raises_non_retryable_error( api_mock.get_database_settings("db-id") +def test_log_api_output_serializes_payloads(caplog) -> None: + caplog.set_level(logging.DEBUG, logger="exasol.saas.client.api_access") + + _log_api_output( + "list_allowed_i_ps.sync", + [ + allowed_ip_response("ip-1"), + ApiError.from_dict({"status": 404, "message": "not found"}), + None, + ], + account_id="A1", + ) + + assert "list_allowed_i_ps.sync response {'account_id': 'A1'}" in caplog.text + assert "'id': 'ip-1'" in caplog.text + assert "'status': 404" in caplog.text + assert "None" in caplog.text + + def test_wait_until_allowed_ip_listed_retries(api_mock, monkeypatch) -> None: list_allowed_ips_mock( monkeypatch, @@ -315,6 +336,25 @@ def test_wait_until_allowed_ip_deleted_retries(api_mock, monkeypatch) -> None: ) +def test_wait_until_allowed_ip_listed_logs_visible_ids( + api_mock, monkeypatch, caplog +) -> None: + caplog.set_level(logging.DEBUG, logger="exasol.saas.client.api_access") + list_allowed_ips_mock( + monkeypatch, + [[allowed_ip_response("ip-1")]], + ) + + api_mock.wait_until_allowed_ip_listed( + "ip-1", + timeout=timedelta(seconds=1), + interval=timedelta(milliseconds=10), + ) + + assert "wait_until_allowed_ip_listed visible IDs" in caplog.text + assert "list_allowed_i_ps.sync response" in caplog.text + + def test_list_allowed_ip_ids_skips_deleted_entries(api_mock, monkeypatch) -> None: list_allowed_ips_mock( monkeypatch, @@ -414,6 +454,21 @@ def test_list_database_ids_skips_deleted_databases(api_mock, monkeypatch) -> Non assert list(api_mock.list_database_ids()) == ["db-id"] +def test_list_database_ids_logs_visible_ids(api_mock, monkeypatch, caplog) -> None: + from exasol.saas.client.api_access import list_databases as api + + caplog.set_level(logging.DEBUG, logger="exasol.saas.client.api_access") + monkeypatch.setattr( + api, + "sync", + Mock(return_value=[database_response("active-db")]), + ) + + assert list(api_mock.list_database_ids()) == ["db-id"] + assert "list_databases.sync response" in caplog.text + assert "list_database_ids visible IDs: ['db-id']" in caplog.text + + def immediate_retry(*_args, **_kwargs): def decorate(func): def wrapped(): From 3fb663c629f93a0218c6c824f457d607249a2585 Mon Sep 17 00:00:00 2001 From: Torsten Kilias Date: Sat, 27 Jun 2026 11:05:27 +0200 Subject: [PATCH 10/22] Fix stale delete polling in integration helpers --- exasol/saas/client/api_access.py | 61 ++++++++++++++++++++------ test/unit/test_api_access.py | 74 ++++++++++++++++++++++++-------- 2 files changed, 106 insertions(+), 29 deletions(-) diff --git a/exasol/saas/client/api_access.py b/exasol/saas/client/api_access.py index 2ff7e7a..53f5192 100644 --- a/exasol/saas/client/api_access.py +++ b/exasol/saas/client/api_access.py @@ -46,6 +46,7 @@ from exasol.saas.client.openapi.api.security import ( add_allowed_ip, delete_allowed_ip, + get_allowed_ip, list_allowed_i_ps, ) from exasol.saas.client.openapi.models import ( @@ -397,16 +398,19 @@ def verify_deleted() -> bool: if resp.status in terminal: return True - if resp.status in in_progress: - LOG.info("- Database deletion status: %s ...", resp.status) - raise TryAgain - visible_database_ids = list(self.list_database_ids()) LOG.debug( "wait_until_deleted visible database IDs {'database_id': %s}: %s", database_id, visible_database_ids, ) + if database_id not in visible_database_ids: + return True + + if resp.status in in_progress: + LOG.info("- Database deletion status: %s ...", resp.status) + raise TryAgain + if database_id in visible_database_ids: LOG.info("- Database deletion status: %s ...", resp.status) raise TryAgain @@ -659,13 +663,25 @@ def wait_until_allowed_ip_listed( ) -> None: @interval_retry(interval, timeout) def verify_listed() -> bool: - visible_allowed_ip_ids = list(self.list_allowed_ip_ids()) + resp = get_allowed_ip.sync( + self._account_id, + allowed_ip_id, + client=self._client, + ) + _log_api_output( + "get_allowed_ip.sync", + resp, + account_id=self._account_id, + allowed_ip_id=allowed_ip_id, + ) LOG.debug( - "wait_until_allowed_ip_listed visible IDs {'allowed_ip_id': %s}: %s", + "wait_until_allowed_ip_listed get result {'allowed_ip_id': %s}: %s", allowed_ip_id, - visible_allowed_ip_ids, + _serialize_api_output(resp), ) - if allowed_ip_id not in visible_allowed_ip_ids: + if isinstance(resp, ApiError): + raise TryAgain + if resp is None: raise TryAgain return True @@ -679,13 +695,34 @@ def wait_until_allowed_ip_deleted( ) -> None: @interval_retry(interval, timeout) def verify_deleted() -> bool: - visible_allowed_ip_ids = list(self.list_allowed_ip_ids()) + resp = get_allowed_ip.sync( + self._account_id, + allowed_ip_id, + client=self._client, + ) + _log_api_output( + "get_allowed_ip.sync", + resp, + account_id=self._account_id, + allowed_ip_id=allowed_ip_id, + ) LOG.debug( - "wait_until_allowed_ip_deleted visible IDs {'allowed_ip_id': %s}: %s", + "wait_until_allowed_ip_deleted get result {'allowed_ip_id': %s}: %s", allowed_ip_id, - visible_allowed_ip_ids, + _serialize_api_output(resp), ) - if allowed_ip_id in visible_allowed_ip_ids: + if isinstance(resp, ApiError): + if resp.status == 404: + return True + raise OpenApiError( + f"Failed to get allowed IP {allowed_ip_id}", + resp, + ) + if resp is None: + raise TryAgain + if resp.deleted_at is not UNSET or resp.deleted_by is not UNSET: + return True + if resp.id == allowed_ip_id: raise TryAgain return True diff --git a/test/unit/test_api_access.py b/test/unit/test_api_access.py index 8bb72b0..f2e1345 100644 --- a/test/unit/test_api_access.py +++ b/test/unit/test_api_access.py @@ -80,6 +80,14 @@ def list_allowed_ips_mock(monkeypatch, side_effect) -> Mock: return mock +def get_allowed_ip_mock(monkeypatch, side_effect) -> Mock: + from exasol.saas.client.api_access import get_allowed_ip as api + + mock = Mock(side_effect=side_effect) + monkeypatch.setattr(api, "sync", mock) + return mock + + def get_database_mock(monkeypatch, side_effect) -> Mock: from exasol.saas.client.api_access import get_database as api @@ -311,9 +319,9 @@ def test_log_api_output_serializes_payloads(caplog) -> None: def test_wait_until_allowed_ip_listed_retries(api_mock, monkeypatch) -> None: - list_allowed_ips_mock( + get_allowed_ip_mock( monkeypatch, - [[], [allowed_ip_response("ip-1")]], + [api_error(404, "Item not found"), allowed_ip_response("ip-1")], ) api_mock.wait_until_allowed_ip_listed( @@ -324,9 +332,12 @@ def test_wait_until_allowed_ip_listed_retries(api_mock, monkeypatch) -> None: def test_wait_until_allowed_ip_deleted_retries(api_mock, monkeypatch) -> None: - list_allowed_ips_mock( + get_allowed_ip_mock( monkeypatch, - [[allowed_ip_response("ip-1")], []], + [ + allowed_ip_response("ip-1"), + api_error(404, "Item not found"), + ], ) api_mock.wait_until_allowed_ip_deleted( @@ -340,9 +351,9 @@ def test_wait_until_allowed_ip_listed_logs_visible_ids( api_mock, monkeypatch, caplog ) -> None: caplog.set_level(logging.DEBUG, logger="exasol.saas.client.api_access") - list_allowed_ips_mock( + get_allowed_ip_mock( monkeypatch, - [[allowed_ip_response("ip-1")]], + [allowed_ip_response("ip-1")], ) api_mock.wait_until_allowed_ip_listed( @@ -351,8 +362,8 @@ def test_wait_until_allowed_ip_listed_logs_visible_ids( interval=timedelta(milliseconds=10), ) - assert "wait_until_allowed_ip_listed visible IDs" in caplog.text - assert "list_allowed_i_ps.sync response" in caplog.text + assert "wait_until_allowed_ip_listed get result" in caplog.text + assert "get_allowed_ip.sync response" in caplog.text def test_list_allowed_ip_ids_skips_deleted_entries(api_mock, monkeypatch) -> None: @@ -376,17 +387,15 @@ def test_list_allowed_ip_ids_skips_deleted_entries(api_mock, monkeypatch) -> Non def test_wait_until_allowed_ip_deleted_ignores_soft_deleted_entries( api_mock, monkeypatch ) -> None: - list_allowed_ips_mock( + get_allowed_ip_mock( monkeypatch, [ - [allowed_ip_response("ip-1")], - [ - allowed_ip_response( - "ip-1", - deleted_at=datetime(2026, 1, 2), - deleted_by="tester", - ) - ], + allowed_ip_response("ip-1"), + allowed_ip_response( + "ip-1", + deleted_at=datetime(2026, 1, 2), + deleted_by="tester", + ), ], ) @@ -523,6 +532,37 @@ def test_wait_until_deleted_uses_get_database_until_not_found( api_mock.wait_until_deleted("db-id") +def test_wait_until_deleted_accepts_stale_todelete_when_database_not_listed( + api_mock, monkeypatch +) -> None: + monkeypatch.setattr( + "exasol.saas.client.api_access.interval_retry", + immediate_retry, + ) + get_database_mock( + monkeypatch, + [ + ExasolDatabase( + status=Status.TODELETE, + id="db-id", + name="db-active", + clusters=ExasolDatabaseClusters(total=1, running=0), + provider="aws", + region="eu-central-1", + created_at=datetime(2026, 1, 1), + created_by="tester", + ) + ], + ) + monkeypatch.setattr( + api_mock, + "list_database_ids", + Mock(return_value=iter([])), + ) + + api_mock.wait_until_deleted("db-id") + + def test_wait_until_deleted_accepts_soft_deleted_database( api_mock, monkeypatch ) -> None: From 037a1e16e65e3b9f08eba9b126b3b54d1f4a3ae6 Mon Sep 17 00:00:00 2001 From: Torsten Kilias Date: Sat, 27 Jun 2026 11:48:39 +0200 Subject: [PATCH 11/22] Handle transitional database delete states --- exasol/saas/client/api_access.py | 8 +++-- test/unit/test_api_access.py | 56 ++++++++++++++++++++++++++++++++ 2 files changed, 62 insertions(+), 2 deletions(-) diff --git a/exasol/saas/client/api_access.py b/exasol/saas/client/api_access.py index 53f5192..808e3bd 100644 --- a/exasol/saas/client/api_access.py +++ b/exasol/saas/client/api_access.py @@ -361,7 +361,7 @@ def _ignore_failures(self, ignore: bool = False): def wait_until_deleted( self, database_id: str, - timeout: timedelta = timedelta(minutes=10), + timeout: timedelta = timedelta(minutes=20), interval: timedelta = timedelta(seconds=10), ): terminal = {Status.DELETED} @@ -487,7 +487,11 @@ def list_database_ids(self) -> Iterable[str]: # actually list[ExasolDatabase] dbs = ensure_type(list, resp, "Failed to list databases") active_database_ids = [ - db.id for db in dbs if db.deleted_at is UNSET and db.deleted_by is UNSET + db.id + for db in dbs + if db.deleted_at is UNSET + and db.deleted_by is UNSET + and db.status not in {Status.DELETING, Status.TODELETE} ] LOG.debug("list_database_ids visible IDs: %s", active_database_ids) return iter(active_database_ids) diff --git a/test/unit/test_api_access.py b/test/unit/test_api_access.py index f2e1345..f4fd84f 100644 --- a/test/unit/test_api_access.py +++ b/test/unit/test_api_access.py @@ -456,6 +456,16 @@ def test_list_database_ids_skips_deleted_databases(api_mock, monkeypatch) -> Non deleted_at=datetime(2026, 1, 2), deleted_by="tester", ), + ExasolDatabase( + status=Status.TODELETE, + id="todelete-db-id", + name="todelete-db", + clusters=ExasolDatabaseClusters(total=1, running=0), + provider="aws", + region="eu-central-1", + created_at=datetime(2026, 1, 1), + created_by="tester", + ), ] ), ) @@ -563,6 +573,52 @@ def test_wait_until_deleted_accepts_stale_todelete_when_database_not_listed( api_mock.wait_until_deleted("db-id") +def test_wait_until_deleted_accepts_todelete_when_helper_list_filters_it( + api_mock, monkeypatch +) -> None: + monkeypatch.setattr( + "exasol.saas.client.api_access.interval_retry", + immediate_retry, + ) + get_database_mock( + monkeypatch, + [ + ExasolDatabase( + status=Status.TODELETE, + id="db-id", + name="db-active", + clusters=ExasolDatabaseClusters(total=1, running=0), + provider="aws", + region="eu-central-1", + created_at=datetime(2026, 1, 1), + created_by="tester", + ) + ], + ) + from exasol.saas.client.api_access import list_databases as api + + monkeypatch.setattr( + api, + "sync", + Mock( + return_value=[ + ExasolDatabase( + status=Status.TODELETE, + id="db-id", + name="db-active", + clusters=ExasolDatabaseClusters(total=1, running=0), + provider="aws", + region="eu-central-1", + created_at=datetime(2026, 1, 1), + created_by="tester", + ) + ] + ), + ) + + api_mock.wait_until_deleted("db-id") + + def test_wait_until_deleted_accepts_soft_deleted_database( api_mock, monkeypatch ) -> None: From c239b0733b8494a914ac13ab1327143414da3197 Mon Sep 17 00:00:00 2001 From: Torsten Kilias Date: Sat, 27 Jun 2026 11:52:40 +0200 Subject: [PATCH 12/22] Split database integration tests --- test/integration/conftest.py | 9 +++++ ...atabases.py => test_database_lifecycle.py} | 37 +------------------ test/integration/test_database_num_nodes.py | 14 +++++++ 3 files changed, 24 insertions(+), 36 deletions(-) rename test/integration/{test_databases.py => test_database_lifecycle.py} (60%) create mode 100644 test/integration/test_database_num_nodes.py diff --git a/test/integration/conftest.py b/test/integration/conftest.py index 272ba1b..37912e5 100644 --- a/test/integration/conftest.py +++ b/test/integration/conftest.py @@ -67,6 +67,15 @@ def database_name(project_short_tag) -> str: return timestamp_name(project_short_tag) +@pytest.fixture +def local_name(project_short_tag: str | None) -> str: + """ + Other than global fixture database_name this fixture uses scope + "function" to generate an individual name for each test case. + """ + return timestamp_name(project_short_tag) + + @pytest.fixture(scope="session") def allow_connection(api_access) -> None: """ diff --git a/test/integration/test_databases.py b/test/integration/test_database_lifecycle.py similarity index 60% rename from test/integration/test_databases.py rename to test/integration/test_database_lifecycle.py index ba24432..95c3b3b 100644 --- a/test/integration/test_databases.py +++ b/test/integration/test_database_lifecycle.py @@ -1,16 +1,10 @@ import logging -from datetime import ( - datetime, - timedelta, -) +from datetime import timedelta import pytest from tenacity import RetryError from exasol.saas.client import PROMISING_STATES -from exasol.saas.client.api_access import ( - timestamp_name, -) from exasol.saas.client.openapi.models.exasol_database import ExasolDatabase LOG = logging.getLogger(__name__) @@ -20,15 +14,6 @@ ) -@pytest.fixture -def local_name(project_short_tag: str | None) -> str: - """ - Other than global fixture database_name this fixture uses scope - "function" to generate an individual name for each test case in this file. - """ - return timestamp_name(project_short_tag) - - def test_lifecycle(api_access, local_name): """ This integration test uses the database created and provided by pytest @@ -52,36 +37,16 @@ def get_connection(db: ExasolDatabase): return api_access.get_connection(db.id, clusters[0].id) with api_access.database(local_name, ignore_delete_failure=True) as db: - start = datetime.now() - # verify state and clusters of created database assert db.status in PROMISING_STATES and db.clusters.total == 1 with pytest.raises(RetryError): wait_until_running_too_short(db) - # verify database is listed assert db.id in api_access.list_database_ids() con = get_connection(db) assert con.db_username is not None and con.port == 8563 - # delete database and verify database is not listed anymore api_access.delete_database(db.id) api_access.wait_until_deleted(db.id) assert db.id not in api_access.list_database_ids() - - -def test_create_database_with_two_nodes(api_access, local_name): - """ - This integration test verifies that the handwritten helper forwards - `num_nodes` and the created database reports that setting back. - """ - with api_access.database( - local_name, - ignore_delete_failure=True, - num_nodes=2, - ) as db: - settings = api_access.get_database_settings(db.id) - - assert settings is not None - assert settings.num_nodes == 2 diff --git a/test/integration/test_database_num_nodes.py b/test/integration/test_database_num_nodes.py new file mode 100644 index 0000000..e2f6a45 --- /dev/null +++ b/test/integration/test_database_num_nodes.py @@ -0,0 +1,14 @@ +def test_create_database_with_two_nodes(api_access, local_name): + """ + This integration test verifies that the handwritten helper forwards + `num_nodes` and the created database reports that setting back. + """ + with api_access.database( + local_name, + ignore_delete_failure=True, + num_nodes=2, + ) as db: + settings = api_access.get_database_settings(db.id) + + assert settings is not None + assert settings.num_nodes == 2 From 2ae97daaf18e149c29be792777abc31de39ee77f Mon Sep 17 00:00:00 2001 From: Torsten Kilias Date: Sat, 27 Jun 2026 13:17:06 +0200 Subject: [PATCH 13/22] Resolve allowed IPs from visible list --- exasol/saas/client/api_access.py | 78 +++++++++++++++----------------- test/unit/test_api_access.py | 57 +++++++++++++++-------- 2 files changed, 75 insertions(+), 60 deletions(-) diff --git a/exasol/saas/client/api_access.py b/exasol/saas/client/api_access.py index 808e3bd..838ec7b 100644 --- a/exasol/saas/client/api_access.py +++ b/exasol/saas/client/api_access.py @@ -667,25 +667,13 @@ def wait_until_allowed_ip_listed( ) -> None: @interval_retry(interval, timeout) def verify_listed() -> bool: - resp = get_allowed_ip.sync( - self._account_id, - allowed_ip_id, - client=self._client, - ) - _log_api_output( - "get_allowed_ip.sync", - resp, - account_id=self._account_id, - allowed_ip_id=allowed_ip_id, - ) + visible_allowed_ip_ids = list(self.list_allowed_ip_ids()) LOG.debug( - "wait_until_allowed_ip_listed get result {'allowed_ip_id': %s}: %s", + "wait_until_allowed_ip_listed visible IDs {'allowed_ip_id': %s}: %s", allowed_ip_id, - _serialize_api_output(resp), + visible_allowed_ip_ids, ) - if isinstance(resp, ApiError): - raise TryAgain - if resp is None: + if allowed_ip_id not in visible_allowed_ip_ids: raise TryAgain return True @@ -699,34 +687,13 @@ def wait_until_allowed_ip_deleted( ) -> None: @interval_retry(interval, timeout) def verify_deleted() -> bool: - resp = get_allowed_ip.sync( - self._account_id, - allowed_ip_id, - client=self._client, - ) - _log_api_output( - "get_allowed_ip.sync", - resp, - account_id=self._account_id, - allowed_ip_id=allowed_ip_id, - ) + visible_allowed_ip_ids = list(self.list_allowed_ip_ids()) LOG.debug( - "wait_until_allowed_ip_deleted get result {'allowed_ip_id': %s}: %s", + "wait_until_allowed_ip_deleted visible IDs {'allowed_ip_id': %s}: %s", allowed_ip_id, - _serialize_api_output(resp), + visible_allowed_ip_ids, ) - if isinstance(resp, ApiError): - if resp.status == 404: - return True - raise OpenApiError( - f"Failed to get allowed IP {allowed_ip_id}", - resp, - ) - if resp is None: - raise TryAgain - if resp.deleted_at is not UNSET or resp.deleted_by is not UNSET: - return True - if resp.id == allowed_ip_id: + if allowed_ip_id in visible_allowed_ip_ids: raise TryAgain return True @@ -757,11 +724,38 @@ def add_allowed_ip( account_id=self._account_id, cidr_ip=cidr_ip, ) - return ensure_type( + created_ip = ensure_type( openapi.models.AllowedIP, resp, f"Failed to add allowed IP address {cidr_ip}", ) + return self._resolve_allowed_ip(created_ip) + + def _resolve_allowed_ip( + self, + created_ip: openapi.models.AllowedIP, + timeout: timedelta = timedelta(minutes=1), + interval: timedelta = timedelta(seconds=5), + ) -> openapi.models.AllowedIP: + @interval_retry(interval, timeout) + def resolve() -> openapi.models.AllowedIP: + resp = list_allowed_i_ps.sync(self._account_id, client=self._client) or [] + _log_api_output( + "list_allowed_i_ps.sync", + resp, + account_id=self._account_id, + allowed_ip_name=created_ip.name, + cidr_ip=created_ip.cidr_ip, + ) + ips = ensure_type(list, resp, "Failed to retrieve the list of allowed ips") + for ip in ips: + if ip.deleted_at is not UNSET or ip.deleted_by is not UNSET: + continue + if ip.name == created_ip.name and ip.cidr_ip == created_ip.cidr_ip: + return ip + raise TryAgain + + return resolve() def delete_allowed_ip(self, id: str, ignore_failures=False) -> Any | None: with self._ignore_failures(ignore_failures) as client: diff --git a/test/unit/test_api_access.py b/test/unit/test_api_access.py index f4fd84f..e1c7c42 100644 --- a/test/unit/test_api_access.py +++ b/test/unit/test_api_access.py @@ -319,9 +319,9 @@ def test_log_api_output_serializes_payloads(caplog) -> None: def test_wait_until_allowed_ip_listed_retries(api_mock, monkeypatch) -> None: - get_allowed_ip_mock( + list_allowed_ips_mock( monkeypatch, - [api_error(404, "Item not found"), allowed_ip_response("ip-1")], + [[], [allowed_ip_response("ip-1")]], ) api_mock.wait_until_allowed_ip_listed( @@ -332,12 +332,9 @@ def test_wait_until_allowed_ip_listed_retries(api_mock, monkeypatch) -> None: def test_wait_until_allowed_ip_deleted_retries(api_mock, monkeypatch) -> None: - get_allowed_ip_mock( + list_allowed_ips_mock( monkeypatch, - [ - allowed_ip_response("ip-1"), - api_error(404, "Item not found"), - ], + [[allowed_ip_response("ip-1")], []], ) api_mock.wait_until_allowed_ip_deleted( @@ -351,9 +348,9 @@ def test_wait_until_allowed_ip_listed_logs_visible_ids( api_mock, monkeypatch, caplog ) -> None: caplog.set_level(logging.DEBUG, logger="exasol.saas.client.api_access") - get_allowed_ip_mock( + list_allowed_ips_mock( monkeypatch, - [allowed_ip_response("ip-1")], + [[allowed_ip_response("ip-1")]], ) api_mock.wait_until_allowed_ip_listed( @@ -362,8 +359,8 @@ def test_wait_until_allowed_ip_listed_logs_visible_ids( interval=timedelta(milliseconds=10), ) - assert "wait_until_allowed_ip_listed get result" in caplog.text - assert "get_allowed_ip.sync response" in caplog.text + assert "wait_until_allowed_ip_listed visible IDs" in caplog.text + assert "list_allowed_i_ps.sync response" in caplog.text def test_list_allowed_ip_ids_skips_deleted_entries(api_mock, monkeypatch) -> None: @@ -387,15 +384,17 @@ def test_list_allowed_ip_ids_skips_deleted_entries(api_mock, monkeypatch) -> Non def test_wait_until_allowed_ip_deleted_ignores_soft_deleted_entries( api_mock, monkeypatch ) -> None: - get_allowed_ip_mock( + list_allowed_ips_mock( monkeypatch, [ - allowed_ip_response("ip-1"), - allowed_ip_response( - "ip-1", - deleted_at=datetime(2026, 1, 2), - deleted_by="tester", - ), + [allowed_ip_response("ip-1")], + [ + allowed_ip_response( + "ip-1", + deleted_at=datetime(2026, 1, 2), + deleted_by="tester", + ) + ], ], ) @@ -406,6 +405,28 @@ def test_wait_until_allowed_ip_deleted_ignores_soft_deleted_entries( ) +def test_add_allowed_ip_resolves_visible_rule_from_list(api_mock, monkeypatch) -> None: + from exasol.saas.client.api_access import add_allowed_ip as add_api + + monkeypatch.setattr( + add_api, + "sync", + Mock(return_value=allowed_ip_response("raw-ip")), + ) + list_allowed_ips_mock( + monkeypatch, + [ + [], + [allowed_ip_response("resolved-ip")], + ], + ) + + result = api_mock.add_allowed_ip() + + assert result is not None + assert result.id == "resolved-ip" + + def test_api_error_from_dict_tolerates_missing_fields() -> None: error = ApiError.from_dict( { From 5e27802ecaee9d7c4a2b4ef63386a61465f713a6 Mon Sep 17 00:00:00 2001 From: Torsten Kilias Date: Sat, 27 Jun 2026 13:22:37 +0200 Subject: [PATCH 14/22] Remove unused allowed IP import --- exasol/saas/client/api_access.py | 1 - 1 file changed, 1 deletion(-) diff --git a/exasol/saas/client/api_access.py b/exasol/saas/client/api_access.py index 838ec7b..6b58e66 100644 --- a/exasol/saas/client/api_access.py +++ b/exasol/saas/client/api_access.py @@ -46,7 +46,6 @@ from exasol.saas.client.openapi.api.security import ( add_allowed_ip, delete_allowed_ip, - get_allowed_ip, list_allowed_i_ps, ) from exasol.saas.client.openapi.models import ( From a7e86e985eb387b5247a10e0bf7b007f8c99a606 Mon Sep 17 00:00:00 2001 From: Torsten Kilias Date: Sat, 27 Jun 2026 13:27:06 +0200 Subject: [PATCH 15/22] Remove stale allowed IP test helper --- test/unit/test_api_access.py | 8 -------- 1 file changed, 8 deletions(-) diff --git a/test/unit/test_api_access.py b/test/unit/test_api_access.py index e1c7c42..050b669 100644 --- a/test/unit/test_api_access.py +++ b/test/unit/test_api_access.py @@ -80,14 +80,6 @@ def list_allowed_ips_mock(monkeypatch, side_effect) -> Mock: return mock -def get_allowed_ip_mock(monkeypatch, side_effect) -> Mock: - from exasol.saas.client.api_access import get_allowed_ip as api - - mock = Mock(side_effect=side_effect) - monkeypatch.setattr(api, "sync", mock) - return mock - - def get_database_mock(monkeypatch, side_effect) -> Mock: from exasol.saas.client.api_access import get_database as api From 636429c8b4b92e217f441340e1a9e7c62ab07b18 Mon Sep 17 00:00:00 2001 From: Torsten Kilias Date: Sat, 27 Jun 2026 13:48:27 +0200 Subject: [PATCH 16/22] Use unique CIDR in allowed IP integration test --- test/integration/test_allowed_ip.py | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) diff --git a/test/integration/test_allowed_ip.py b/test/integration/test_allowed_ip.py index 5ab5e9e..773dba0 100644 --- a/test/integration/test_allowed_ip.py +++ b/test/integration/test_allowed_ip.py @@ -1,6 +1,22 @@ +from uuid import uuid4 + + +def _test_only_allowed_ip_cidr() -> str: + # Avoid 0.0.0.0/0 here because the shared SaaS account accumulates old broad + # rules and the backend does not reliably surface a fresh one by name. + documentation_networks = ("192.0.2", "198.51.100", "203.0.113") + token = uuid4().int + network = documentation_networks[token % len(documentation_networks)] + host = ((token >> 8) % 254) + 1 + return f"{network}.{host}/32" + + def test_lifecycle(api_access): testee = api_access - with testee.allowed_ip(ignore_delete_failure=True) as ip: + with testee.allowed_ip( + cidr_ip=_test_only_allowed_ip_cidr(), + ignore_delete_failure=True, + ) as ip: testee.wait_until_allowed_ip_listed(ip.id) # verify allowed ip is listed assert ip.id in testee.list_allowed_ip_ids() From aa06d1b95ccc2e43cae4b6225e084434a7f7656f Mon Sep 17 00:00:00 2001 From: Torsten Kilias Date: Sat, 27 Jun 2026 14:02:39 +0200 Subject: [PATCH 17/22] Use allowed IP get endpoint for polling --- exasol/saas/client/api_access.py | 63 ++++++++++++++++----------- test/integration/test_allowed_ip.py | 8 ++-- test/unit/test_api_access.py | 66 ++++++++++++++++++++--------- 3 files changed, 89 insertions(+), 48 deletions(-) diff --git a/exasol/saas/client/api_access.py b/exasol/saas/client/api_access.py index 6b58e66..bdd9e26 100644 --- a/exasol/saas/client/api_access.py +++ b/exasol/saas/client/api_access.py @@ -46,6 +46,9 @@ from exasol.saas.client.openapi.api.security import ( add_allowed_ip, delete_allowed_ip, +) +from exasol.saas.client.openapi.api.security import get_allowed_ip as get_allowed_ip_api +from exasol.saas.client.openapi.api.security import ( list_allowed_i_ps, ) from exasol.saas.client.openapi.models import ( @@ -658,6 +661,23 @@ def list_allowed_ip_ids(self) -> Iterable[str]: LOG.debug("list_allowed_ip_ids visible IDs: %s", visible_allowed_ip_ids) return iter(visible_allowed_ip_ids) + def get_allowed_ip( + self, + allowed_ip_id: str, + ) -> openapi.models.AllowedIP | ApiError | None: + resp = get_allowed_ip_api.sync( + self._account_id, + allowed_ip_id, + client=self._client, + ) + _log_api_output( + "get_allowed_ip.sync", + resp, + account_id=self._account_id, + allowed_ip_id=allowed_ip_id, + ) + return resp + def wait_until_allowed_ip_listed( self, allowed_ip_id: str, @@ -666,13 +686,12 @@ def wait_until_allowed_ip_listed( ) -> None: @interval_retry(interval, timeout) def verify_listed() -> bool: - visible_allowed_ip_ids = list(self.list_allowed_ip_ids()) LOG.debug( - "wait_until_allowed_ip_listed visible IDs {'allowed_ip_id': %s}: %s", + "wait_until_allowed_ip_listed state {'allowed_ip_id': %s}", allowed_ip_id, - visible_allowed_ip_ids, ) - if allowed_ip_id not in visible_allowed_ip_ids: + allowed_ip = self.get_allowed_ip(allowed_ip_id) + if not self._is_active_allowed_ip(allowed_ip): raise TryAgain return True @@ -686,13 +705,12 @@ def wait_until_allowed_ip_deleted( ) -> None: @interval_retry(interval, timeout) def verify_deleted() -> bool: - visible_allowed_ip_ids = list(self.list_allowed_ip_ids()) LOG.debug( - "wait_until_allowed_ip_deleted visible IDs {'allowed_ip_id': %s}: %s", + "wait_until_allowed_ip_deleted state {'allowed_ip_id': %s}", allowed_ip_id, - visible_allowed_ip_ids, ) - if allowed_ip_id in visible_allowed_ip_ids: + allowed_ip = self.get_allowed_ip(allowed_ip_id) + if self._is_active_allowed_ip(allowed_ip): raise TryAgain return True @@ -728,34 +746,31 @@ def add_allowed_ip( resp, f"Failed to add allowed IP address {cidr_ip}", ) - return self._resolve_allowed_ip(created_ip) + return self._resolve_allowed_ip(created_ip.id) def _resolve_allowed_ip( self, - created_ip: openapi.models.AllowedIP, + allowed_ip_id: str, timeout: timedelta = timedelta(minutes=1), interval: timedelta = timedelta(seconds=5), ) -> openapi.models.AllowedIP: @interval_retry(interval, timeout) def resolve() -> openapi.models.AllowedIP: - resp = list_allowed_i_ps.sync(self._account_id, client=self._client) or [] - _log_api_output( - "list_allowed_i_ps.sync", - resp, - account_id=self._account_id, - allowed_ip_name=created_ip.name, - cidr_ip=created_ip.cidr_ip, - ) - ips = ensure_type(list, resp, "Failed to retrieve the list of allowed ips") - for ip in ips: - if ip.deleted_at is not UNSET or ip.deleted_by is not UNSET: - continue - if ip.name == created_ip.name and ip.cidr_ip == created_ip.cidr_ip: - return ip + allowed_ip = self.get_allowed_ip(allowed_ip_id) + if self._is_active_allowed_ip(allowed_ip): + return cast(openapi.models.AllowedIP, allowed_ip) raise TryAgain return resolve() + @staticmethod + def _is_active_allowed_ip( + allowed_ip: openapi.models.AllowedIP | ApiError | None, + ) -> bool: + if allowed_ip is None or isinstance(allowed_ip, ApiError): + return False + return allowed_ip.deleted_at is UNSET and allowed_ip.deleted_by is UNSET + def delete_allowed_ip(self, id: str, ignore_failures=False) -> Any | None: with self._ignore_failures(ignore_failures) as client: resp = delete_allowed_ip.sync(self._account_id, id, client=client) diff --git a/test/integration/test_allowed_ip.py b/test/integration/test_allowed_ip.py index 773dba0..47d9d40 100644 --- a/test/integration/test_allowed_ip.py +++ b/test/integration/test_allowed_ip.py @@ -1,5 +1,7 @@ from uuid import uuid4 +from exasol.saas.client.openapi.models.api_error import ApiError + def _test_only_allowed_ip_cidr() -> str: # Avoid 0.0.0.0/0 here because the shared SaaS account accumulates old broad @@ -18,10 +20,10 @@ def test_lifecycle(api_access): ignore_delete_failure=True, ) as ip: testee.wait_until_allowed_ip_listed(ip.id) - # verify allowed ip is listed - assert ip.id in testee.list_allowed_ip_ids() + assert testee.get_allowed_ip(ip.id) is not None # delete allowed ip and verify it is not listed anymore testee.delete_allowed_ip(ip.id) testee.wait_until_allowed_ip_deleted(ip.id) - assert ip.id not in testee.list_allowed_ip_ids() + deleted_ip = testee.get_allowed_ip(ip.id) + assert deleted_ip is None or isinstance(deleted_ip, ApiError) diff --git a/test/unit/test_api_access.py b/test/unit/test_api_access.py index 050b669..3e5e527 100644 --- a/test/unit/test_api_access.py +++ b/test/unit/test_api_access.py @@ -80,6 +80,14 @@ def list_allowed_ips_mock(monkeypatch, side_effect) -> Mock: return mock +def get_allowed_ip_mock(monkeypatch, side_effect) -> Mock: + from exasol.saas.client.api_access import get_allowed_ip_api as api + + mock = Mock(side_effect=side_effect) + monkeypatch.setattr(api, "sync", mock) + return mock + + def get_database_mock(monkeypatch, side_effect) -> Mock: from exasol.saas.client.api_access import get_database as api @@ -311,9 +319,9 @@ def test_log_api_output_serializes_payloads(caplog) -> None: def test_wait_until_allowed_ip_listed_retries(api_mock, monkeypatch) -> None: - list_allowed_ips_mock( + get_allowed_ip_mock( monkeypatch, - [[], [allowed_ip_response("ip-1")]], + [None, allowed_ip_response("ip-1")], ) api_mock.wait_until_allowed_ip_listed( @@ -324,9 +332,9 @@ def test_wait_until_allowed_ip_listed_retries(api_mock, monkeypatch) -> None: def test_wait_until_allowed_ip_deleted_retries(api_mock, monkeypatch) -> None: - list_allowed_ips_mock( + get_allowed_ip_mock( monkeypatch, - [[allowed_ip_response("ip-1")], []], + [allowed_ip_response("ip-1"), None], ) api_mock.wait_until_allowed_ip_deleted( @@ -340,9 +348,9 @@ def test_wait_until_allowed_ip_listed_logs_visible_ids( api_mock, monkeypatch, caplog ) -> None: caplog.set_level(logging.DEBUG, logger="exasol.saas.client.api_access") - list_allowed_ips_mock( + get_allowed_ip_mock( monkeypatch, - [[allowed_ip_response("ip-1")]], + [allowed_ip_response("ip-1")], ) api_mock.wait_until_allowed_ip_listed( @@ -351,8 +359,8 @@ def test_wait_until_allowed_ip_listed_logs_visible_ids( interval=timedelta(milliseconds=10), ) - assert "wait_until_allowed_ip_listed visible IDs" in caplog.text - assert "list_allowed_i_ps.sync response" in caplog.text + assert "wait_until_allowed_ip_listed state" in caplog.text + assert "get_allowed_ip.sync response" in caplog.text def test_list_allowed_ip_ids_skips_deleted_entries(api_mock, monkeypatch) -> None: @@ -376,17 +384,15 @@ def test_list_allowed_ip_ids_skips_deleted_entries(api_mock, monkeypatch) -> Non def test_wait_until_allowed_ip_deleted_ignores_soft_deleted_entries( api_mock, monkeypatch ) -> None: - list_allowed_ips_mock( + get_allowed_ip_mock( monkeypatch, [ - [allowed_ip_response("ip-1")], - [ - allowed_ip_response( - "ip-1", - deleted_at=datetime(2026, 1, 2), - deleted_by="tester", - ) - ], + allowed_ip_response("ip-1"), + allowed_ip_response( + "ip-1", + deleted_at=datetime(2026, 1, 2), + deleted_by="tester", + ), ], ) @@ -397,7 +403,7 @@ def test_wait_until_allowed_ip_deleted_ignores_soft_deleted_entries( ) -def test_add_allowed_ip_resolves_visible_rule_from_list(api_mock, monkeypatch) -> None: +def test_add_allowed_ip_resolves_visible_rule_by_id(api_mock, monkeypatch) -> None: from exasol.saas.client.api_access import add_allowed_ip as add_api monkeypatch.setattr( @@ -405,11 +411,11 @@ def test_add_allowed_ip_resolves_visible_rule_from_list(api_mock, monkeypatch) - "sync", Mock(return_value=allowed_ip_response("raw-ip")), ) - list_allowed_ips_mock( + get_allowed_ip_mock( monkeypatch, [ - [], - [allowed_ip_response("resolved-ip")], + None, + allowed_ip_response("resolved-ip"), ], ) @@ -419,6 +425,24 @@ def test_add_allowed_ip_resolves_visible_rule_from_list(api_mock, monkeypatch) - assert result.id == "resolved-ip" +def test_wait_until_allowed_ip_deleted_treats_api_error_as_deleted( + api_mock, monkeypatch +) -> None: + get_allowed_ip_mock( + monkeypatch, + [ + allowed_ip_response("ip-1"), + ApiError.from_dict({"status": 404, "message": "not found"}), + ], + ) + + api_mock.wait_until_allowed_ip_deleted( + "ip-1", + timeout=timedelta(seconds=1), + interval=timedelta(milliseconds=10), + ) + + def test_api_error_from_dict_tolerates_missing_fields() -> None: error = ApiError.from_dict( { From 77acb98a9bc023e894e7c732c36faff764ab6fe1 Mon Sep 17 00:00:00 2001 From: Torsten Kilias Date: Sat, 27 Jun 2026 14:09:36 +0200 Subject: [PATCH 18/22] Increase allowed IP polling timeouts --- exasol/saas/client/api_access.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/exasol/saas/client/api_access.py b/exasol/saas/client/api_access.py index bdd9e26..4d09770 100644 --- a/exasol/saas/client/api_access.py +++ b/exasol/saas/client/api_access.py @@ -681,7 +681,7 @@ def get_allowed_ip( def wait_until_allowed_ip_listed( self, allowed_ip_id: str, - timeout: timedelta = timedelta(minutes=1), + timeout: timedelta = timedelta(minutes=20), interval: timedelta = timedelta(seconds=5), ) -> None: @interval_retry(interval, timeout) @@ -751,7 +751,7 @@ def add_allowed_ip( def _resolve_allowed_ip( self, allowed_ip_id: str, - timeout: timedelta = timedelta(minutes=1), + timeout: timedelta = timedelta(minutes=20), interval: timedelta = timedelta(seconds=5), ) -> openapi.models.AllowedIP: @interval_retry(interval, timeout) From 836e09753030ad821629f918ddb3560a0ddf45bd Mon Sep 17 00:00:00 2001 From: Torsten Kilias Date: Sat, 27 Jun 2026 15:51:34 +0200 Subject: [PATCH 19/22] Revert "Support configurable database node counts" This reverts commit 7fffb5a721e353718dcd672e76c95bd01568a065. --- doc/changes/unreleased.md | 2 -- exasol/saas/client/api_access.py | 6 ----- test/unit/test_api_access.py | 46 -------------------------------- 3 files changed, 54 deletions(-) diff --git a/doc/changes/unreleased.md b/doc/changes/unreleased.md index 2c86bd8..332dfd9 100644 --- a/doc/changes/unreleased.md +++ b/doc/changes/unreleased.md @@ -15,8 +15,6 @@ Besides there are many formal and syntactic changes ## Summary -* #147: Added `num_nodes` support to the handwritten database creation helpers. - ## Refactorings * #160: Updated PTB to version 7.0.0 diff --git a/exasol/saas/client/api_access.py b/exasol/saas/client/api_access.py index 4d09770..e9da7e4 100644 --- a/exasol/saas/client/api_access.py +++ b/exasol/saas/client/api_access.py @@ -311,7 +311,6 @@ def create_database( cluster_size: str = "XS", region: str = "eu-central-1", idle_time: timedelta | None = None, - num_nodes: int | None = None, ) -> ExasolDatabase | None: def minutes(x: timedelta) -> int: return x.seconds // 60 @@ -333,9 +332,6 @@ def minutes(x: timedelta) -> int: region=region, stream_type="innovation-release", ) - if num_nodes is not None: - database_spec.num_nodes = num_nodes - resp = create_database.sync( self._account_id, client=self._client, @@ -505,14 +501,12 @@ def database( keep: bool = False, ignore_delete_failure: bool = False, idle_time: timedelta | None = None, - num_nodes: int | None = None, ): db = None try: db = self.create_database( name, idle_time=idle_time, - num_nodes=num_nodes, ) yield db finally: diff --git a/test/unit/test_api_access.py b/test/unit/test_api_access.py index 3e5e527..8656448 100644 --- a/test/unit/test_api_access.py +++ b/test/unit/test_api_access.py @@ -56,14 +56,6 @@ def delete_mock(monkeypatch, side_effect) -> Mock: return mock -def create_database_mock(monkeypatch, side_effect) -> Mock: - from exasol.saas.client.api_access import create_database as api - - mock = Mock(side_effect=side_effect) - monkeypatch.setattr(api, "sync", mock) - return mock - - def get_database_settings_mock(monkeypatch, side_effect) -> Mock: from exasol.saas.client.api_access import get_database_settings as api @@ -223,44 +215,6 @@ def test_timestamp_name() -> None: assert all(tag == "TEST" for tag in tags) -@pytest.mark.parametrize( - "num_nodes, expected_num_nodes", - [ - pytest.param(None, UNSET, id="uses_backend_default"), - pytest.param(2, 2, id="forwards_explicit_value"), - ], -) -def test_create_database_num_nodes( - api_mock, monkeypatch, num_nodes, expected_num_nodes -) -> None: - create = create_database_mock( - monkeypatch, - [database_response("db-with-nodes")], - ) - - result = api_mock.create_database("db-with-nodes", num_nodes=num_nodes) - - assert result is not None - assert create.called - body = create.call_args.kwargs["body"] - assert body.num_nodes == expected_num_nodes - - -def test_database_context_forwards_num_nodes(api_mock, monkeypatch) -> None: - create = Mock(return_value=database_response("db-with-context")) - delete = Mock() - monkeypatch.setattr(api_mock, "create_database", create) - monkeypatch.setattr(api_mock, "delete_database", delete) - - with api_mock.database("db-with-context", num_nodes=2) as db: - assert db is not None - assert db.name == "db-with-context" - - assert create.call_args.args == ("db-with-context",) - assert create.call_args.kwargs == {"idle_time": None, "num_nodes": 2} - delete.assert_called_once_with("db-id", False) - - def test_get_database_settings_retries_transient_not_found( api_mock, monkeypatch ) -> None: From ef6725d8d317799032c6db2577c85a8de5808e04 Mon Sep 17 00:00:00 2001 From: Torsten Kilias Date: Sat, 27 Jun 2026 15:52:40 +0200 Subject: [PATCH 20/22] Revert "Format API access tests" This reverts commit a8c2d739c79c0ea953147f9f3d1bbba243d6ecdf. --- .codex | 0 .github/workflows/matrix-slow.yml | 37 ------------------- .github/workflows/slow-checks.yml | 19 ++-------- noxfile.py | 19 ---------- test/integration/conftest.py | 9 ----- test/integration/test_database_num_nodes.py | 14 ------- ...atabase_lifecycle.py => test_databases.py} | 37 ++++++++++++++++++- 7 files changed, 39 insertions(+), 96 deletions(-) create mode 100644 .codex delete mode 100644 .github/workflows/matrix-slow.yml delete mode 100644 test/integration/test_database_num_nodes.py rename test/integration/{test_database_lifecycle.py => test_databases.py} (60%) diff --git a/.codex b/.codex new file mode 100644 index 0000000..e69de29 diff --git a/.github/workflows/matrix-slow.yml b/.github/workflows/matrix-slow.yml deleted file mode 100644 index 734f9b1..0000000 --- a/.github/workflows/matrix-slow.yml +++ /dev/null @@ -1,37 +0,0 @@ -name: Build Matrix (Slow Tests) - -on: - workflow_call: - outputs: - matrix: - description: "Generates the slow test build matrix" - value: ${{ jobs.set-matrix-slow.outputs.matrix }} - -jobs: - set-matrix-slow: - runs-on: "ubuntu-24.04" - permissions: - contents: read - steps: - - name: Check out Repository - id: check-out-repository - uses: actions/checkout@v6 - - - name: Set up Python & Poetry Environment - id: set-up-python-and-poetry-environment - uses: exasol/python-toolbox/.github/actions/python-environment@v7 - with: - python-version: "3.10" - poetry-version: "2.3.0" - - - name: Generate Matrix - id: generate-matrix - run: poetry run -- nox -s matrix:slow - - - name: Set Matrix - id: set-matrix - run: | - echo "matrix=$(poetry run -- nox -s matrix:slow)" >> $GITHUB_OUTPUT - - outputs: - matrix: ${{ steps.set-matrix.outputs.matrix }} diff --git a/.github/workflows/slow-checks.yml b/.github/workflows/slow-checks.yml index ff33e3e..70a744c 100644 --- a/.github/workflows/slow-checks.yml +++ b/.github/workflows/slow-checks.yml @@ -4,22 +4,12 @@ on: workflow_call: jobs: - build-slow-matrix: - name: Build Slow Matrix - uses: ./.github/workflows/matrix-slow.yml - permissions: - contents: read run-integration-tests: - name: Run Integration Tests (${{ matrix.test-name }}) - needs: - - build-slow-matrix + name: Run Integration Tests runs-on: "ubuntu-24.04" permissions: contents: read - strategy: - fail-fast: false - matrix: ${{ fromJSON(needs.build-slow-matrix.outputs.matrix) }} env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} steps: @@ -36,10 +26,7 @@ jobs: - name: Run Integration Tests id: run-integration-tests - run: | - poetry run coverage run \ - --rcfile=pyproject.toml \ - -m pytest -v "${{ matrix.test-path }}" + run: poetry run -- nox -s test:integration -- --coverage env: SAAS_HOST: ${{ secrets.INTEGRATION_TEAM_SAAS_STAGING_HOST }} SAAS_ACCOUNT_ID: ${{ secrets.INTEGRATION_TEAM_SAAS_STAGING_ACCOUNT_ID }} @@ -51,6 +38,6 @@ jobs: id: upload-artifacts uses: actions/upload-artifact@v7 with: - name: coverage-python3.10-slow-${{ matrix.test-name }} + name: coverage-python3.10-slow path: .coverage include-hidden-files: true diff --git a/noxfile.py b/noxfile.py index da0fa24..17822bb 100644 --- a/noxfile.py +++ b/noxfile.py @@ -26,19 +26,6 @@ DEST_DIR = "exasol/saas/client/openapi" -def _slow_test_matrix() -> dict[str, list[dict[str, str]]]: - files = sorted(Path("test/integration").glob("test_*.py")) - return { - "include": [ - { - "test-name": path.stem.removeprefix("test_").replace("_", "-"), - "test-path": str(path), - } - for path in files - ] - } - - def _download_openapi_json() -> Path: url = f"{SAAS_HOST}/openapi.json" response = requests.get(url) @@ -140,9 +127,3 @@ def check_api_outdated(session: Session): """ generate_api(session) session.run("git", "diff", "--exit-code", DEST_DIR) - - -@nox.session(name="matrix:slow", python=False) -def slow_matrix(session: Session): - """Output the build matrix for slow integration tests as JSON.""" - print(json.dumps(_slow_test_matrix())) diff --git a/test/integration/conftest.py b/test/integration/conftest.py index 37912e5..272ba1b 100644 --- a/test/integration/conftest.py +++ b/test/integration/conftest.py @@ -67,15 +67,6 @@ def database_name(project_short_tag) -> str: return timestamp_name(project_short_tag) -@pytest.fixture -def local_name(project_short_tag: str | None) -> str: - """ - Other than global fixture database_name this fixture uses scope - "function" to generate an individual name for each test case. - """ - return timestamp_name(project_short_tag) - - @pytest.fixture(scope="session") def allow_connection(api_access) -> None: """ diff --git a/test/integration/test_database_num_nodes.py b/test/integration/test_database_num_nodes.py deleted file mode 100644 index e2f6a45..0000000 --- a/test/integration/test_database_num_nodes.py +++ /dev/null @@ -1,14 +0,0 @@ -def test_create_database_with_two_nodes(api_access, local_name): - """ - This integration test verifies that the handwritten helper forwards - `num_nodes` and the created database reports that setting back. - """ - with api_access.database( - local_name, - ignore_delete_failure=True, - num_nodes=2, - ) as db: - settings = api_access.get_database_settings(db.id) - - assert settings is not None - assert settings.num_nodes == 2 diff --git a/test/integration/test_database_lifecycle.py b/test/integration/test_databases.py similarity index 60% rename from test/integration/test_database_lifecycle.py rename to test/integration/test_databases.py index 95c3b3b..ba24432 100644 --- a/test/integration/test_database_lifecycle.py +++ b/test/integration/test_databases.py @@ -1,10 +1,16 @@ import logging -from datetime import timedelta +from datetime import ( + datetime, + timedelta, +) import pytest from tenacity import RetryError from exasol.saas.client import PROMISING_STATES +from exasol.saas.client.api_access import ( + timestamp_name, +) from exasol.saas.client.openapi.models.exasol_database import ExasolDatabase LOG = logging.getLogger(__name__) @@ -14,6 +20,15 @@ ) +@pytest.fixture +def local_name(project_short_tag: str | None) -> str: + """ + Other than global fixture database_name this fixture uses scope + "function" to generate an individual name for each test case in this file. + """ + return timestamp_name(project_short_tag) + + def test_lifecycle(api_access, local_name): """ This integration test uses the database created and provided by pytest @@ -37,16 +52,36 @@ def get_connection(db: ExasolDatabase): return api_access.get_connection(db.id, clusters[0].id) with api_access.database(local_name, ignore_delete_failure=True) as db: + start = datetime.now() + # verify state and clusters of created database assert db.status in PROMISING_STATES and db.clusters.total == 1 with pytest.raises(RetryError): wait_until_running_too_short(db) + # verify database is listed assert db.id in api_access.list_database_ids() con = get_connection(db) assert con.db_username is not None and con.port == 8563 + # delete database and verify database is not listed anymore api_access.delete_database(db.id) api_access.wait_until_deleted(db.id) assert db.id not in api_access.list_database_ids() + + +def test_create_database_with_two_nodes(api_access, local_name): + """ + This integration test verifies that the handwritten helper forwards + `num_nodes` and the created database reports that setting back. + """ + with api_access.database( + local_name, + ignore_delete_failure=True, + num_nodes=2, + ) as db: + settings = api_access.get_database_settings(db.id) + + assert settings is not None + assert settings.num_nodes == 2 From e956bb8e18d34ec87780c72a6b6e5b8deabd0792 Mon Sep 17 00:00:00 2001 From: Torsten Kilias Date: Sat, 27 Jun 2026 15:52:56 +0200 Subject: [PATCH 21/22] Remove accidental .codex artifact --- .codex | 0 1 file changed, 0 insertions(+), 0 deletions(-) delete mode 100644 .codex diff --git a/.codex b/.codex deleted file mode 100644 index e69de29..0000000 From 845215da62865d521709251149ad65790a5554cf Mon Sep 17 00:00:00 2001 From: Torsten Kilias Date: Sat, 27 Jun 2026 17:16:38 +0200 Subject: [PATCH 22/22] Remove num_nodes leak from PR1 --- exasol/saas/client/api_access.py | 34 ---------------------- test/integration/test_databases.py | 16 ----------- test/unit/test_api_access.py | 46 ------------------------------ 3 files changed, 96 deletions(-) diff --git a/exasol/saas/client/api_access.py b/exasol/saas/client/api_access.py index e9da7e4..21312be 100644 --- a/exasol/saas/client/api_access.py +++ b/exasol/saas/client/api_access.py @@ -40,7 +40,6 @@ create_database, delete_database, get_database, - get_database_settings, list_databases, ) from exasol.saas.client.openapi.api.security import ( @@ -538,39 +537,6 @@ def get_database( ExasolDatabase, resp, f"Failed to get database {database_id}" ) - def get_database_settings( - self, - database_id: str, - ) -> openapi.models.DatabaseSettings | None: - def is_retry(resp: ApiError) -> bool: - return _is_not_found(resp) - - @interval_retry( - interval=timedelta(seconds=5), - timeout=timedelta(minutes=2), - ) - def retrieve_settings() -> openapi.models.DatabaseSettings: - resp = get_database_settings.sync( - self._account_id, - database_id, - client=self._client, - ) - _log_api_output( - "get_database_settings.sync", - resp, - account_id=self._account_id, - database_id=database_id, - ) - if isinstance(resp, ApiError) and is_retry(resp): - raise TryAgain - return ensure_type( - openapi.models.DatabaseSettings, - resp, - f"Failed to get settings of database {database_id}", - ) - - return retrieve_settings() - def wait_until_running( self, database_id: str, diff --git a/test/integration/test_databases.py b/test/integration/test_databases.py index ba24432..f338799 100644 --- a/test/integration/test_databases.py +++ b/test/integration/test_databases.py @@ -69,19 +69,3 @@ def get_connection(db: ExasolDatabase): api_access.delete_database(db.id) api_access.wait_until_deleted(db.id) assert db.id not in api_access.list_database_ids() - - -def test_create_database_with_two_nodes(api_access, local_name): - """ - This integration test verifies that the handwritten helper forwards - `num_nodes` and the created database reports that setting back. - """ - with api_access.database( - local_name, - ignore_delete_failure=True, - num_nodes=2, - ) as db: - settings = api_access.get_database_settings(db.id) - - assert settings is not None - assert settings.num_nodes == 2 diff --git a/test/unit/test_api_access.py b/test/unit/test_api_access.py index 8656448..631c810 100644 --- a/test/unit/test_api_access.py +++ b/test/unit/test_api_access.py @@ -56,14 +56,6 @@ def delete_mock(monkeypatch, side_effect) -> Mock: return mock -def get_database_settings_mock(monkeypatch, side_effect) -> Mock: - from exasol.saas.client.api_access import get_database_settings as api - - mock = Mock(side_effect=side_effect) - monkeypatch.setattr(api, "sync", mock) - return mock - - def list_allowed_ips_mock(monkeypatch, side_effect) -> Mock: from exasol.saas.client.api_access import list_allowed_i_ps as api @@ -215,44 +207,6 @@ def test_timestamp_name() -> None: assert all(tag == "TEST" for tag in tags) -def test_get_database_settings_retries_transient_not_found( - api_mock, monkeypatch -) -> None: - monkeypatch.setattr( - "exasol.saas.client.api_access.interval_retry", - immediate_retry, - ) - get_settings = get_database_settings_mock( - monkeypatch, - [ - api_error(404, "User/Database not found"), - database_settings_response(), - ], - ) - - result = api_mock.get_database_settings("db-id") - - assert result is not None - assert result.num_nodes == 2 - assert get_settings.call_count == 2 - - -def test_get_database_settings_raises_non_retryable_error( - api_mock, monkeypatch -) -> None: - monkeypatch.setattr( - "exasol.saas.client.api_access.interval_retry", - lambda *_args, **_kwargs: (lambda func: func), - ) - get_database_settings_mock( - monkeypatch, - [api_error(500, "boom")], - ) - - with pytest.raises(OpenApiError, match="Failed to get settings of database db-id"): - api_mock.get_database_settings("db-id") - - def test_log_api_output_serializes_payloads(caplog) -> None: caplog.set_level(logging.DEBUG, logger="exasol.saas.client.api_access")