From 519ee86a7c800468b84e1c114ba4ba1c0e4ceee2 Mon Sep 17 00:00:00 2001 From: Simon Schrottner Date: Mon, 24 Aug 2026 13:57:14 +0200 Subject: [PATCH 01/29] feat(flagd): run the provider conformance suite against both resolvers Adopts the OpenFeature provider conformance suite in the flagd provider, for both resolvers, as two separate suites. They are separate because they are separately conformant. flagd resolves flags two quite different ways -- RPC evaluates remotely over gRPC, in-process syncs the ruleset and evaluates locally -- and any difference between the two results is a difference an application would see when it switches resolver, which is exactly the class of thing the suite exists to surface. flagd-testbed is not modified and the existing e2e suites are untouched. The TCK drives the testbed's launchpad through the standardised control API, which the launchpad already implements, and reuses the container lifecycle already in tests/e2e. The stack is started once per session and never restarted; scenario isolation comes from the control API instead, because container orchestrators assign host ports dynamically and cannot reliably preserve them across a restart, and a restarted backend on a new port looks like a flaky provider rather than a broken test. One stack and one HttpControl serve both suites. One flagd process serves both resolver ports, so there is nothing a second stack would isolate -- and the control has to be shared, because it tracks whether a disconnect has left the backend down so the next scenario starts it rather than merely resetting flag state. Two instances would each hold half of that knowledge. Every capability is declared on the strength of a line of provider code rather than of a green run, and each declaration carries its file and line. Both resolvers get the same set: EVENTS, STALE, CONFIGURATION_CHANGE, OBJECT, UNAVAILABLE_INIT and STRICT_NUMERIC_TYPING. TARGETING and CACHING are withheld from both, because no scenario carries either tag and a capability nothing exercises would be a claim with no evidence behind it. Worth recording that STALE is declared for RPC. The Go provider's RPC resolver never emits PROVIDER_STALE -- it sends ProviderError directly on connection loss (go-sdk-contrib#939) -- and the Go adoption withholds the capability for that reason. Python has no such asymmetry: both resolvers emit PROVIDER_STALE from the same channel-connectivity callback shape on TRANSIENT_FAILURE, and only escalate to PROVIDER_ERROR once the retry grace period expires. That is the behaviour the specification describes, and it is the reason the grace period is set well above the length of the scenario's outage: too short a value turns a scenario about staleness into one about failure. Signed-off-by: Simon Schrottner --- .../openfeature-provider-flagd/pyproject.toml | 5 + .../tests/tck/__init__.py | 0 .../tests/tck/conftest.py | 73 ++++++++ .../tests/tck/suite.py | 167 ++++++++++++++++++ .../tests/tck/test_in_process_conformance.py | 96 ++++++++++ .../tests/tck/test_rpc_conformance.py | 97 ++++++++++ 6 files changed, 438 insertions(+) create mode 100644 providers/openfeature-provider-flagd/tests/tck/__init__.py create mode 100644 providers/openfeature-provider-flagd/tests/tck/conftest.py create mode 100644 providers/openfeature-provider-flagd/tests/tck/suite.py create mode 100644 providers/openfeature-provider-flagd/tests/tck/test_in_process_conformance.py create mode 100644 providers/openfeature-provider-flagd/tests/tck/test_rpc_conformance.py diff --git a/providers/openfeature-provider-flagd/pyproject.toml b/providers/openfeature-provider-flagd/pyproject.toml index 2c1c1b3a..7cb0492d 100644 --- a/providers/openfeature-provider-flagd/pyproject.toml +++ b/providers/openfeature-provider-flagd/pyproject.toml @@ -35,6 +35,10 @@ dev = [ "coverage[toml]>=7.10.0,<8.0.0", "grpcio-health-checking>=1.82.1,<2.0.0", "mypy>=1.18.0,<2.0.0", + # The OpenFeature provider conformance suite. Ships the feature files, the flag + # set and the control-API client, and registers its step definitions through a + # pytest11 entry point, so tests/tck needs no conftest of its own for them. + "openfeature-provider-tck", "poethepoet>=0.37.0", "pytest>=9.0.0,<10.0.0", "pytest-bdd>=8.1.0,<9.0.0", @@ -112,6 +116,7 @@ warn_unused_ignores = false [tool.uv.sources] openfeature-flagd-core = { workspace = true } +openfeature-provider-tck = { workspace = true } [tool.pytest] strict = true diff --git a/providers/openfeature-provider-flagd/tests/tck/__init__.py b/providers/openfeature-provider-flagd/tests/tck/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/providers/openfeature-provider-flagd/tests/tck/conftest.py b/providers/openfeature-provider-flagd/tests/tck/conftest.py new file mode 100644 index 00000000..b67feaaa --- /dev/null +++ b/providers/openfeature-provider-flagd/tests/tck/conftest.py @@ -0,0 +1,73 @@ +"""One testbed stack and one control, shared by both conformance suites. + +The stack is started once per session and **never restarted**. Scenario +isolation comes from the control API instead, because container orchestrators +assign host ports dynamically and cannot reliably preserve them across a +restart: a restarted backend comes back on a different host port, silently +invalidating every provider already pointed at the old one, and the failure +looks like a flaky provider rather than a broken test. See the +no-container-restart invariant in the TCK's ``control-api.yaml``. + +``flagd-testbed`` is not modified and the existing e2e suites are untouched: the +TCK drives the testbed's launchpad through the standardised control API, which +the launchpad already implements, and reuses the container lifecycle already in +``tests/e2e``. +""" + +from __future__ import annotations + +import socket +import typing + +import pytest + +from openfeature.contrib.tools.provider_tck import HttpControl +from tests.e2e.flagd_container import FlagdContainer + + +@pytest.fixture(scope="session") +def flagd_testbed() -> typing.Iterator[FlagdContainer]: + """The testbed stack, up for the whole session. + + One stack for both suites because one flagd process serves both ports the + resolvers use -- 8013 for RPC and 8015 for sync -- so there is nothing a + second stack would isolate. + """ + container = FlagdContainer() + container.start() + try: + yield container + finally: + container.stop() + + +@pytest.fixture(scope="session") +def flagd_control(flagd_testbed: FlagdContainer) -> HttpControl: + """The control API client, shared by both suites. + + Shared rather than one per suite, and that matters: the two suites drive the + *same* backend, and ``HttpControl`` tracks whether a disconnect has left it + down so the next scenario starts it rather than merely resetting flag state. + Two instances would each hold half of that knowledge. + + The launchpad registers only ``/start``, ``/restart``, ``/stop`` and + ``/change`` (flagd-testbed ``launchpad/main.go``), so ``/reset`` answers 404 + and every ``prepare_scenario`` takes the documented ``/start`` fallback. The + probe costs one 404 for the whole session. + """ + return HttpControl(flagd_testbed.get_launchpad_url()) + + +@pytest.fixture(scope="session") +def closed_port(flagd_testbed: FlagdContainer) -> int: + """A port on localhost with nothing listening, for the ``@unavailable`` scenarios. + + Discovered by binding and releasing rather than hard-coded, because the + testbed's own host ports are mapped dynamically and a hard-coded number + could collide with one. Depending on ``flagd_testbed`` orders this after the + stack has taken its ports, which is what makes the remaining race + negligible. + """ + with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as probe: + probe.bind(("127.0.0.1", 0)) + return int(probe.getsockname()[1]) diff --git a/providers/openfeature-provider-flagd/tests/tck/suite.py b/providers/openfeature-provider-flagd/tests/tck/suite.py new file mode 100644 index 00000000..8a3c3e35 --- /dev/null +++ b/providers/openfeature-provider-flagd/tests/tck/suite.py @@ -0,0 +1,167 @@ +"""Shared wiring for the two flagd conformance suites. + +flagd resolves flags two quite different ways -- RPC evaluates remotely over +gRPC, in-process syncs the ruleset and evaluates locally -- and they are separate +suites because they are separately conformant. Any difference between the two +results is a difference an application would see when it switches resolver, +which is exactly the class of thing the conformance suite exists to surface. + +Everything they share lives here; everything that differs lives in the two +``test_*_conformance`` modules next to it, where a reader can see the whole of a +resolver's declaration in one place. +""" + +from __future__ import annotations + +from dataclasses import dataclass + +from openfeature.contrib.provider.flagd import FlagdProvider +from openfeature.contrib.provider.flagd.config import ResolverType +from openfeature.contrib.tools.provider_tck import ( + Capability, + HttpControl, + TckConfig, +) +from openfeature.provider import FeatureProvider +from tests.e2e.flagd_container import FlagdContainer + +__all__ = ["ResolverSuite", "build_config"] + +# Timings. flagd exposes several and they interact, so they are named here once +# rather than scattered through two suites. + +DEADLINE_MS = 5000 +"""How long a provider blocks during initialisation before giving up. + +This, and not ``TckConfig.ready_timeout``, is what actually bounds flagd's +initialisation: the resolvers block inside ``initialize`` and raise +``ProviderNotReadyError`` when it expires (grpc.py:175, grpc_watcher.py:151). +Generous because every scenario is preceded by a control-API ``/start``, which +restarts the flagd process -- so a provider is routinely built against a backend +that came up milliseconds ago. + +For the RPC resolver it also bounds each individual resolution call. +""" + +RETRY_BACKOFF_MS = 500 +"""Initial delay before a reconnect attempt. Short, so an ended outage is noticed quickly.""" + +RETRY_BACKOFF_MAX_MS = 5000 +"""Longest delay between reconnect attempts, and the wait between stream retries. + +Kept at or above :data:`DEADLINE_MS`: the provider passes the deadline to gRPC as +``grpc.min_reconnect_backoff_ms`` and this as ``grpc.max_reconnect_backoff_ms`` +(grpc.py:90-92), so a smaller value here would be a channel configured with a +minimum backoff above its maximum. +""" + +RETRY_GRACE_PERIOD_SECONDS = 30 +"""How long a disconnected provider stays STALE before escalating to ERROR. + +Load-bearing for the ``@stale`` scenario. Both resolvers go STALE the moment the +channel fails and start a timer that escalates to ERROR when this expires +(grpc.py:204-210, grpc_watcher.py:180-186). Too short a value turns a scenario +about staleness into one about failure, because the outage lasts as long as the +scenario needs to observe it. +""" + +STREAM_DEADLINE_MS = 0 +"""Disable periodic stream recycling, as the existing e2e suites do. + +A recycle is invisible to the provider contract, but disabling it removes a +source of background reconnects from a suite whose whole subject is what a +reconnect looks like. +""" + +UNAVAILABLE_DEADLINE_MS = 500 +UNAVAILABLE_GRACE_PERIOD_SECONDS = 1 +UNAVAILABLE_BACKOFF_MS = 5000 +"""Deliberately impatient settings for the provider that cannot reach a backend. + +The ``@unavailable`` scenarios assert that failure is reported *promptly*, so a +provider taking 30 seconds to give up would pass a test about eventual failure +while failing the one that matters. The backoff is long for the opposite reason: +after the error is reported, the next failed reconnect attempt would emit +another ``PROVIDER_STALE`` and move the provider out of the ERROR state the +scenario is about to assert. +""" + +EVENT_TIMEOUT = 20.0 +"""Seconds to wait for a provider event. + +Comfortably above :data:`RETRY_BACKOFF_MAX_MS`, which is how long a provider may +wait before the reconnect attempt that produces the ``PROVIDER_READY`` ending +the ``@stale`` scenario. +""" + + +@dataclass(frozen=True) +class ResolverSuite: + """What differs between the two resolvers' suites.""" + + name: str + """Names the suite in test output and scopes its OpenFeature domain.""" + + resolver_type: ResolverType + + capabilities: frozenset[Capability] + """Derived from reading the resolver's event emission, not from running the suite. + + See each suite module for the evidence behind every entry, and behind every + omission. + """ + + ready_timeout: float + + +def build_config( + suite: ResolverSuite, + container: FlagdContainer, + control: HttpControl, + closed_port: int, +) -> TckConfig: + """Wire one resolver up to the running testbed. + + The ports are read here, after the stack is up: the testbed maps host ports + dynamically, so they do not exist earlier -- and they stay valid for the + whole session because nothing ever restarts a container. Outages are + simulated inside the running stack through ``control`` instead. + """ + port = container.get_port(suite.resolver_type) + + def new_provider() -> FeatureProvider: + return FlagdProvider( + resolver_type=suite.resolver_type, + host="localhost", + port=port, + deadline_ms=DEADLINE_MS, + stream_deadline_ms=STREAM_DEADLINE_MS, + retry_backoff_ms=RETRY_BACKOFF_MS, + retry_backoff_max_ms=RETRY_BACKOFF_MAX_MS, + retry_grace_period=RETRY_GRACE_PERIOD_SECONDS, + ) + + def new_unavailable_provider() -> FeatureProvider: + # Pointed at a closed port on localhost, never at the backend under + # test: that has to stay up, and simulated outages belong to the control + # API. + return FlagdProvider( + resolver_type=suite.resolver_type, + host="localhost", + port=closed_port, + deadline_ms=UNAVAILABLE_DEADLINE_MS, + stream_deadline_ms=STREAM_DEADLINE_MS, + retry_backoff_ms=UNAVAILABLE_BACKOFF_MS, + retry_backoff_max_ms=UNAVAILABLE_BACKOFF_MS, + retry_grace_period=UNAVAILABLE_GRACE_PERIOD_SECONDS, + ) + + return TckConfig( + name=suite.name, + control=control, + new_provider=new_provider, + new_unavailable_provider=new_unavailable_provider, + capabilities=suite.capabilities, + event_timeout=EVENT_TIMEOUT, + ready_timeout=suite.ready_timeout, + ) diff --git a/providers/openfeature-provider-flagd/tests/tck/test_in_process_conformance.py b/providers/openfeature-provider-flagd/tests/tck/test_in_process_conformance.py new file mode 100644 index 00000000..b2a79822 --- /dev/null +++ b/providers/openfeature-provider-flagd/tests/tck/test_in_process_conformance.py @@ -0,0 +1,96 @@ +"""The OpenFeature provider conformance suite, run against flagd's in-process resolver. + +In-process syncs the whole ruleset over flagd's sync API and evaluates locally, +so unlike RPC the type-checking, the variant selection and the reason all come +from ``openfeature-flagd-core`` in this process rather than from the server. Any +difference in the results is a difference an application would see when it +switches resolver, which is why this is a separate suite rather than a +parametrisation of the RPC one. +""" + +from __future__ import annotations + +import pytest +from pytest_bdd import scenarios + +from openfeature.contrib.provider.flagd.config import ResolverType +from openfeature.contrib.tools.provider_tck import ( + Capability, + HttpControl, + TckConfig, + features_path, +) +from tests.e2e.flagd_container import FlagdContainer +from tests.tck.suite import ResolverSuite, build_config + +# Every capability below is declared on the strength of a line of provider code, +# not on the strength of a green run. +# +# EVENTS +# grpc_watcher.py:262 emits PROVIDER_READY once the first sync payload has +# been applied -- note "applied", not "received": the ruleset is written to +# the evaluator at grpc_watcher.py:254 before ready is emitted, so a scenario +# that evaluates immediately after ready cannot race the first sync. +# +# STALE +# grpc_watcher.py:178-190: the channel-connectivity callback emits +# PROVIDER_STALE on TRANSIENT_FAILURE and starts a timer that escalates to +# PROVIDER_ERROR only once retry_grace_period expires. +# +# CONFIGURATION_CHANGE +# in_process.py:34 emits PROVIDER_CONFIGURATION_CHANGED naming exactly the +# keys that FlagdCore reports as changed, from every sync payload the watcher +# applies. +# +# OBJECT +# in_process.py:122 resolves structured values from the local ruleset. +# +# UNAVAILABLE_INIT +# grpc_watcher.py:151 raises ProviderNotReadyError once the blocking init +# deadline passes without a synced ruleset, which the SDK's registry turns +# into PROVIDER_ERROR. +# +# STRICT_NUMERIC_TYPING +# Local, and strict: flagd_core.py:25 admits only `int` for an integer +# request, and flagd_core.py:228-231 raises TypeMismatchError for anything +# else -- so `float-flag`'s 0.5 is reported as a mismatch rather than +# narrowed to 0. (The float mapping at flagd_core.py:26 is deliberately the +# wider one, `(int, float)`, but widening towards float loses nothing.) +# +# Not declared, and why: +# +# TARGETING, CACHING +# Reserved in the Capability enum; no scenario carries either tag. Declaring +# a capability nothing exercises would be a claim with no evidence behind it, +# so they are left out of both suites. +IN_PROCESS_CAPABILITIES = frozenset( + { + Capability.EVENTS, + Capability.STALE, + Capability.CONFIGURATION_CHANGE, + Capability.OBJECT, + Capability.UNAVAILABLE_INIT, + Capability.STRICT_NUMERIC_TYPING, + } +) + +IN_PROCESS_SUITE = ResolverSuite( + name="flagd-in-process", + resolver_type=ResolverType.IN_PROCESS, + capabilities=IN_PROCESS_CAPABILITIES, + # In-process transfers and applies the whole ruleset before reporting ready, + # so it needs more headroom than RPC. + ready_timeout=60.0, +) + + +@pytest.fixture(scope="session") +def tck_config( + flagd_testbed: FlagdContainer, + flagd_control: HttpControl, + closed_port: int, +) -> TckConfig: + return build_config(IN_PROCESS_SUITE, flagd_testbed, flagd_control, closed_port) + + +scenarios(features_path()) diff --git a/providers/openfeature-provider-flagd/tests/tck/test_rpc_conformance.py b/providers/openfeature-provider-flagd/tests/tck/test_rpc_conformance.py new file mode 100644 index 00000000..5aa38982 --- /dev/null +++ b/providers/openfeature-provider-flagd/tests/tck/test_rpc_conformance.py @@ -0,0 +1,97 @@ +"""The OpenFeature provider conformance suite, run against flagd's RPC resolver. + +RPC asks flagd to evaluate each flag over gRPC and maps the response onto typed +resolution details, so what is under test here is that mapping plus the +lifecycle the evaluation stream drives. +""" + +from __future__ import annotations + +import pytest +from pytest_bdd import scenarios + +from openfeature.contrib.provider.flagd.config import ResolverType +from openfeature.contrib.tools.provider_tck import ( + Capability, + HttpControl, + TckConfig, + features_path, +) +from tests.e2e.flagd_container import FlagdContainer +from tests.tck.suite import ResolverSuite, build_config + +# Every capability below is declared on the strength of a line of provider code, +# not on the strength of a green run. +# +# EVENTS +# grpc.py:261 emits PROVIDER_READY when the evaluation stream delivers its +# 'provider_ready' message. +# +# STALE +# grpc.py:202-212: the channel-connectivity callback emits PROVIDER_STALE on +# TRANSIENT_FAILURE and only then starts a timer that escalates to +# PROVIDER_ERROR once retry_grace_period expires. +# +# Worth calling out, because the Go provider does NOT do this: its RPC +# resolver sends ProviderError directly on connection loss and never emits +# PROVIDER_STALE, which is filed as go-sdk-contrib#939 and is why the Go +# adoption withholds this capability for RPC. Python has no such asymmetry -- +# both of its resolvers share the same state-change callback shape -- so the +# capability is declared here. +# +# CONFIGURATION_CHANGE +# grpc.py:302 emits PROVIDER_CONFIGURATION_CHANGED with the changed keys, and +# grpc.py:298-300 evicts exactly those keys from the LRU cache, so the +# re-evaluation the scenario performs afterwards cannot be served a stale +# cached value. +# +# OBJECT +# grpc.py:336 resolves structured values through ResolveObject. +# +# UNAVAILABLE_INIT +# grpc.py:175 raises ProviderNotReadyError once the blocking init deadline +# passes without a connection, which the SDK's registry turns into +# PROVIDER_ERROR. +# +# STRICT_NUMERIC_TYPING +# RPC does not type-check locally; it asks flagd for an Int and flagd answers +# INVALID_ARGUMENT for a float-valued flag, which grpc.py:461-462 maps to +# TypeMismatchError. So 0.5 is never narrowed to 0. +# +# Not declared, and why: +# +# TARGETING, CACHING +# Reserved in the Capability enum; no scenario carries either tag. Declaring +# a capability nothing exercises would be a claim with no evidence behind it, +# so they are left out of both suites. +RPC_CAPABILITIES = frozenset( + { + Capability.EVENTS, + Capability.STALE, + Capability.CONFIGURATION_CHANGE, + Capability.OBJECT, + Capability.UNAVAILABLE_INIT, + Capability.STRICT_NUMERIC_TYPING, + } +) + +RPC_SUITE = ResolverSuite( + name="flagd-rpc", + resolver_type=ResolverType.RPC, + capabilities=RPC_CAPABILITIES, + # RPC holds no ruleset of its own: it is ready as soon as the evaluation + # stream is up, so it needs less headroom than in-process. + ready_timeout=30.0, +) + + +@pytest.fixture(scope="session") +def tck_config( + flagd_testbed: FlagdContainer, + flagd_control: HttpControl, + closed_port: int, +) -> TckConfig: + return build_config(RPC_SUITE, flagd_testbed, flagd_control, closed_port) + + +scenarios(features_path()) From 5433c82407eb34a26e598185268eb13324666f7f Mon Sep 17 00:00:00 2001 From: Simon Schrottner Date: Mon, 24 Aug 2026 14:06:46 +0200 Subject: [PATCH 02/29] fix(flagd): add the provider conformance suite to uv.lock The flagd dev group now depends on openfeature-provider-tck as a workspace source, so the lock has to carry it or uv sync --frozen fails for every package in the workspace. Signed-off-by: Simon Schrottner --- uv.lock | 2 ++ 1 file changed, 2 insertions(+) diff --git a/uv.lock b/uv.lock index ea025adb..98b43a43 100644 --- a/uv.lock +++ b/uv.lock @@ -1886,6 +1886,7 @@ dev = [ { name = "coverage", extra = ["toml"] }, { name = "grpcio-health-checking" }, { name = "mypy" }, + { name = "openfeature-provider-tck" }, { name = "poethepoet" }, { name = "pytest" }, { name = "pytest-bdd" }, @@ -1911,6 +1912,7 @@ dev = [ { name = "coverage", extras = ["toml"], specifier = ">=7.10.0,<8.0.0" }, { name = "grpcio-health-checking", specifier = ">=1.82.1,<2.0.0" }, { name = "mypy", specifier = ">=1.18.0,<2.0.0" }, + { name = "openfeature-provider-tck", editable = "tools/openfeature-provider-tck" }, { name = "poethepoet", specifier = ">=0.37.0" }, { name = "pytest", specifier = ">=9.0.0,<10.0.0" }, { name = "pytest-bdd", specifier = ">=8.1.0,<9.0.0" }, From a95bbd79b5073a4b3195a7a2d70511c08016188b Mon Sep 17 00:00:00 2001 From: Simon Schrottner Date: Fri, 11 Sep 2026 09:53:12 +0200 Subject: [PATCH 03/29] test(flagd): follow the @numeric-coercion rename The capability was renamed on the base branch, so these declarations named a symbol that no longer exists. Both resolvers keep declaring it: this provider satisfies the rule in both paths, which is worth noting because the other two languages' flagd providers do not. The rationale comments already recorded why, and they are now the evidence that the defect is narrower than it was described as being. RPC asks flagd for an Int and flagd answers INVALID_ARGUMENT for a float-valued flag, which the provider maps to TypeMismatchError. In-process is local and strict: an integer request admits only int, and the float mapping is deliberately the wider (int, float) because widening towards float loses nothing -- which is the lossless-coercion rule arrived at independently, before it had a name here. So flagd's float narrowing is a property of the Go and Java provider layers rather than of flagd itself. The Go adoption's comment currently says the opposite, calling it "a flagd-wide issue", and needs correcting. Signed-off-by: Simon Schrottner --- .../tests/tck/test_in_process_conformance.py | 4 ++-- .../tests/tck/test_rpc_conformance.py | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/providers/openfeature-provider-flagd/tests/tck/test_in_process_conformance.py b/providers/openfeature-provider-flagd/tests/tck/test_in_process_conformance.py index b2a79822..9678bfd8 100644 --- a/providers/openfeature-provider-flagd/tests/tck/test_in_process_conformance.py +++ b/providers/openfeature-provider-flagd/tests/tck/test_in_process_conformance.py @@ -50,7 +50,7 @@ # deadline passes without a synced ruleset, which the SDK's registry turns # into PROVIDER_ERROR. # -# STRICT_NUMERIC_TYPING +# NUMERIC_COERCION # Local, and strict: flagd_core.py:25 admits only `int` for an integer # request, and flagd_core.py:228-231 raises TypeMismatchError for anything # else -- so `float-flag`'s 0.5 is reported as a mismatch rather than @@ -70,7 +70,7 @@ Capability.CONFIGURATION_CHANGE, Capability.OBJECT, Capability.UNAVAILABLE_INIT, - Capability.STRICT_NUMERIC_TYPING, + Capability.NUMERIC_COERCION, } ) diff --git a/providers/openfeature-provider-flagd/tests/tck/test_rpc_conformance.py b/providers/openfeature-provider-flagd/tests/tck/test_rpc_conformance.py index 5aa38982..87e91a71 100644 --- a/providers/openfeature-provider-flagd/tests/tck/test_rpc_conformance.py +++ b/providers/openfeature-provider-flagd/tests/tck/test_rpc_conformance.py @@ -53,7 +53,7 @@ # passes without a connection, which the SDK's registry turns into # PROVIDER_ERROR. # -# STRICT_NUMERIC_TYPING +# NUMERIC_COERCION # RPC does not type-check locally; it asks flagd for an Int and flagd answers # INVALID_ARGUMENT for a float-valued flag, which grpc.py:461-462 maps to # TypeMismatchError. So 0.5 is never narrowed to 0. @@ -71,7 +71,7 @@ Capability.CONFIGURATION_CHANGE, Capability.OBJECT, Capability.UNAVAILABLE_INIT, - Capability.STRICT_NUMERIC_TYPING, + Capability.NUMERIC_COERCION, } ) From 4e1dabf085c271a3f3d906015cddff15a9f9696b Mon Sep 17 00:00:00 2001 From: Simon Schrottner Date: Fri, 11 Sep 2026 12:36:13 +0200 Subject: [PATCH 04/29] test(flagd): declare @large-integers and withdraw @numeric-coercion Follow the conformance assets at spec@15fe8611, which the base now pins: @numeric-coercion requires lossless coercion in both directions, and the new @large-integers tag asks for 2^53 - 1. Neither resolver satisfies all three @numeric-coercion scenarios, so neither declares it: - RPC passes back whatever flagd answers, and flagd v0.16.0 -- the image flagd-testbed v3.8.0 builds on -- resolves an integer request as a float64 and casts it with a bare int64(val), so float-flag's 0.5 is narrowed to 0 with no error code. The earlier claim in this file that flagd answers INVALID_ARGUMENT was not borne out by the server source. open-feature/flagd#1996 implements the lossless rule. - In-process admits only int for an integer request (flagd_core.py:25), so integral-float-flag's 10.0 is a TYPE_MISMATCH where 10 is required. The other direction is widened (flagd_core.py:113-114), which is why the lossy scenario alone was passing before. Both declare @large-integers: RPC carries the value as an int64 that the provider hands over as a Python int, and in-process parses the ruleset with json.loads, so nothing narrows 9007199254740991. Record in the conftest that flagd-testbed v3.8.0 does not seed the six flags the assets added, so the untagged falsy and precision scenarios, and the @large-integers one, fail FLAG_NOT_FOUND until the testbed is updated. Left as failures rather than xfails: the backend is behind, not the provider. Signed-off-by: Simon Schrottner --- .../tests/tck/conftest.py | 19 ++++++++++ .../tests/tck/test_in_process_conformance.py | 28 +++++++++++---- .../tests/tck/test_rpc_conformance.py | 35 ++++++++++++++++--- 3 files changed, 70 insertions(+), 12 deletions(-) diff --git a/providers/openfeature-provider-flagd/tests/tck/conftest.py b/providers/openfeature-provider-flagd/tests/tck/conftest.py index b67feaaa..b0ea60e0 100644 --- a/providers/openfeature-provider-flagd/tests/tck/conftest.py +++ b/providers/openfeature-provider-flagd/tests/tck/conftest.py @@ -12,6 +12,25 @@ TCK drives the testbed's launchpad through the standardised control API, which the launchpad already implements, and reuses the container lifecycle already in ``tests/e2e``. + +**The testbed does not yet serve the whole canonical flag set.** The +conformance assets at spec@15fe8611 added six flags -- ``false-flag``, +``zero-flag``, ``empty-string-flag``, ``large-integer-flag``, +``huge-integer-flag`` and ``integral-float-flag`` -- and flagd-testbed v3.8.0 +(``openfeature/test-harness/version.txt``) seeds none of them. Until +open-feature/flagd-testbed catches up, both suites fail these scenarios with +``FLAG_NOT_FOUND``, for every resolver alike: + +* ``A falsy value is a value, not an absence`` -- three rows, untagged; +* ``A large integer resolves without loss of precision`` -- untagged; +* ``An integer beyond 32 bits resolves without loss of precision`` -- under + ``@large-integers``, which both suites declare. + +``integral-float-flag`` is asked for only under ``@numeric-coercion``, which +neither suite declares, so its scenario is skipped rather than failed. The +failures are deliberately left as failures: they say something true about the +stack under test, and an ``xfail`` would say the provider is at fault when it is +the backend that is behind. """ from __future__ import annotations diff --git a/providers/openfeature-provider-flagd/tests/tck/test_in_process_conformance.py b/providers/openfeature-provider-flagd/tests/tck/test_in_process_conformance.py index 9678bfd8..527e4a49 100644 --- a/providers/openfeature-provider-flagd/tests/tck/test_in_process_conformance.py +++ b/providers/openfeature-provider-flagd/tests/tck/test_in_process_conformance.py @@ -50,15 +50,29 @@ # deadline passes without a synced ruleset, which the SDK's registry turns # into PROVIDER_ERROR. # -# NUMERIC_COERCION -# Local, and strict: flagd_core.py:25 admits only `int` for an integer -# request, and flagd_core.py:228-231 raises TypeMismatchError for anything -# else -- so `float-flag`'s 0.5 is reported as a mismatch rather than -# narrowed to 0. (The float mapping at flagd_core.py:26 is deliberately the -# wider one, `(int, float)`, but widening towards float loses nothing.) +# LARGE_INTEGERS +# The ruleset arrives as JSON text over the sync stream and FlagdCore parses +# it with `json.loads` (flagd_core.py:73), which gives an unbounded Python +# int for 9007199254740991; nothing between the parser and the SDK routes +# the value through a float or a 32-bit field. # # Not declared, and why: # +# NUMERIC_COERCION +# Local, and strict in one direction only. flagd_core.py:25 admits only +# `int` for an integer request and `_check_type` (flagd_core.py:228-231) +# raises TypeMismatchError for anything else, so `float-flag`'s 0.5 is a +# mismatch rather than 0 -- the lossy half holds. The float mapping at +# flagd_core.py:26 is the wider `(int, float)`, and `resolve_float_value` +# (flagd_core.py:113-114) widens an int result to a float, so `integer-flag` +# requested as a Float is 10.0 -- that lossless half holds too. But the +# same `(int,)` rule rejects `integral-float-flag`'s 10.0 requested as an +# Integer, where the tag requires 10: two of three, and a declaration is +# all or nothing. flagd's numeric-coercion ADR +# (docs/architecture-decisions/numeric-coercion.md) commits every flagd +# implementation to the lossless rule; when openfeature-flagd-core follows +# it, this is declared again. +# # TARGETING, CACHING # Reserved in the Capability enum; no scenario carries either tag. Declaring # a capability nothing exercises would be a claim with no evidence behind it, @@ -70,7 +84,7 @@ Capability.CONFIGURATION_CHANGE, Capability.OBJECT, Capability.UNAVAILABLE_INIT, - Capability.NUMERIC_COERCION, + Capability.LARGE_INTEGERS, } ) diff --git a/providers/openfeature-provider-flagd/tests/tck/test_rpc_conformance.py b/providers/openfeature-provider-flagd/tests/tck/test_rpc_conformance.py index 87e91a71..6c77587c 100644 --- a/providers/openfeature-provider-flagd/tests/tck/test_rpc_conformance.py +++ b/providers/openfeature-provider-flagd/tests/tck/test_rpc_conformance.py @@ -53,13 +53,38 @@ # passes without a connection, which the SDK's registry turns into # PROVIDER_ERROR. # -# NUMERIC_COERCION -# RPC does not type-check locally; it asks flagd for an Int and flagd answers -# INVALID_ARGUMENT for a float-valued flag, which grpc.py:461-462 maps to -# TypeMismatchError. So 0.5 is never narrowed to 0. +# LARGE_INTEGERS +# RPC never narrows an integer. flagd holds every numeric variant as a +# float64 -- Go's encoding/json decodes an untyped number into one -- and +# 2^53 - 1 is exactly the largest integer a float64 represents without +# rounding, which is why the canonical set asks for nothing larger. The +# server casts it to the int64 of ResolveIntResponse.value, and grpc.py:448 +# hands that to the SDK as a Python int, unbounded. Nothing in between is +# 32 bits wide. # # Not declared, and why: # +# NUMERIC_COERCION +# RPC does not type-check locally: grpc.py:444-448 asks flagd for an Int and +# passes back whatever the server answers, so the whole decision is flagd's. +# flagd's evaluator resolves the variant as a float64 and casts it with a +# bare `int64(val)` (core/pkg/evaluator/json.go, ResolveIntValue, at the +# v0.16.0 the testbed's `flagd/Dockerfile` builds on), so `float-flag`'s 0.5 +# comes back as 0 with reason STATIC and no error code -- silently narrowed, +# which is the one thing the lossy scenario forbids. The two lossless +# scenarios pass for the same reason: 10.0 casts to 10, and a float +# accessor sees the float64 the server already holds. One of three is a +# failure, and a declaration is all or nothing. +# +# An earlier revision of this file claimed flagd answers INVALID_ARGUMENT +# here. The server source says otherwise: INVALID_ARGUMENT is what +# grpc.py:461-462 would map to TypeMismatchError if it ever arrived, and +# for a float-valued flag it does not. The Java reference adoption recorded +# the same narrowing against the same server. flagd's numeric-coercion ADR +# (docs/architecture-decisions/numeric-coercion.md) commits it to lossless +# coercion, tracked as open-feature/flagd#1996; this is declared again once +# the testbed ships a flagd that implements it. +# # TARGETING, CACHING # Reserved in the Capability enum; no scenario carries either tag. Declaring # a capability nothing exercises would be a claim with no evidence behind it, @@ -71,7 +96,7 @@ Capability.CONFIGURATION_CHANGE, Capability.OBJECT, Capability.UNAVAILABLE_INIT, - Capability.NUMERIC_COERCION, + Capability.LARGE_INTEGERS, } ) From 3516c8f7016e8df1b136b69268473be962490011 Mon Sep 17 00:00:00 2001 From: Simon Schrottner Date: Fri, 11 Sep 2026 13:52:48 +0200 Subject: [PATCH 05/29] test(flagd): the falsy flags were a naming gap, not a backend gap The suite's note on what flagd-testbed does not serve was written against spec@15fe8611 and listed six missing flags. Three of them were never missing. spec@ba002ce8 renames them to boolean-zero-flag, integer-zero-flag and string-zero-flag, which is what flagd-testbed's own flags/zero-flags.json has called them all along, with the same zero/non-zero variants the scenarios assert. Verified against flagd-testbed v3.8.0: all three resolve with variant zero and reason STATIC on both resolvers. What remains genuinely unseeded is large-integer-flag, huge-integer-flag and integral-float-flag, so the note now names those three and lists only the two scenarios that still fail. The reasoning underneath is unchanged: the failures stay failures rather than xfails, because they say something true about the backend rather than about the provider. Signed-off-by: Simon Schrottner --- .../tests/tck/conftest.py | 20 +++++++++++-------- 1 file changed, 12 insertions(+), 8 deletions(-) diff --git a/providers/openfeature-provider-flagd/tests/tck/conftest.py b/providers/openfeature-provider-flagd/tests/tck/conftest.py index b0ea60e0..7f261a84 100644 --- a/providers/openfeature-provider-flagd/tests/tck/conftest.py +++ b/providers/openfeature-provider-flagd/tests/tck/conftest.py @@ -14,14 +14,12 @@ ``tests/e2e``. **The testbed does not yet serve the whole canonical flag set.** The -conformance assets at spec@15fe8611 added six flags -- ``false-flag``, -``zero-flag``, ``empty-string-flag``, ``large-integer-flag``, -``huge-integer-flag`` and ``integral-float-flag`` -- and flagd-testbed v3.8.0 -(``openfeature/test-harness/version.txt``) seeds none of them. Until -open-feature/flagd-testbed catches up, both suites fail these scenarios with -``FLAG_NOT_FOUND``, for every resolver alike: - -* ``A falsy value is a value, not an absence`` -- three rows, untagged; +conformance assets at spec@ba002ce8 ask for three flags that flagd-testbed +v3.8.0 (``openfeature/test-harness/version.txt``) does not seed: +``large-integer-flag``, ``huge-integer-flag`` and ``integral-float-flag``. +Until open-feature/flagd-testbed catches up, both suites fail these scenarios +with ``FLAG_NOT_FOUND``, for every resolver alike: + * ``A large integer resolves without loss of precision`` -- untagged; * ``An integer beyond 32 bits resolves without loss of precision`` -- under ``@large-integers``, which both suites declare. @@ -31,6 +29,12 @@ failures are deliberately left as failures: they say something true about the stack under test, and an ``xfail`` would say the provider is at fault when it is the backend that is behind. + +The falsy flags used to fail the same way and no longer do. ``ba002ce8`` renamed +them to ``boolean-zero-flag``, ``integer-zero-flag`` and ``string-zero-flag``, +which is what ``flags/zero-flags.json`` in the testbed has always called them, +with the same ``zero``/``non-zero`` variants the scenarios assert. Those three +rows were never a gap in the backend, only a disagreement about names. """ from __future__ import annotations From e2acbed6faf806d2e1649d3cf0d2b5448d88ae76 Mon Sep 17 00:00:00 2001 From: Simon Schrottner Date: Fri, 11 Sep 2026 16:31:05 +0200 Subject: [PATCH 06/29] test(flagd): withhold @reinitialization, and say what was measured spec@fc99d5ac gates "A provider that was shut down can be initialized again" behind a new @reinitialization tag. Neither resolver declares it, and the reasoning is recorded next to each declaration because the arithmetic that suggested declaring it was wrong in an instructive way. The premise was that the scenario had been untagged at the previous pin, so it ran, and since it was not among the two failures it must have passed. It did not run. lifecycle.feature carries @lifecycle at the feature level, so every scenario in it inherits that tag, and the gate skips a scenario when any capability gating it is undeclared. Neither resolver declares LIFECYCLE, so the scenario is one of the six @lifecycle skips each resolver already reported -- absence from the failure list was absence from the run. Measured rather than reasoned about, by declaring LIFECYCLE and REINITIALIZATION together locally and letting the scenario run: in-process passes -- reuse genuinely works rpc fails -- boolean-flag resolves to the code default because grpc.py:420 raises "Cannot invoke RPC on closed channel!"; shutdown() closes the channel and the second initialize() does not rebuild it So the two resolvers genuinely differ, and RPC exhibits precisely the shape the specification's note on this capability describes: the provider evaluates against a closed connection rather than failing outright. Withheld for both regardless. For RPC because reuse does not work; for in-process because declaring REINITIALIZATION without LIFECYCLE leaves the scenario skipped on @lifecycle and the claim unexamined, which is the vacuous declaration this suite keeps the reserved tags out for. Whether to declare LIFECYCLE is a separate question and is not settled here. No KnownDeviation entry either way: Requirement 2.5.2 says a provider SHOULD revert to its uninitialized state and that "some providers MAY allow reinitialization", so declining reuse is a choice the specification offers rather than a requirement to deviate from. Tallies are unchanged from the previous pin, which is the check that no scenario silently changed status: rpc 29 passed / 2 failed / 9 skipped, in-process 28 / 3 / 9, nine skips per resolver being six @lifecycle and three @numeric-coercion. The failures remain large-integer-flag and huge-integer-flag, absent from flagd-testbed, plus in-process's boolean-flag-as-Float, which is #417. Signed-off-by: Simon Schrottner --- .../tests/tck/conftest.py | 2 +- .../tests/tck/test_in_process_conformance.py | 27 ++++++++++++++++ .../tests/tck/test_rpc_conformance.py | 31 +++++++++++++++++++ 3 files changed, 59 insertions(+), 1 deletion(-) diff --git a/providers/openfeature-provider-flagd/tests/tck/conftest.py b/providers/openfeature-provider-flagd/tests/tck/conftest.py index 7f261a84..f427020a 100644 --- a/providers/openfeature-provider-flagd/tests/tck/conftest.py +++ b/providers/openfeature-provider-flagd/tests/tck/conftest.py @@ -14,7 +14,7 @@ ``tests/e2e``. **The testbed does not yet serve the whole canonical flag set.** The -conformance assets at spec@ba002ce8 ask for three flags that flagd-testbed +conformance assets at spec@fc99d5ac ask for three flags that flagd-testbed v3.8.0 (``openfeature/test-harness/version.txt``) does not seed: ``large-integer-flag``, ``huge-integer-flag`` and ``integral-float-flag``. Until open-feature/flagd-testbed catches up, both suites fail these scenarios diff --git a/providers/openfeature-provider-flagd/tests/tck/test_in_process_conformance.py b/providers/openfeature-provider-flagd/tests/tck/test_in_process_conformance.py index 527e4a49..a82515a5 100644 --- a/providers/openfeature-provider-flagd/tests/tck/test_in_process_conformance.py +++ b/providers/openfeature-provider-flagd/tests/tck/test_in_process_conformance.py @@ -73,6 +73,33 @@ # implementation to the lossless rule; when openfeature-flagd-core follows # it, this is declared again. # +# REINITIALIZATION +# New at spec@fc99d5ac, which gated the scenario "A provider that was shut +# down can be initialized again" that had been untagged before it. Withheld +# here even though this resolver does support reuse, which is the +# interesting half of the story and was measured rather than assumed: +# declaring LIFECYCLE and REINITIALIZATION together locally makes the +# scenario run, and in-process passes it, while RPC fails it against a +# closed channel. The two resolvers genuinely differ. +# +# It stays withheld because declaring it would be vacuous. The scenario +# lives in lifecycle.feature, which carries @lifecycle at the feature level, +# so it inherits that tag and carries both; the gate skips a scenario when +# any capability gating it is undeclared, and this suite does not declare +# LIFECYCLE. Declaring REINITIALIZATION alone would leave the scenario +# skipped on @lifecycle and the claim unexamined -- the same +# declare-what-nothing-exercises error the reserved tags below are kept out +# for. Declaring LIFECYCLE is a separate question from this one and is not +# settled here. +# +# Requirement 2.5.2 says a provider SHOULD revert to its uninitialized +# state and that "some providers MAY allow reinitialization", so reuse is +# permitted rather than required and withholding needs no KnownDeviation. +# +# Worth recording that this scenario never ran here, at this pin or the one +# before it: it is one of the six @lifecycle skips each resolver reports, +# not a scenario that used to pass. +# # TARGETING, CACHING # Reserved in the Capability enum; no scenario carries either tag. Declaring # a capability nothing exercises would be a claim with no evidence behind it, diff --git a/providers/openfeature-provider-flagd/tests/tck/test_rpc_conformance.py b/providers/openfeature-provider-flagd/tests/tck/test_rpc_conformance.py index 6c77587c..08a53e22 100644 --- a/providers/openfeature-provider-flagd/tests/tck/test_rpc_conformance.py +++ b/providers/openfeature-provider-flagd/tests/tck/test_rpc_conformance.py @@ -85,6 +85,37 @@ # coercion, tracked as open-feature/flagd#1996; this is declared again once # the testbed ships a flagd that implements it. # +# REINITIALIZATION +# New at spec@fc99d5ac, which gated the scenario "A provider that was shut +# down can be initialized again" that had been untagged before it. Withheld +# for two independent reasons, either of which is sufficient. +# +# First, RPC genuinely does not support reuse, which was measured rather +# than reasoned about: declaring LIFECYCLE and REINITIALIZATION together +# locally makes the scenario run, and it fails with `boolean-flag` resolving +# to the code default because grpc.py:420 raises "Cannot invoke RPC on +# closed channel!". shutdown() closes the channel and the second initialize() +# does not rebuild it, so the provider evaluates against a closed connection +# rather than failing outright -- exactly the shape the specification's own +# note on this capability describes. Requirement 2.5.2 says a provider +# SHOULD revert to its uninitialized state and that "some providers MAY +# allow reinitialization", so reuse is permitted rather than required and +# declining it is a choice the specification offers. Hence no +# KnownDeviation entry: there is no requirement to deviate from. +# +# Second, and why this cannot be declared even where reuse does work: the +# scenario lives in lifecycle.feature, which carries @lifecycle at the +# feature level, so it inherits that tag and carries both. The gate skips a +# scenario when any capability gating it is undeclared, and neither resolver +# declares LIFECYCLE. Declaring REINITIALIZATION alone would leave the +# scenario skipped on @lifecycle and the claim unexamined -- a vacuous +# declaration of the kind the reserved tags below are kept out for. +# +# Worth recording that this scenario never ran here, at this pin or the one +# before it: it is one of the six @lifecycle skips each resolver reports, +# not a scenario that used to pass. Reading its absence from the failure +# list as evidence of support is the mistake this note exists to prevent. +# # TARGETING, CACHING # Reserved in the Capability enum; no scenario carries either tag. Declaring # a capability nothing exercises would be a claim with no evidence behind it, From 6d2b13265e58af8e07bb8f51654a4d1297be0ce1 Mon Sep 17 00:00:00 2001 From: Simon Schrottner Date: Fri, 11 Sep 2026 23:43:55 +0200 Subject: [PATCH 07/29] test(flagd): declare @variants and @targeting, on a measured run Both resolvers, both declared, suite run, scenarios seen to pass. The numbers: 79 passed, 18 skipped, 7 failed across the two suites, which is 104 = 2 x 52 canonical scenario instances. @targeting is the interesting one, and it costs the testbed nothing. targeting-key-flag is the flag flagd-testbed's own targeting.feature already uses -- same key, same hit/miss variants, same uuid -- seeded from flags/testing-flags.json by the launchpad's default configuration. All three scenarios pass on both resolvers: RPC because grpc.py:492 puts the targeting key into the request context, in-process because targeting.py:40-41 puts it into the JSON-logic context. @variants passes seven of its eight rows on both resolvers. The eighth asks for large-integer-flag's max-int32 and fails with FLAG_NOT_FOUND, which is the testbed gap already recorded in conftest.py rather than a new one: flagd-testbed v3.8.0 seeds neither large-integer-flag nor huge-integer-flag, and the same absence already fails the two precision scenarios. Withholding the capability over it would say flagd does not name variants, which the other seven rows show is false, and would attribute a missing flag to a capability the provider has. It is not a KnownDeviation either: a deviation is for a behaviour the provider is required to have and does not. Skips are unchanged in kind: six @lifecycle and three @numeric-coercion per resolver, each with its reason. The header claiming these declarations rest "on the strength of a line of provider code, not on the strength of a green run" is gone. Appendix F states the opposite rule as of 26362f85, and this file already contradicted itself -- its @reinitialization note says what was measured, and measuring is what showed the two resolvers genuinely differ. The code references stay, as places a reader can check a claim rather than as the evidence for it. Signed-off-by: Simon Schrottner --- .../tests/tck/conftest.py | 21 ++++++++++-- .../tests/tck/suite.py | 9 +++-- .../tests/tck/test_in_process_conformance.py | 34 +++++++++++++++---- .../tests/tck/test_rpc_conformance.py | 34 +++++++++++++++---- 4 files changed, 78 insertions(+), 20 deletions(-) diff --git a/providers/openfeature-provider-flagd/tests/tck/conftest.py b/providers/openfeature-provider-flagd/tests/tck/conftest.py index f427020a..4faaf980 100644 --- a/providers/openfeature-provider-flagd/tests/tck/conftest.py +++ b/providers/openfeature-provider-flagd/tests/tck/conftest.py @@ -14,7 +14,7 @@ ``tests/e2e``. **The testbed does not yet serve the whole canonical flag set.** The -conformance assets at spec@fc99d5ac ask for three flags that flagd-testbed +conformance assets at spec@26362f85 ask for three flags that flagd-testbed v3.8.0 (``openfeature/test-harness/version.txt``) does not seed: ``large-integer-flag``, ``huge-integer-flag`` and ``integral-float-flag``. Until open-feature/flagd-testbed catches up, both suites fail these scenarios @@ -22,13 +22,28 @@ * ``A large integer resolves without loss of precision`` -- untagged; * ``An integer beyond 32 bits resolves without loss of precision`` -- under - ``@large-integers``, which both suites declare. + ``@large-integers``, which both suites declare; +* ``The resolved details name the variant``, the ``large-integer-flag`` row of + it -- under ``@variants``, which both suites declare. New at spec@26362f85, + and the same gap rather than a new one: the row asks for the variant + ``max-int32`` of a flag that is not there, so it fails with the same + ``FLAG_NOT_FOUND`` as the two above. The other seven rows pass on both + resolvers, which is the evidence the capability is declared on -- withholding + it would say flagd does not name variants, which is false, and would + attribute a missing flag to a capability the provider has. ``integral-float-flag`` is asked for only under ``@numeric-coercion``, which neither suite declares, so its scenario is skipped rather than failed. The failures are deliberately left as failures: they say something true about the stack under test, and an ``xfail`` would say the provider is at fault when it is -the backend that is behind. +the backend that is behind. None of them is a ``KnownDeviation`` either: a +deviation is for a behaviour the *provider* is required to have and does not. + +``targeting-key-flag``, new in the canonical set at the same revision, needs no +testbed change. It is the flag flagd-testbed's own ``targeting.feature`` already +uses -- same key, same ``hit``/``miss`` variants, same uuid -- seeded from +``flags/testing-flags.json`` by the launchpad's ``default`` configuration, so +the three ``@targeting`` scenarios pass on both resolvers as they stand. The falsy flags used to fail the same way and no longer do. ``ba002ce8`` renamed them to ``boolean-zero-flag``, ``integer-zero-flag`` and ``string-zero-flag``, diff --git a/providers/openfeature-provider-flagd/tests/tck/suite.py b/providers/openfeature-provider-flagd/tests/tck/suite.py index 8a3c3e35..2066ba6f 100644 --- a/providers/openfeature-provider-flagd/tests/tck/suite.py +++ b/providers/openfeature-provider-flagd/tests/tck/suite.py @@ -105,10 +105,13 @@ class ResolverSuite: resolver_type: ResolverType capabilities: frozenset[Capability] - """Derived from reading the resolver's event emission, not from running the suite. + """What this resolver was run against the suite and seen to satisfy. - See each suite module for the evidence behind every entry, and behind every - omission. + Evidence from a run rather than from reading the resolver, which is Appendix + F's rule and not merely a preference: source inspection has been wrong in + both directions here, and the two resolvers differ on reuse in a way that + reading them does not show. See each suite module for the evidence behind + every entry, and behind every omission. """ ready_timeout: float diff --git a/providers/openfeature-provider-flagd/tests/tck/test_in_process_conformance.py b/providers/openfeature-provider-flagd/tests/tck/test_in_process_conformance.py index a82515a5..e629e508 100644 --- a/providers/openfeature-provider-flagd/tests/tck/test_in_process_conformance.py +++ b/providers/openfeature-provider-flagd/tests/tck/test_in_process_conformance.py @@ -23,8 +23,14 @@ from tests.e2e.flagd_container import FlagdContainer from tests.tck.suite import ResolverSuite, build_config -# Every capability below is declared on the strength of a line of provider code, -# not on the strength of a green run. +# Every capability below was declared, the suite run, and the scenarios seen to +# pass. The code references say where the behaviour lives, so a reader can check +# the claim -- they are not the evidence for it. +# +# The distinction is Appendix F's, stated there since spec@26362f85 and worth +# repeating here because this file used to get it backwards: source inspection is +# unreliable in both directions, which @reinitialization below shows from the +# other side -- this resolver does support reuse, and reading it does not say so. # # EVENTS # grpc_watcher.py:262 emits PROVIDER_READY once the first sync payload has @@ -45,6 +51,17 @@ # OBJECT # in_process.py:122 resolves structured values from the local ruleset. # +# VARIANTS +# flagd_core.py returns the variant name it selected with every resolution, +# and in_process.py carries it into the resolution details. The ruleset is +# keyed by variant, so there is always one to report. +# +# TARGETING +# targeting.py:40-41 puts the evaluation context's targeting key into the +# JSON-logic context under `targetingKey`, and flagd_core.py:154 evaluates +# the flag's rule against it, so targeting-key-flag selects `hit` or `miss` +# locally. +# # UNAVAILABLE_INIT # grpc_watcher.py:151 raises ProviderNotReadyError once the blocking init # deadline passes without a synced ruleset, which the SDK's registry turns @@ -88,7 +105,7 @@ # any capability gating it is undeclared, and this suite does not declare # LIFECYCLE. Declaring REINITIALIZATION alone would leave the scenario # skipped on @lifecycle and the claim unexamined -- the same -# declare-what-nothing-exercises error the reserved tags below are kept out +# declare-what-nothing-exercises error the reserved tag below is kept out # for. Declaring LIFECYCLE is a separate question from this one and is not # settled here. # @@ -100,16 +117,19 @@ # before it: it is one of the six @lifecycle skips each resolver reports, # not a scenario that used to pass. # -# TARGETING, CACHING -# Reserved in the Capability enum; no scenario carries either tag. Declaring -# a capability nothing exercises would be a claim with no evidence behind it, -# so they are left out of both suites. +# CACHING +# Reserved in the Capability enum; no scenario carries the tag. Declaring a +# capability nothing exercises would be a claim with no evidence behind it, +# so it is left out of both suites. @targeting was reserved alongside it +# until spec@26362f85 gave it three scenarios, and is now declared above. IN_PROCESS_CAPABILITIES = frozenset( { Capability.EVENTS, Capability.STALE, Capability.CONFIGURATION_CHANGE, Capability.OBJECT, + Capability.VARIANTS, + Capability.TARGETING, Capability.UNAVAILABLE_INIT, Capability.LARGE_INTEGERS, } diff --git a/providers/openfeature-provider-flagd/tests/tck/test_rpc_conformance.py b/providers/openfeature-provider-flagd/tests/tck/test_rpc_conformance.py index 08a53e22..c006fb7a 100644 --- a/providers/openfeature-provider-flagd/tests/tck/test_rpc_conformance.py +++ b/providers/openfeature-provider-flagd/tests/tck/test_rpc_conformance.py @@ -20,8 +20,15 @@ from tests.e2e.flagd_container import FlagdContainer from tests.tck.suite import ResolverSuite, build_config -# Every capability below is declared on the strength of a line of provider code, -# not on the strength of a green run. +# Every capability below was declared, the suite run, and the scenarios seen to +# pass. The code references say where the behaviour lives, so a reader can check +# the claim -- they are not the evidence for it. +# +# The distinction is Appendix F's, stated there since spec@26362f85 and worth +# repeating here because this file used to get it backwards: source inspection is +# unreliable in both directions. @reinitialization below is the case in point -- +# RPC's shutdown reverts enough of its own state to read as support for reuse, +# and the channel underneath cannot be rebuilt. # # EVENTS # grpc.py:261 emits PROVIDER_READY when the evaluation stream delivers its @@ -48,6 +55,16 @@ # OBJECT # grpc.py:336 resolves structured values through ResolveObject. # +# VARIANTS +# grpc.py:449 carries the response's `variant` field into the resolution +# details for every typed call, and flagd names a variant for every flag in +# the testbed's set. +# +# TARGETING +# grpc.py:492 puts the evaluation context's targeting key into the request's +# context struct, so the server evaluates targeting-key-flag's rule against +# it and answers `hit` or `miss`. +# # UNAVAILABLE_INIT # grpc.py:175 raises ProviderNotReadyError once the blocking init deadline # passes without a connection, which the SDK's registry turns into @@ -109,23 +126,26 @@ # scenario when any capability gating it is undeclared, and neither resolver # declares LIFECYCLE. Declaring REINITIALIZATION alone would leave the # scenario skipped on @lifecycle and the claim unexamined -- a vacuous -# declaration of the kind the reserved tags below are kept out for. +# declaration of the kind the reserved tag below is kept out for. # # Worth recording that this scenario never ran here, at this pin or the one # before it: it is one of the six @lifecycle skips each resolver reports, # not a scenario that used to pass. Reading its absence from the failure # list as evidence of support is the mistake this note exists to prevent. # -# TARGETING, CACHING -# Reserved in the Capability enum; no scenario carries either tag. Declaring -# a capability nothing exercises would be a claim with no evidence behind it, -# so they are left out of both suites. +# CACHING +# Reserved in the Capability enum; no scenario carries the tag. Declaring a +# capability nothing exercises would be a claim with no evidence behind it, +# so it is left out of both suites. @targeting was reserved alongside it +# until spec@26362f85 gave it three scenarios, and is now declared above. RPC_CAPABILITIES = frozenset( { Capability.EVENTS, Capability.STALE, Capability.CONFIGURATION_CHANGE, Capability.OBJECT, + Capability.VARIANTS, + Capability.TARGETING, Capability.UNAVAILABLE_INIT, Capability.LARGE_INTEGERS, } From d38044800757ffbeaa5f72eb04ff8e42ace437c3 Mon Sep 17 00:00:00 2001 From: Simon Schrottner Date: Sat, 12 Sep 2026 12:50:05 +0200 Subject: [PATCH 08/29] test(flagd): declare @disabled-flags on both resolvers, measured New at spec@009afe06. Declared for RPC and for in-process, on a run: 7 failed, 87 passed, 18 skipped, twice in a row, which is the same seven failures as before the pin moved and eight more passes -- the four rows per resolver. The four disabled-* flags need no testbed change, and for a sturdier reason than a coincidence of names: they are flagd-testbed's own, from flags/disabled-flags.json, which the launchpad merges into flags/allFlags.json along with every other non-selector- file in rawflags and serves under the default configuration. RPC passing is worth stating plainly, because the appendix's own rationale for gating the tag predicts it would not: it reasons that a provider "whose backend decides, such as one speaking OFREP", cannot substitute a default the server never saw. RPC is a remote evaluator by exactly that description and substitutes anyway. flagd answers a disabled flag with reason DISABLED, no variant and the zero value of the response proto, and grpc.py:468-472 replaces that with the caller's default_value on precisely that signal. So the substitution is local even though the evaluation is not: what crosses the wire is the signal, and the provider already holds the default. In-process is the unsurprising half. flagd_core.py:143-145 returns the caller's default_value with reason Reason.DISABLED before any targeting or variant selection, and flagd_core.py:199-200 skips the type check for that reason, which is what stops a substituted default from being re-typed against the flag it did not come from. Probed directly, both resolvers resolve all four flags to the caller's default with no variant and no error code. They differ on the reason: in-process returns the SDK's Reason.DISABLED, RPC the server's bare 'DISABLED' string. The scenarios assert neither and 2.2.5 requires neither, so it is recorded rather than acted on -- it is the sort of divergence between the two resolvers this pair of suites exists to surface. The harness also seeds disabled-object-flag and cross-flagset-flag, which the canonical set deliberately does not ask for: an Object resolution needs @object as well, and a scenario needing two capability tags cannot be one row of a single outline. Signed-off-by: Simon Schrottner --- .../tests/tck/conftest.py | 13 ++++++++++- .../tests/tck/test_in_process_conformance.py | 18 +++++++++++++++ .../tests/tck/test_rpc_conformance.py | 22 +++++++++++++++++++ 3 files changed, 52 insertions(+), 1 deletion(-) diff --git a/providers/openfeature-provider-flagd/tests/tck/conftest.py b/providers/openfeature-provider-flagd/tests/tck/conftest.py index 4faaf980..0bd0bb26 100644 --- a/providers/openfeature-provider-flagd/tests/tck/conftest.py +++ b/providers/openfeature-provider-flagd/tests/tck/conftest.py @@ -14,7 +14,7 @@ ``tests/e2e``. **The testbed does not yet serve the whole canonical flag set.** The -conformance assets at spec@26362f85 ask for three flags that flagd-testbed +conformance assets at spec@009afe06 ask for three flags that flagd-testbed v3.8.0 (``openfeature/test-harness/version.txt``) does not seed: ``large-integer-flag``, ``huge-integer-flag`` and ``integral-float-flag``. Until open-feature/flagd-testbed catches up, both suites fail these scenarios @@ -45,6 +45,17 @@ ``flags/testing-flags.json`` by the launchpad's ``default`` configuration, so the three ``@targeting`` scenarios pass on both resolvers as they stand. +The four ``disabled-*`` flags, new in the canonical set at spec@009afe06, need +no testbed change either, and for a sturdier reason than a coincidence of names: +they *are* flagd-testbed's own, from ``flags/disabled-flags.json``, which the +launchpad merges into ``flags/allFlags.json`` along with every other non- +``selector-`` file in ``rawflags`` (``launchpad/pkg/json.go``) and serves under +the ``default`` configuration. So the four ``@disabled-flags`` rows pass on both +resolvers as they stand. The harness also seeds ``disabled-object-flag`` and +``cross-flagset-flag``, which the canonical set deliberately does not ask for -- +an Object resolution would need ``@object`` as well, and a scenario needing two +capability tags cannot be one row of a single outline. + The falsy flags used to fail the same way and no longer do. ``ba002ce8`` renamed them to ``boolean-zero-flag``, ``integer-zero-flag`` and ``string-zero-flag``, which is what ``flags/zero-flags.json`` in the testbed has always called them, diff --git a/providers/openfeature-provider-flagd/tests/tck/test_in_process_conformance.py b/providers/openfeature-provider-flagd/tests/tck/test_in_process_conformance.py index e629e508..00cd5320 100644 --- a/providers/openfeature-provider-flagd/tests/tck/test_in_process_conformance.py +++ b/providers/openfeature-provider-flagd/tests/tck/test_in_process_conformance.py @@ -56,6 +56,23 @@ # and in_process.py carries it into the resolution details. The ruleset is # keyed by variant, so there is always one to report. # +# DISABLED_FLAGS +# New at spec@009afe06. All four rows pass, and here that is the +# unsurprising half of the story: evaluation is local, so the resolver has +# the flag's state and the caller's default in the same call. +# flagd_core.py:143-145 returns the caller's `default_value` with reason +# Reason.DISABLED the moment a flag's state is DISABLED, before any +# targeting or variant selection. flagd_core.py:199-200 then skips the type +# check for that reason, which is what stops the substituted default from +# being re-typed against the flag it did not come from. +# +# Probed directly, each of the four flags resolves to the caller's default +# with reason Reason.DISABLED, no variant and no error code -- the SDK's +# enum, where RPC hands back the server's bare 'DISABLED' string. Neither +# is asserted by the scenarios and 2.2.5 requires neither, so the difference +# is recorded rather than acted on. It is the sort of divergence between the +# two resolvers this pair of suites exists to surface. +# # TARGETING # targeting.py:40-41 puts the evaluation context's targeting key into the # JSON-logic context under `targetingKey`, and flagd_core.py:154 evaluates @@ -129,6 +146,7 @@ Capability.CONFIGURATION_CHANGE, Capability.OBJECT, Capability.VARIANTS, + Capability.DISABLED_FLAGS, Capability.TARGETING, Capability.UNAVAILABLE_INIT, Capability.LARGE_INTEGERS, diff --git a/providers/openfeature-provider-flagd/tests/tck/test_rpc_conformance.py b/providers/openfeature-provider-flagd/tests/tck/test_rpc_conformance.py index c006fb7a..516a7f7f 100644 --- a/providers/openfeature-provider-flagd/tests/tck/test_rpc_conformance.py +++ b/providers/openfeature-provider-flagd/tests/tck/test_rpc_conformance.py @@ -60,6 +60,27 @@ # details for every typed call, and flagd names a variant for every flag in # the testbed's set. # +# DISABLED_FLAGS +# New at spec@009afe06. All four rows pass, which is worth saying plainly +# because the appendix's own rationale for gating the tag predicts they +# would not: it reasons that a provider "whose backend decides, such as one +# speaking OFREP", cannot substitute a default the server never saw. RPC is +# a remote evaluator by exactly that description, and it substitutes anyway. +# +# Measured, and then read back to find out how. flagd answers a disabled +# flag with reason DISABLED, no variant, and the zero value of the response +# proto -- ResolveBoolean's `value` field is simply unset -- and +# grpc.py:468-472 replaces that with the caller's `default_value` whenever +# the reason is DEFAULT or DISABLED and no variant came back. So the +# substitution is local even though the evaluation is not: what crosses the +# wire is the signal, and the provider already holds the default. +# +# Probed directly, each of the four flags resolves to the caller's default +# with reason 'DISABLED', no variant and no error code. The reason arrives +# as the server's bare string rather than the SDK's Reason enum, which the +# scenarios do not assert and 2.2.5 does not require -- worth noting only +# because the in-process resolver differs there, returning Reason.DISABLED. +# # TARGETING # grpc.py:492 puts the evaluation context's targeting key into the request's # context struct, so the server evaluates targeting-key-flag's rule against @@ -145,6 +166,7 @@ Capability.CONFIGURATION_CHANGE, Capability.OBJECT, Capability.VARIANTS, + Capability.DISABLED_FLAGS, Capability.TARGETING, Capability.UNAVAILABLE_INIT, Capability.LARGE_INTEGERS, From fb130d5645e53671440884ffaf06ff5bb92a3d71 Mon Sep 17 00:00:00 2001 From: Simon Schrottner Date: Sat, 12 Sep 2026 14:23:39 +0200 Subject: [PATCH 09/29] test(flagd): declare the stack instead of wiring it The TCK now owns the container lifecycle, so this adoption declares a Compose file and stops doing everything else: - the session-scoped testbed fixture, the control fixture built from a launchpad port read off the compose object, and the reach into tests.e2e.flagd_container for the container class are gone. The TCK's tck_backend fixture starts the stack, discovers the mapped host ports, builds the HttpControl against the launchpad and waits for it to accept commands; - tests/tck/docker-compose.yaml is new, and is the same stack the Java adoption uses: the unmodified testbed image, service named `backend`, three unpinned container ports. It is deliberately not the testbed's own compose file, which stands up envoy for the forbidden-endpoint e2e scenarios, names its service `flagd` and mounts a flags directory from the host. None of that serves the TCK, which drives the launchpad over HTTP; - the dev dependency becomes openfeature-tck[compose]; the extra is what pulls testcontainers in for the harness. The TCK adoption no longer imports anything from tests/e2e, which is the part that mattered beyond the line count: the two harnesses were coupled through a class whose job is the e2e suites' needs, and a change there would have broken conformance results for reasons unrelated to conformance. The provider host now comes from the endpoint rather than being written as "localhost". That was a latent bug: with a remote Docker daemon, Docker Desktop on some platforms or a rootless setup the stack is reachable at neither localhost nor anything predictable. Measured before and after, twice each: 7 failed, 87 passed, 18 skipped, identical to the run before the rewrite, scenario for scenario. Dropping Java's 50ms post-command settle -- which this suite never had -- changed nothing, so there is no flakiness to report against the testbed. The seventh failure is now recorded, which it was not. Three of the seven are the documented testbed gaps on each resolver; the fourth is on in-process alone and belongs to openfeature-flagd-core rather than to the testbed: bool is a subclass of int in Python, so the int-to-float widening in flagd_core.py sees boolean-flag as an integer and answers a Float request with 1.0 instead of the caller's default. RPC passes that row because the server type-checks it. A reader comparing "7 failed" against the three documented gaps had no way to account for the difference. Signed-off-by: Simon Schrottner --- .../openfeature-provider-flagd/pyproject.toml | 12 +- .../tests/tck/conftest.py | 135 ++++++++++-------- .../tests/tck/docker-compose.yaml | 27 ++++ .../tests/tck/suite.py | 51 +++++-- .../tests/tck/test_in_process_conformance.py | 27 ++-- .../tests/tck/test_rpc_conformance.py | 27 ++-- uv.lock | 4 +- 7 files changed, 182 insertions(+), 101 deletions(-) create mode 100644 providers/openfeature-provider-flagd/tests/tck/docker-compose.yaml diff --git a/providers/openfeature-provider-flagd/pyproject.toml b/providers/openfeature-provider-flagd/pyproject.toml index 7cb0492d..3dba7a43 100644 --- a/providers/openfeature-provider-flagd/pyproject.toml +++ b/providers/openfeature-provider-flagd/pyproject.toml @@ -35,10 +35,12 @@ dev = [ "coverage[toml]>=7.10.0,<8.0.0", "grpcio-health-checking>=1.82.1,<2.0.0", "mypy>=1.18.0,<2.0.0", - # The OpenFeature provider conformance suite. Ships the feature files, the flag - # set and the control-API client, and registers its step definitions through a - # pytest11 entry point, so tests/tck needs no conftest of its own for them. - "openfeature-provider-tck", + # The OpenFeature conformance suite. Ships the feature files, the flag set, + # the control-API client and the Compose harness that owns the container + # stack, and registers its step definitions through a pytest11 entry point -- + # so tests/tck declares a Compose file and nothing else. The `compose` extra + # is what pulls testcontainers in for the harness. + "openfeature-tck[compose]", "poethepoet>=0.37.0", "pytest>=9.0.0,<10.0.0", "pytest-bdd>=8.1.0,<9.0.0", @@ -116,7 +118,7 @@ warn_unused_ignores = false [tool.uv.sources] openfeature-flagd-core = { workspace = true } -openfeature-provider-tck = { workspace = true } +openfeature-tck = { workspace = true } [tool.pytest] strict = true diff --git a/providers/openfeature-provider-flagd/tests/tck/conftest.py b/providers/openfeature-provider-flagd/tests/tck/conftest.py index 0bd0bb26..b7d2d080 100644 --- a/providers/openfeature-provider-flagd/tests/tck/conftest.py +++ b/providers/openfeature-provider-flagd/tests/tck/conftest.py @@ -1,24 +1,42 @@ -"""One testbed stack and one control, shared by both conformance suites. - -The stack is started once per session and **never restarted**. Scenario -isolation comes from the control API instead, because container orchestrators -assign host ports dynamically and cannot reliably preserve them across a -restart: a restarted backend comes back on a different host port, silently -invalidating every provider already pointed at the old one, and the failure -looks like a flaky provider rather than a broken test. See the -no-container-restart invariant in the TCK's ``control-api.yaml``. - -``flagd-testbed`` is not modified and the existing e2e suites are untouched: the -TCK drives the testbed's launchpad through the standardised control API, which -the launchpad already implements, and reuses the container lifecycle already in -``tests/e2e``. - -**The testbed does not yet serve the whole canonical flag set.** The -conformance assets at spec@009afe06 ask for three flags that flagd-testbed -v3.8.0 (``openfeature/test-harness/version.txt``) does not seed: -``large-integer-flag``, ``huge-integer-flag`` and ``integral-float-flag``. -Until open-feature/flagd-testbed catches up, both suites fail these scenarios -with ``FLAG_NOT_FOUND``, for every resolver alike: +"""One testbed stack, declared rather than wired, and shared by both suites. + +The whole of the container lifecycle belongs to the TCK now: it starts the +Compose file once per session, discovers the dynamically mapped host ports, +builds the ``HttpControl`` against the launchpad and waits until it accepts +commands, and tears the stack down after the last scenario. What is left here is +the declaration -- which Compose file, which ports the provider connects to -- +and one free port for the scenarios that need a backend that is not there. + +The stack is started once per session and **never restarted** -- container +orchestrators assign host ports dynamically and cannot reliably preserve them +across a restart, so a restarted backend comes back on a different host port, +silently invalidating every provider already pointed at the old one, and the +failure looks like a flaky provider rather than a broken test. Scenario isolation +and every simulated outage go through the control API instead. + +One stack for both suites because one flagd process serves both ports the +resolvers use -- 8013 for RPC and 8015 for sync -- so there is nothing a second +stack would isolate. One ``HttpControl`` with it, which matters and is not merely +tidy: the control tracks whether a disconnect has left the backend down so the +next scenario starts it rather than merely resetting flag state, and two +instances would each hold half of that knowledge. A session-scoped +``tck_backend`` is what makes both true by construction. + +The launchpad registers only ``/start``, ``/restart``, ``/stop`` and ``/change`` +(flagd-testbed ``launchpad/main.go``), so ``/reset`` answers 404 and every +``prepare_scenario`` takes the documented ``/start`` fallback. The probe costs +one 404 for the whole session. It serves no ``/healthz`` either, which the +control API document defines as ready -- so the readiness wait rests on the +control port accepting a connection, which the harness establishes before it +probes. + +**The testbed does not yet serve the whole canonical flag set.** The conformance +assets at spec@009afe06 ask for three flags that flagd-testbed v3.8.0 +(``openfeature/test-harness/version.txt``, and the tag pinned in +``docker-compose.yaml`` beside this file) does not seed: ``large-integer-flag``, +``huge-integer-flag`` and ``integral-float-flag``. Until +open-feature/flagd-testbed catches up, both suites fail these scenarios with +``FLAG_NOT_FOUND``, for every resolver alike: * ``A large integer resolves without loss of precision`` -- untagged; * ``An integer beyond 32 bits resolves without loss of precision`` -- under @@ -39,6 +57,23 @@ the backend that is behind. None of them is a ``KnownDeviation`` either: a deviation is for a behaviour the *provider* is required to have and does not. +**The seventh failure is the provider's, not the testbed's.** A full run is +``7 failed, 87 passed, 18 skipped`` -- three of the failures above on each +resolver, and one more on in-process alone: ``boolean-flag`` requested as a +Float resolves to ``1.0`` with reason ``STATIC`` and no error code, where the +mandatory wrong-type scenario asks for the caller's default. ``bool`` is a +subclass of ``int`` in Python, so the widening at ``flagd_core.py:113-114`` -- +``if isinstance(result.value, int): result.value = float(result.value)`` -- sees +a boolean as an integer, after ``_resolve`` has already let it through for a +Float request. It is the same shape as the boolean-satisfies-an-Integer finding +the suite's own README records against the in-memory provider, and it is a gap +in ``openfeature-flagd-core`` rather than in either resolver's transport: RPC +passes the row, because the server type-checks it. Recorded here rather than +declared as a ``KnownDeviation`` because the scenario is mandatory and +ungated -- it fails visibly on every run, which is the report, and a deviation +would add nothing a reader cannot already see. It is not a testbed gap and does +not go away when the image is bumped. + ``targeting-key-flag``, new in the canonical set at the same revision, needs no testbed change. It is the flag flagd-testbed's own ``targeting.feature`` already uses -- same key, same ``hit``/``miss`` variants, same uuid -- seeded from @@ -66,56 +101,44 @@ from __future__ import annotations import socket -import typing +from pathlib import Path import pytest -from openfeature.contrib.tools.provider_tck import HttpControl -from tests.e2e.flagd_container import FlagdContainer +from openfeature.contrib.tools.tck import ComposeBackend, RunningBackend +from tests.tck.suite import IN_PROCESS_PORT, RPC_PORT @pytest.fixture(scope="session") -def flagd_testbed() -> typing.Iterator[FlagdContainer]: - """The testbed stack, up for the whole session. - - One stack for both suites because one flagd process serves both ports the - resolvers use -- 8013 for RPC and 8015 for sync -- so there is nothing a - second stack would isolate. - """ - container = FlagdContainer() - container.start() - try: - yield container - finally: - container.stop() +def compose_backend() -> ComposeBackend: + """The stack under test, as the TCK's ``tck_backend`` fixture wants it. + Both resolver ports are declared even though each suite uses one of them, + because both suites share this stack and the harness checks at startup that + the Compose file publishes everything it was told about. The launchpad's own + 8080 is exposed automatically and must not be listed. -@pytest.fixture(scope="session") -def flagd_control(flagd_testbed: FlagdContainer) -> HttpControl: - """The control API client, shared by both suites. - - Shared rather than one per suite, and that matters: the two suites drive the - *same* backend, and ``HttpControl`` tracks whether a disconnect has left it - down so the next scenario starts it rather than merely resetting flag state. - Two instances would each hold half of that knowledge. - - The launchpad registers only ``/start``, ``/restart``, ``/stop`` and - ``/change`` (flagd-testbed ``launchpad/main.go``), so ``/reset`` answers 404 - and every ``prepare_scenario`` takes the documented ``/start`` fallback. The - probe costs one 404 for the whole session. + The path is absolute rather than relative to the package directory -- which + is what the harness resolves a relative one against, and where ``poe test`` + runs from -- so that running pytest from the repository root works too. """ - return HttpControl(flagd_testbed.get_launchpad_url()) + return ComposeBackend( + compose_file=Path(__file__).parent / "docker-compose.yaml", + backend_ports=[RPC_PORT, IN_PROCESS_PORT], + ) @pytest.fixture(scope="session") -def closed_port(flagd_testbed: FlagdContainer) -> int: +def closed_port(tck_backend: RunningBackend) -> int: """A port on localhost with nothing listening, for the ``@unavailable`` scenarios. Discovered by binding and releasing rather than hard-coded, because the - testbed's own host ports are mapped dynamically and a hard-coded number - could collide with one. Depending on ``flagd_testbed`` orders this after the - stack has taken its ports, which is what makes the remaining race - negligible. + testbed's own host ports are mapped dynamically and a hard-coded number could + collide with one. Depending on ``tck_backend`` orders this after the stack has + taken its ports, which is what makes the remaining race negligible. + + Deliberately not a port on the Compose stack: that has to stay up for the + whole session, and simulated outages belong to the control API. """ with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as probe: probe.bind(("127.0.0.1", 0)) diff --git a/providers/openfeature-provider-flagd/tests/tck/docker-compose.yaml b/providers/openfeature-provider-flagd/tests/tck/docker-compose.yaml new file mode 100644 index 00000000..0887be9a --- /dev/null +++ b/providers/openfeature-provider-flagd/tests/tck/docker-compose.yaml @@ -0,0 +1,27 @@ +# Backend stack for the OpenFeature TCK, wrapping the unmodified flagd testbed image. +# +# The image serves flagd itself and the "launchpad" control API on 8080, whose endpoints the +# TCK's control API contract was derived from. Nothing here is flagd-testbed's own compose +# file: that one stands up envoy as well, for the forbidden-endpoint e2e scenarios, names its +# service `flagd`, and mounts a flags directory from the host. The TCK needs none of that -- +# it drives the launchpad over HTTP -- so this is the smallest stack that serves the suite, +# and it is the same one the Java adoption uses. +# +# The service is named `backend` because that is the name every language's TCK defaults to. A +# stack that calls it something else is fine, and says so through +# `ComposeBackend(backend_service=...)`. +# +# Note there are no host port bindings. The TCK requires dynamically mapped ports and +# discovers them after startup -- a pinned host port would make the suite unrunnable in +# parallel and would collide with a developer's local flagd. +# +# The image does not yet serve the whole canonical flag set; tests/tck/conftest.py says which +# flags are missing and which scenarios that fails. Bump the tag when +# open-feature/flagd-testbed#392 lands. +services: + backend: + image: ghcr.io/open-feature/flagd-testbed:v3.8.0 + ports: + - 8013 # flagd RPC evaluation (gRPC) + - 8015 # flagd in-process sync (gRPC) + - 8080 # launchpad control API diff --git a/providers/openfeature-provider-flagd/tests/tck/suite.py b/providers/openfeature-provider-flagd/tests/tck/suite.py index 2066ba6f..aba88c5b 100644 --- a/providers/openfeature-provider-flagd/tests/tck/suite.py +++ b/providers/openfeature-provider-flagd/tests/tck/suite.py @@ -9,6 +9,11 @@ Everything they share lives here; everything that differs lives in the two ``test_*_conformance`` modules next to it, where a reader can see the whole of a resolver's declaration in one place. + +There is no container wiring here any more. The TCK owns the stack -- see +``conftest.py`` -- so what is left is the timings, which are flagd's own and +interact, and the one function that turns a resolver plus a running endpoint into +a ``TckConfig``. """ from __future__ import annotations @@ -17,15 +22,23 @@ from openfeature.contrib.provider.flagd import FlagdProvider from openfeature.contrib.provider.flagd.config import ResolverType -from openfeature.contrib.tools.provider_tck import ( +from openfeature.contrib.tools.tck import ( Capability, - HttpControl, + RunningBackend, TckConfig, ) from openfeature.provider import FeatureProvider -from tests.e2e.flagd_container import FlagdContainer -__all__ = ["ResolverSuite", "build_config"] +__all__ = ["IN_PROCESS_PORT", "RPC_PORT", "ResolverSuite", "build_config"] + +RPC_PORT = 8013 +IN_PROCESS_PORT = 8015 +"""Container-internal ports flagd serves remote evaluation and the sync stream on. + +Both are declared in one ``ComposeBackend`` because one flagd process serves both +and both suites share the stack. Named here rather than in ``conftest.py`` so +that the declaration and the resolver that uses it cannot drift apart. +""" # Timings. flagd exposes several and they interact, so they are named here once # rather than scattered through two suites. @@ -104,6 +117,12 @@ class ResolverSuite: resolver_type: ResolverType + backend_port: int + """The container-internal port this resolver connects to. + + Resolved to a host port through the endpoint, after the stack is up. + """ + capabilities: frozenset[Capability] """What this resolver was run against the suite and seen to satisfy. @@ -119,24 +138,28 @@ class ResolverSuite: def build_config( suite: ResolverSuite, - container: FlagdContainer, - control: HttpControl, + backend: RunningBackend, closed_port: int, ) -> TckConfig: """Wire one resolver up to the running testbed. - The ports are read here, after the stack is up: the testbed maps host ports - dynamically, so they do not exist earlier -- and they stay valid for the - whole session because nothing ever restarts a container. Outages are - simulated inside the running stack through ``control`` instead. + The host and port are read off ``backend.endpoint`` here, inside the factory + and after the stack is up: Compose maps host ports dynamically, so they do + not exist earlier -- and they stay valid for the whole session because + nothing ever restarts a container. Outages are simulated inside the running + stack through ``backend.control`` instead. + + The host comes from the endpoint rather than being written as + ``"localhost"``: with a remote Docker daemon, Docker Desktop on some + platforms or a rootless setup it is neither localhost nor predictable. """ - port = container.get_port(suite.resolver_type) + endpoint = backend.endpoint def new_provider() -> FeatureProvider: return FlagdProvider( resolver_type=suite.resolver_type, - host="localhost", - port=port, + host=endpoint.host, + port=endpoint.port(suite.backend_port), deadline_ms=DEADLINE_MS, stream_deadline_ms=STREAM_DEADLINE_MS, retry_backoff_ms=RETRY_BACKOFF_MS, @@ -161,7 +184,7 @@ def new_unavailable_provider() -> FeatureProvider: return TckConfig( name=suite.name, - control=control, + control=backend.control, new_provider=new_provider, new_unavailable_provider=new_unavailable_provider, capabilities=suite.capabilities, diff --git a/providers/openfeature-provider-flagd/tests/tck/test_in_process_conformance.py b/providers/openfeature-provider-flagd/tests/tck/test_in_process_conformance.py index 00cd5320..1f81dd1a 100644 --- a/providers/openfeature-provider-flagd/tests/tck/test_in_process_conformance.py +++ b/providers/openfeature-provider-flagd/tests/tck/test_in_process_conformance.py @@ -14,14 +14,13 @@ from pytest_bdd import scenarios from openfeature.contrib.provider.flagd.config import ResolverType -from openfeature.contrib.tools.provider_tck import ( +from openfeature.contrib.tools.tck import ( Capability, - HttpControl, + RunningBackend, TckConfig, - features_path, + feature_paths, ) -from tests.e2e.flagd_container import FlagdContainer -from tests.tck.suite import ResolverSuite, build_config +from tests.tck.suite import IN_PROCESS_PORT, ResolverSuite, build_config # Every capability below was declared, the suite run, and the scenarios seen to # pass. The code references say where the behaviour lives, so a reader can check @@ -156,6 +155,7 @@ IN_PROCESS_SUITE = ResolverSuite( name="flagd-in-process", resolver_type=ResolverType.IN_PROCESS, + backend_port=IN_PROCESS_PORT, capabilities=IN_PROCESS_CAPABILITIES, # In-process transfers and applies the whole ruleset before reporting ready, # so it needs more headroom than RPC. @@ -164,12 +164,15 @@ @pytest.fixture(scope="session") -def tck_config( - flagd_testbed: FlagdContainer, - flagd_control: HttpControl, - closed_port: int, -) -> TckConfig: - return build_config(IN_PROCESS_SUITE, flagd_testbed, flagd_control, closed_port) +def tck_config(tck_backend: RunningBackend, closed_port: int) -> TckConfig: + """The whole of this adoption's wiring. + ``tck_backend`` is the TCK's own session-scoped fixture: it has already + started the Compose file ``tests/tck/conftest.py`` declares, discovered the + dynamically mapped host ports, built the control against the launchpad and + waited for it to accept commands. + """ + return build_config(IN_PROCESS_SUITE, tck_backend, closed_port) -scenarios(features_path()) + +scenarios(*feature_paths()) diff --git a/providers/openfeature-provider-flagd/tests/tck/test_rpc_conformance.py b/providers/openfeature-provider-flagd/tests/tck/test_rpc_conformance.py index 516a7f7f..6c535eb0 100644 --- a/providers/openfeature-provider-flagd/tests/tck/test_rpc_conformance.py +++ b/providers/openfeature-provider-flagd/tests/tck/test_rpc_conformance.py @@ -11,14 +11,13 @@ from pytest_bdd import scenarios from openfeature.contrib.provider.flagd.config import ResolverType -from openfeature.contrib.tools.provider_tck import ( +from openfeature.contrib.tools.tck import ( Capability, - HttpControl, + RunningBackend, TckConfig, - features_path, + feature_paths, ) -from tests.e2e.flagd_container import FlagdContainer -from tests.tck.suite import ResolverSuite, build_config +from tests.tck.suite import RPC_PORT, ResolverSuite, build_config # Every capability below was declared, the suite run, and the scenarios seen to # pass. The code references say where the behaviour lives, so a reader can check @@ -176,6 +175,7 @@ RPC_SUITE = ResolverSuite( name="flagd-rpc", resolver_type=ResolverType.RPC, + backend_port=RPC_PORT, capabilities=RPC_CAPABILITIES, # RPC holds no ruleset of its own: it is ready as soon as the evaluation # stream is up, so it needs less headroom than in-process. @@ -184,12 +184,15 @@ @pytest.fixture(scope="session") -def tck_config( - flagd_testbed: FlagdContainer, - flagd_control: HttpControl, - closed_port: int, -) -> TckConfig: - return build_config(RPC_SUITE, flagd_testbed, flagd_control, closed_port) +def tck_config(tck_backend: RunningBackend, closed_port: int) -> TckConfig: + """The whole of this adoption's wiring. + ``tck_backend`` is the TCK's own session-scoped fixture: it has already + started the Compose file ``tests/tck/conftest.py`` declares, discovered the + dynamically mapped host ports, built the control against the launchpad and + waited for it to accept commands. + """ + return build_config(RPC_SUITE, tck_backend, closed_port) -scenarios(features_path()) + +scenarios(*feature_paths()) diff --git a/uv.lock b/uv.lock index 98b43a43..8017df40 100644 --- a/uv.lock +++ b/uv.lock @@ -1886,7 +1886,7 @@ dev = [ { name = "coverage", extra = ["toml"] }, { name = "grpcio-health-checking" }, { name = "mypy" }, - { name = "openfeature-provider-tck" }, + { name = "openfeature-tck", extra = ["compose"] }, { name = "poethepoet" }, { name = "pytest" }, { name = "pytest-bdd" }, @@ -1912,7 +1912,7 @@ dev = [ { name = "coverage", extras = ["toml"], specifier = ">=7.10.0,<8.0.0" }, { name = "grpcio-health-checking", specifier = ">=1.82.1,<2.0.0" }, { name = "mypy", specifier = ">=1.18.0,<2.0.0" }, - { name = "openfeature-provider-tck", editable = "tools/openfeature-provider-tck" }, + { name = "openfeature-tck", extras = ["compose"], editable = "tools/openfeature-tck" }, { name = "poethepoet", specifier = ">=0.37.0" }, { name = "pytest", specifier = ">=9.0.0,<10.0.0" }, { name = "pytest-bdd", specifier = ">=8.1.0,<9.0.0" }, From 2030a6a2af1f1a90434c1290ce624a394d85cfe3 Mon Sep 17 00:00:00 2001 From: Simon Schrottner Date: Sat, 12 Sep 2026 16:48:01 +0200 Subject: [PATCH 10/29] test(flagd): keep the conformance suite out of the default build, and say so The TCK suite is excluded from `poe test` and `poe cov` and gets a task of its own, `poe test-tck`, which a maintainer runs before merging a change to it. The README says the same, with the reason and the current tally. It was running in the default build until now, and that is worth being plain about: the premise this was handed to me on -- that no CI job in any language executes a containerised suite -- is false for Python. `poe cov` is `pytest tests`, which collected 112 TCK tests alongside the 576 long-standing `tests/e2e` ones, on a runner that has Docker. So the flagd branch's build was red, for the same reason Java's ofrep one was. Docker is therefore not the reason for the exclusion; `tests/e2e` needs it too and keeps running. The reason is that a conformance suite reports what is true of the stack under test, and a full run is 7 failed, 87 passed, 18 skipped: three canonical flags flagd-testbed v3.8.0 does not seed, failing on each resolver, plus the `openfeature-flagd-core` widening that answers a Float request for `boolean-flag` with 1.0. Those failures are the report. A gate that has to be green cannot hold a suite whose honest output is red, and an xfail to make it green would blame the provider for the backend. Also follows the base's re-pin: the paragraph naming the revision whose assets ask for the three unseeded flags now names 93eb1a58. The two spec@009afe06 mentions that remain are provenance -- the revision a flag was introduced at -- and stay true. Signed-off-by: Simon Schrottner --- .../openfeature-provider-flagd/README.md | 28 +++++++++++++++++++ .../openfeature-provider-flagd/pyproject.toml | 16 +++++++++-- .../tests/tck/conftest.py | 2 +- 3 files changed, 43 insertions(+), 3 deletions(-) diff --git a/providers/openfeature-provider-flagd/README.md b/providers/openfeature-provider-flagd/README.md index fbade51f..f77f724c 100644 --- a/providers/openfeature-provider-flagd/README.md +++ b/providers/openfeature-provider-flagd/README.md @@ -182,6 +182,34 @@ api.set_provider(FlagdProvider( )) ``` +## Provider conformance suite + +This provider runs the [OpenFeature Provider Conformance Suite][tck] against a flagd-testbed stack, +once per resolver, in `tests/tck`. The suite owns the container stack: `tests/tck/conftest.py` +declares a Compose file and the two ports the resolvers connect to, and nothing else. + +**It is excluded from the default build, and a maintainer runs it by hand before merging a change to +it.** + +``` +poe test-tck # both resolvers, needs Docker +poe test # everything else, which is what CI runs +``` + +The exclusion is a decision rather than an oversight, so here is the reason. It is not Docker — +`tests/e2e` needs Docker too and does run in the default build. It is that a conformance suite +reports what is true of the *stack* under test, and a full run today is **7 failed, 87 passed, 18 +skipped**: three canonical flags that flagd-testbed v3.8.0 does not seed yet, failing on each +resolver, plus one genuine provider-side gap in `openfeature-flagd-core`. Those failures are the +report. A gate that has to be green cannot hold a suite whose honest output is red, and an `xfail` +to make it green would say the provider is at fault where the backend is. + +`tests/tck/conftest.py` accounts for all seven, individually, with the flag or requirement each one +turns on — so a reviewer running the suite can tell a new failure from a known one, and the number +above is what to expect. + +[tck]: ../../tools/openfeature-tck/README.md + ## License Apache 2.0 - See [LICENSE](./LICENSE) for more information. diff --git a/providers/openfeature-provider-flagd/pyproject.toml b/providers/openfeature-provider-flagd/pyproject.toml index 3dba7a43..86db79c3 100644 --- a/providers/openfeature-provider-flagd/pyproject.toml +++ b/providers/openfeature-provider-flagd/pyproject.toml @@ -124,8 +124,20 @@ openfeature-tck = { workspace = true } strict = true [tool.poe.tasks] -test = "pytest tests" -test-cov = "coverage run -m pytest tests" +# `tests/tck` is excluded from the default build on purpose, and a maintainer +# runs `poe test-tck` by hand before merging a change to it. The suite needs +# Docker, which `tests/e2e` does too -- so that is not the reason. The reason is +# that a conformance suite reports what is true of the *stack* under test, and a +# full run is 7 failed, 87 passed, 18 skipped: six of those failures are three +# canonical flags flagd-testbed does not seed yet, on each of the two resolvers, +# and the seventh is a real provider-side gap. Those failures are the report. A +# gate that has to be green cannot hold a suite whose honest output is red, and +# an xfail to make it green would say the provider is at fault when the backend +# is. See tests/tck/conftest.py, which accounts for all seven, and the TCK's own +# README for the policy. +test = "pytest tests --ignore=tests/tck" +test-cov = "coverage run -m pytest tests --ignore=tests/tck" +test-tck = "pytest tests/tck" cov-report = "coverage xml" cov = [ "test-cov", diff --git a/providers/openfeature-provider-flagd/tests/tck/conftest.py b/providers/openfeature-provider-flagd/tests/tck/conftest.py index b7d2d080..58101693 100644 --- a/providers/openfeature-provider-flagd/tests/tck/conftest.py +++ b/providers/openfeature-provider-flagd/tests/tck/conftest.py @@ -31,7 +31,7 @@ probes. **The testbed does not yet serve the whole canonical flag set.** The conformance -assets at spec@009afe06 ask for three flags that flagd-testbed v3.8.0 +assets at spec@93eb1a58 ask for three flags that flagd-testbed v3.8.0 (``openfeature/test-harness/version.txt``, and the tag pinned in ``docker-compose.yaml`` beside this file) does not seed: ``large-integer-flag``, ``huge-integer-flag`` and ``integral-float-flag``. Until From 81a211db25290346a02fc443871596ff7207749c Mon Sep 17 00:00:00 2001 From: Simon Schrottner Date: Sun, 13 Sep 2026 00:08:20 +0200 Subject: [PATCH 11/29] test(flagd): point at Appendix F for the policy, and keep the suite importing The README and the pyproject comment both carried the reasoning for excluding a conformance suite from the default build, in this package's own words. That reasoning is now Appendix F's "Running the suite in CI", so both point at it instead. Four READMEs paraphrasing one decision is how the known-deviation and control-path decisions came to have three different answers. What stays is what is local: where the exclusion lives, the command a maintainer runs, the current tally with what each of the seven failures is, and that Docker is not what decides the exclusion here -- tests/e2e needs Docker too and does run in the default build. Appendix F also asks that an excluded suite keep building even when it does not run, because one that has quietly stopped building against its harness is worse than one that runs and fails. `--ignore` does not import the suite at all, and mypy in this package is configured over `src`, so nothing checked that. `poe test` and `poe test-cov` now end in a collect-only pass over tests/tck, which imports every test module and resolves the feature files without starting a container -- about a second. The two default tasks become sequences, so their commands move to `test-default` and `test-cov-default`. `poe cov`, which is what build.yml runs, is unchanged in name and now covers the collect as well. Signed-off-by: Simon Schrottner --- .../openfeature-provider-flagd/README.md | 28 +++++++++------- .../openfeature-provider-flagd/pyproject.toml | 32 ++++++++++++------- 2 files changed, 37 insertions(+), 23 deletions(-) diff --git a/providers/openfeature-provider-flagd/README.md b/providers/openfeature-provider-flagd/README.md index f77f724c..6a518666 100644 --- a/providers/openfeature-provider-flagd/README.md +++ b/providers/openfeature-provider-flagd/README.md @@ -196,19 +196,25 @@ poe test-tck # both resolvers, needs Docker poe test # everything else, which is what CI runs ``` -The exclusion is a decision rather than an oversight, so here is the reason. It is not Docker — -`tests/e2e` needs Docker too and does run in the default build. It is that a conformance suite -reports what is true of the *stack* under test, and a full run today is **7 failed, 87 passed, 18 -skipped**: three canonical flags that flagd-testbed v3.8.0 does not seed yet, failing on each -resolver, plus one genuine provider-side gap in `openfeature-flagd-core`. Those failures are the -report. A gate that has to be green cannot hold a suite whose honest output is red, and an `xfail` -to make it green would say the provider is at fault where the backend is. - -`tests/tck/conftest.py` accounts for all seven, individually, with the flag or requirement each one -turns on — so a reviewer running the suite can tell a new failure from a known one, and the number -above is what to expect. +The exclusion lives in `pyproject.toml`: `--ignore=tests/tck` on the two tasks `build.yml` reaches, +with the reason in a comment above them. Why a conformance suite is not a required gate is +[Appendix F, "Running the suite in CI"][appendix-f], and is not restated here. + +Two things that are this provider's rather than the policy's: + +- **Docker is not what decides it.** `tests/e2e` needs Docker too and does run in the default build. + What decides it is the run: **7 failed, 87 passed, 18 skipped**, being three canonical flags that + flagd-testbed v3.8.0 does not seed yet, failing on each resolver, plus one genuine provider-side + gap in `openfeature-flagd-core`. `tests/tck/conftest.py` accounts for all seven individually, with + the flag or requirement each one turns on, so a reviewer running the suite can tell a new failure + from a known one and the number above is what to expect. +- **The default build still collects the suite** — `poe test` and `poe test-cov` end in + `pytest tests/tck --collect-only`, which imports every module and starts no container. An excluded + suite that has quietly stopped importing against the harness is worse than one that runs and + fails, and `mypy` here is configured over `src` alone, so nothing else would notice. [tck]: ../../tools/openfeature-tck/README.md +[appendix-f]: https://github.com/open-feature/spec/blob/main/specification/appendix-f-provider-conformance.md ## License diff --git a/providers/openfeature-provider-flagd/pyproject.toml b/providers/openfeature-provider-flagd/pyproject.toml index 86db79c3..d91cdae4 100644 --- a/providers/openfeature-provider-flagd/pyproject.toml +++ b/providers/openfeature-provider-flagd/pyproject.toml @@ -125,19 +125,27 @@ strict = true [tool.poe.tasks] # `tests/tck` is excluded from the default build on purpose, and a maintainer -# runs `poe test-tck` by hand before merging a change to it. The suite needs -# Docker, which `tests/e2e` does too -- so that is not the reason. The reason is -# that a conformance suite reports what is true of the *stack* under test, and a -# full run is 7 failed, 87 passed, 18 skipped: six of those failures are three -# canonical flags flagd-testbed does not seed yet, on each of the two resolvers, -# and the seventh is a real provider-side gap. Those failures are the report. A -# gate that has to be green cannot hold a suite whose honest output is red, and -# an xfail to make it green would say the provider is at fault when the backend -# is. See tests/tck/conftest.py, which accounts for all seven, and the TCK's own -# README for the policy. -test = "pytest tests --ignore=tests/tck" -test-cov = "coverage run -m pytest tests --ignore=tests/tck" +# runs `poe test-tck` by hand before merging a change to it. Why a conformance +# suite is not a required gate is Appendix F, "Running the suite in CI" -- +# linked from tools/openfeature-tck/README.md -- and is deliberately not +# restated here. +# +# What is local to this package: a full run is 7 failed, 87 passed, 18 skipped. +# Six of those failures are three canonical flags flagd-testbed does not seed +# yet, on each of the two resolvers, and the seventh is a real provider-side +# gap; tests/tck/conftest.py accounts for all seven. Docker is not what decides +# the exclusion -- `tests/e2e` needs Docker too and does run here. +# +# The suite is still *collected* on every default build, so it cannot quietly +# stop importing against the harness while nobody runs it. `--ignore` would +# otherwise leave nothing checking that, since mypy here is configured over +# `src` alone. +test = ["test-default", "test-tck-collect"] +test-cov = ["test-cov-default", "test-tck-collect"] +test-default = "pytest tests --ignore=tests/tck" +test-cov-default = "coverage run -m pytest tests --ignore=tests/tck" test-tck = "pytest tests/tck" +test-tck-collect = "pytest tests/tck --collect-only -q" cov-report = "coverage xml" cov = [ "test-cov", From e94112c0a9c25b7dff7e805bec54aaf2d1a01a4a Mon Sep 17 00:00:00 2001 From: Simon Schrottner Date: Sun, 13 Sep 2026 10:01:31 +0200 Subject: [PATCH 12/29] test(flagd): both resolvers claim the standard resolution reasons spec@c342461a moved every resolution-reason assertion into a new gated gherkin/reason.feature, so the reasons are now a claim a provider declares rather than a house rule every adopter is held to. Declare @standard-reasons on both resolvers. Declared on a run rather than on the source, as Appendix F requires. All nine scenarios pass on each resolver: the four rule-less rows as STATIC, an unknown flag and a type mismatch as ERROR beside their error codes, TARGETING_MATCH for the matched rule and DEFAULT for the miss, DISABLED for a disabled flag. The last three compose with @targeting and @disabled-flags, both of which these suites already declare, so none of the file is skipped here. Worth recording that the DISABLED row passes on RPC despite the reason arriving as flagd's bare string rather than the SDK's Reason enum -- the difference the DISABLED_FLAGS note already called out. The step compares the reason as text, so in-process's Reason.DISABLED and RPC's 'DISABLED' are the same answer. The tally moves from 7 failed, 87 passed, 18 skipped to 7 failed, 105 passed, 18 skipped: eighteen new passing scenarios, nine per resolver, and no new failure. The same seven failures as before -- three canonical flags flagd-testbed v3.8.0 does not seed, on each resolver, plus the boolean-as-Float widening in openfeature-flagd-core. Collection moves from 112 to 130. Signed-off-by: Simon Schrottner --- .../openfeature-provider-flagd/README.md | 2 +- .../tests/tck/conftest.py | 4 ++-- .../tests/tck/test_in_process_conformance.py | 19 +++++++++++++++++ .../tests/tck/test_rpc_conformance.py | 21 +++++++++++++++++++ 4 files changed, 43 insertions(+), 3 deletions(-) diff --git a/providers/openfeature-provider-flagd/README.md b/providers/openfeature-provider-flagd/README.md index 6a518666..a80adc5e 100644 --- a/providers/openfeature-provider-flagd/README.md +++ b/providers/openfeature-provider-flagd/README.md @@ -203,7 +203,7 @@ with the reason in a comment above them. Why a conformance suite is not a requir Two things that are this provider's rather than the policy's: - **Docker is not what decides it.** `tests/e2e` needs Docker too and does run in the default build. - What decides it is the run: **7 failed, 87 passed, 18 skipped**, being three canonical flags that + What decides it is the run: **7 failed, 105 passed, 18 skipped**, being three canonical flags that flagd-testbed v3.8.0 does not seed yet, failing on each resolver, plus one genuine provider-side gap in `openfeature-flagd-core`. `tests/tck/conftest.py` accounts for all seven individually, with the flag or requirement each one turns on, so a reviewer running the suite can tell a new failure diff --git a/providers/openfeature-provider-flagd/tests/tck/conftest.py b/providers/openfeature-provider-flagd/tests/tck/conftest.py index 58101693..5774cd6d 100644 --- a/providers/openfeature-provider-flagd/tests/tck/conftest.py +++ b/providers/openfeature-provider-flagd/tests/tck/conftest.py @@ -31,7 +31,7 @@ probes. **The testbed does not yet serve the whole canonical flag set.** The conformance -assets at spec@93eb1a58 ask for three flags that flagd-testbed v3.8.0 +assets at spec@c342461a ask for three flags that flagd-testbed v3.8.0 (``openfeature/test-harness/version.txt``, and the tag pinned in ``docker-compose.yaml`` beside this file) does not seed: ``large-integer-flag``, ``huge-integer-flag`` and ``integral-float-flag``. Until @@ -58,7 +58,7 @@ deviation is for a behaviour the *provider* is required to have and does not. **The seventh failure is the provider's, not the testbed's.** A full run is -``7 failed, 87 passed, 18 skipped`` -- three of the failures above on each +``7 failed, 105 passed, 18 skipped`` -- three of the failures above on each resolver, and one more on in-process alone: ``boolean-flag`` requested as a Float resolves to ``1.0`` with reason ``STATIC`` and no error code, where the mandatory wrong-type scenario asks for the caller's default. ``bool`` is a diff --git a/providers/openfeature-provider-flagd/tests/tck/test_in_process_conformance.py b/providers/openfeature-provider-flagd/tests/tck/test_in_process_conformance.py index 1f81dd1a..07143892 100644 --- a/providers/openfeature-provider-flagd/tests/tck/test_in_process_conformance.py +++ b/providers/openfeature-provider-flagd/tests/tck/test_in_process_conformance.py @@ -133,6 +133,24 @@ # before it: it is one of the six @lifecycle skips each resolver reports, # not a scenario that used to pass. # +# STANDARD_REASONS +# New at spec@c342461a, which moved every resolution-reason assertion out of +# the other feature files and into reason.feature, gated as a whole. A claim +# rather than an exemption: 2.2.5 is a SHOULD that permits "some other +# string", so declaring the tag says this provider uses the standard +# vocabulary with the standard meanings. +# +# Declared on a run rather than on the source. All nine scenarios pass -- +# the four rule-less rows as STATIC, an unknown flag and a type mismatch as +# ERROR beside their error codes, TARGETING_MATCH for the matched rule and +# DEFAULT for the miss, DISABLED for a disabled flag. The last three need +# @targeting and @disabled-flags as well, which this suite declares, so none +# of the file is skipped here. +# +# The DISABLED row resolves through Reason.DISABLED here, the SDK's own +# enum, where RPC hands back flagd's bare 'DISABLED' string -- noted under +# DISABLED_FLAGS above. The step compares the reason as text, so both pass. +# # CACHING # Reserved in the Capability enum; no scenario carries the tag. Declaring a # capability nothing exercises would be a claim with no evidence behind it, @@ -149,6 +167,7 @@ Capability.TARGETING, Capability.UNAVAILABLE_INIT, Capability.LARGE_INTEGERS, + Capability.STANDARD_REASONS, } ) diff --git a/providers/openfeature-provider-flagd/tests/tck/test_rpc_conformance.py b/providers/openfeature-provider-flagd/tests/tck/test_rpc_conformance.py index 6c535eb0..5827aef8 100644 --- a/providers/openfeature-provider-flagd/tests/tck/test_rpc_conformance.py +++ b/providers/openfeature-provider-flagd/tests/tck/test_rpc_conformance.py @@ -153,6 +153,26 @@ # not a scenario that used to pass. Reading its absence from the failure # list as evidence of support is the mistake this note exists to prevent. # +# STANDARD_REASONS +# New at spec@c342461a, which moved every resolution-reason assertion out of +# the other feature files and into reason.feature, gated as a whole. A claim +# rather than an exemption: 2.2.5 is a SHOULD that permits "some other +# string", so declaring the tag says this provider uses the standard +# vocabulary with the standard meanings. +# +# Declared on a run rather than on the source. All nine scenarios pass -- +# the four rule-less rows as STATIC, an unknown flag and a type mismatch as +# ERROR beside their error codes, TARGETING_MATCH for the matched rule and +# DEFAULT for the miss, DISABLED for a disabled flag. The last three need +# @targeting and @disabled-flags as well, which this suite declares, so none +# of the file is skipped here. +# +# Worth recording that the DISABLED row passes despite the reason arriving as +# flagd's bare string rather than the SDK's Reason enum, noted under +# DISABLED_FLAGS above: the step compares the reason as text, and "DISABLED" +# is "DISABLED" either way. The in-process resolver returns Reason.DISABLED +# and passes identically. +# # CACHING # Reserved in the Capability enum; no scenario carries the tag. Declaring a # capability nothing exercises would be a claim with no evidence behind it, @@ -169,6 +189,7 @@ Capability.TARGETING, Capability.UNAVAILABLE_INIT, Capability.LARGE_INTEGERS, + Capability.STANDARD_REASONS, } ) From acf50216599fd6f24f01174f23e3a3f9cf48e681 Mon Sep 17 00:00:00 2001 From: Simon Schrottner Date: Sun, 13 Sep 2026 15:37:12 +0200 Subject: [PATCH 13/29] test(flagd): say what the numeric-coercion scenarios actually do, measured The base re-pins the conformance assets to spec@89b1519a, which adds a fifth declaring rule to Appendix F and corrects two $comment blocks in canonical-flags.json. No feature file changes and no declaration here moves: the run is 7 failed, 105 passed, 18 skipped before and after, with the same seven failures. The conftest's note about the flags flagd-testbed does not seed names the new revision. Neither suite declares @numeric-coercion and neither starts to. What changes is the evidence: both notes stated what the source would do, and one of them stated it as though it had been observed. Declaring the tag in both suites and running the three scenarios says something different. In-process passes both halves it claimed to -- 0.5 refused as an integer, 10 widened to a float. The third it blamed on the same `(int,)` rule cannot be observed at all: flagd-testbed seeds no `integral-float-flag`, so the scenario fails FLAG_NOT_FOUND, which is the gap the conftest already records for `large-integer-flag` and `huge-integer-flag` two paragraphs earlier. The source reading may well be right about a seeded 10.0; nothing here has seen it. RPC said one of three fails. Two do. The narrowing of 0.5 to 0 is real and measured, and `integer-flag` as a Float passes -- but `integral-float-flag` fails on the missing flag there too, so the note's "the two lossless scenarios pass" was false about an observable. Neither correction changes a declaration, and that is the point worth keeping: both resolvers were withheld for a good reason and the reason recorded was partly someone's reading of a source file. Two resolvers of one provider answering the same three questions differently is also the clearest evidence in this repository that Python can express @numeric-coercion at all -- a language that could not ask the questions could not produce two different answers to them. Signed-off-by: Simon Schrottner --- .../tests/tck/conftest.py | 2 +- .../tests/tck/test_in_process_conformance.py | 17 +++++++++++++---- .../tests/tck/test_rpc_conformance.py | 15 +++++++++++---- 3 files changed, 25 insertions(+), 9 deletions(-) diff --git a/providers/openfeature-provider-flagd/tests/tck/conftest.py b/providers/openfeature-provider-flagd/tests/tck/conftest.py index 5774cd6d..9fc03b79 100644 --- a/providers/openfeature-provider-flagd/tests/tck/conftest.py +++ b/providers/openfeature-provider-flagd/tests/tck/conftest.py @@ -31,7 +31,7 @@ probes. **The testbed does not yet serve the whole canonical flag set.** The conformance -assets at spec@c342461a ask for three flags that flagd-testbed v3.8.0 +assets at spec@89b1519a ask for three flags that flagd-testbed v3.8.0 (``openfeature/test-harness/version.txt``, and the tag pinned in ``docker-compose.yaml`` beside this file) does not seed: ``large-integer-flag``, ``huge-integer-flag`` and ``integral-float-flag``. Until diff --git a/providers/openfeature-provider-flagd/tests/tck/test_in_process_conformance.py b/providers/openfeature-provider-flagd/tests/tck/test_in_process_conformance.py index 07143892..44b740aa 100644 --- a/providers/openfeature-provider-flagd/tests/tck/test_in_process_conformance.py +++ b/providers/openfeature-provider-flagd/tests/tck/test_in_process_conformance.py @@ -98,10 +98,19 @@ # mismatch rather than 0 -- the lossy half holds. The float mapping at # flagd_core.py:26 is the wider `(int, float)`, and `resolve_float_value` # (flagd_core.py:113-114) widens an int result to a float, so `integer-flag` -# requested as a Float is 10.0 -- that lossless half holds too. But the -# same `(int,)` rule rejects `integral-float-flag`'s 10.0 requested as an -# Integer, where the tag requires 10: two of three, and a declaration is -# all or nothing. flagd's numeric-coercion ADR +# requested as a Float is 10.0 -- that lossless half holds too. Both halves +# were measured by declaring the tag and running it, not read off the +# source, and both pass. +# +# The third scenario fails, and the reason it fails is not the one this note +# used to give. The same `(int,)` rule would reject `integral-float-flag`'s +# 10.0 requested as an Integer where the tag requires 10 -- but that is a +# reading of the source and nothing here can observe it, because +# flagd-testbed seeds no `integral-float-flag`: the scenario fails +# FLAG_NOT_FOUND, the same gap the conftest records for `large-integer-flag` +# and `huge-integer-flag`. So this resolver is two of three with the third +# unmeasured, rather than two of three with a known coercion gap, and a +# declaration is all or nothing either way. flagd's numeric-coercion ADR # (docs/architecture-decisions/numeric-coercion.md) commits every flagd # implementation to the lossless rule; when openfeature-flagd-core follows # it, this is declared again. diff --git a/providers/openfeature-provider-flagd/tests/tck/test_rpc_conformance.py b/providers/openfeature-provider-flagd/tests/tck/test_rpc_conformance.py index 5827aef8..08f7b5db 100644 --- a/providers/openfeature-provider-flagd/tests/tck/test_rpc_conformance.py +++ b/providers/openfeature-provider-flagd/tests/tck/test_rpc_conformance.py @@ -108,10 +108,17 @@ # bare `int64(val)` (core/pkg/evaluator/json.go, ResolveIntValue, at the # v0.16.0 the testbed's `flagd/Dockerfile` builds on), so `float-flag`'s 0.5 # comes back as 0 with reason STATIC and no error code -- silently narrowed, -# which is the one thing the lossy scenario forbids. The two lossless -# scenarios pass for the same reason: 10.0 casts to 10, and a float -# accessor sees the float64 the server already holds. One of three is a -# failure, and a declaration is all or nothing. +# which is the one thing the lossy scenario forbids. +# +# Measured by declaring the tag and running it, rather than read off the +# server source: `integer-flag` requested as a Float passes, because a float +# accessor sees the float64 the server already holds. `integral-float-flag` +# requested as an Integer does not, and not for a reason about coercion at +# all -- flagd-testbed seeds no such flag, so it is FLAG_NOT_FOUND, the same +# gap the conftest records for `large-integer-flag`. So two of three fail +# today, one on the narrowing and one on the missing flag, and what the +# server would answer for a seeded 10.0 is unmeasured. A declaration is all +# or nothing either way. # # An earlier revision of this file claimed flagd answers INVALID_ARGUMENT # here. The server source says otherwise: INVALID_ARGUMENT is what From 6ba37d99da997b2f62ea7c69130c4e93dfc447d1 Mon Sep 17 00:00:00 2001 From: Simon Schrottner Date: Sun, 13 Sep 2026 17:53:04 +0200 Subject: [PATCH 14/29] test(flagd): declare what a run supports, per resolver, and say what it measured Three declarations were wrong here, and one run settled all three. They are one commit because they are one decision applied three times -- and because two of them could only be measured once the first was fixed. @LIFECYCLE: declared on both resolvers. It had been withheld since the first pass with nothing anywhere saying why. The only note that came close was the @reinitialization paragraph next door, which argued that declaring @reinitialization alone would be vacuous "and neither resolver declares LIFECYCLE" -- treating the withholding as given rather than justifying it. The consequence is that the six lifecycle scenarios had never been put to this provider at all, in six passes. Settled by running them. All six execute, and five pass on each resolver. The sixth is the @reinitialization scenario, which in-process passes and RPC fails against a closed channel. Java, Go and JavaScript all declare it on both of their flagd resolvers. Go's adoption records having made and reverted this exact mistake and says what it cost: withholding it left that suite blind to six scenarios another language was running against the same provider. Python was the last of the four still doing so. @REINITIALIZATION: declared on in-process, still withheld on RPC. It was withheld on in-process on the vacuity argument alone, which was sound at the time and is now spent: with @lifecycle declared the scenario runs, and it passes. RPC keeps withholding it, now on the measurement alone rather than on two reasons one of which rested on an unjustified absence. 2.5.2 makes reuse permitted rather than required, so neither withholding is a defect and neither carries a known-deviation entry. @NUMERIC-COERCION: declared on both, with the deviation on RPC only. The two resolvers do not agree, and withholding the tag on both hid that. Measured on one run: float-flag (0.5) requested as an Integer is a TYPE_MISMATCH returning the caller's default on in-process, and comes back as 0 with no error code at all on RPC. One provider, two resolvers, opposite answers to the question the capability exists to ask. In-process evaluates locally and never asks flagd; RPC passes back whatever the server answers, and flagd casts a float64 variant with a bare int64(val) -- open-feature/flagd#1996, which flagd's own numeric-coercion ADR commits it to fixing. Both declare the tag, because both attempt the coercion and both get the widening direction right; only RPC gets the narrowing direction wrong. That is the case Appendix F prefers the declared-and-failing shape for: withdrawing the tag from RPC would turn a specific, tracked defect into a skip indistinguishable from a provider that declines to coerce at all. The deviation entry is therefore on RPC only, which is what required a per-resolver known_deviations field on ResolverSuite. Java and Go each attach their equivalent entry to both of their resolvers and are right to -- in those languages both resolvers go through the server and both narrow. Python is the language where mirroring it would publish a defect against a resolver that does not have it. Worth flagging for Go, whose suite currently asserts the opposite: "the Python one does not [narrow], in either resolver: its RPC path asks flagd for an Int and gets INVALID_ARGUMENT for a float-valued flag". That is wrong about this resolver, and it is the same claim an earlier revision of this file made and retracted. INVALID_ARGUMENT is what grpc.py:461-462 would map to TypeMismatchError if it ever arrived, and for a float-valued flag it does not. @LARGE-INTEGERS: withheld, having been declared and failing on every run. Exactly one scenario carries the tag, and it asks for huge-integer-flag, which flagd-testbed v3.8.0 does not seed. The declaration rested on no evidence either way, and its failure read as a provider defect while establishing nothing. Nothing in either resolver would narrow the value. Go and JavaScript withhold it for this same reason; Java cannot declare it at all, its integer accessor being 32 bits, which is a third thing again. This looks like the same missing-fixture problem as integral-float-flag under @numeric-coercion, which this commit declares and leaves failing. The difference is worth stating as a rule rather than as two judgement calls: Declare when at least one scenario gating the tag can actually be put to the provider, and record the backend's gap where the others fail. Withhold when none of them can. @numeric-coercion has three scenarios and this backend can ask two -- and the two resolvers answer them differently, which is the finding a withholding would have buried. @large-integers has one, and this backend can ask none of it. One rule, two answers, because the facts differ. Neither gap gets a known-deviation entry: the gap is in the fixture, and an entry would attribute it to a provider that was never given the flag to get wrong. One finding that is neither a pass nor a fail. The in-process scenario "Shutting down a provider that cannot reach its backend completes promptly" passes and leaves a thread exception behind it: gRPC's connectivity poller raises "Cannot invoke RPC: Channel closed!" after shutdown has closed the channel underneath it, so the provider closes the channel without first stopping the watcher still using it. Reproducible on every run. Not a deviation -- nothing required is unmet -- but a stray traceback in a host application's logs during its own shutdown, and the one result the six lifecycle scenarios produced that neither a pass nor a failure would have carried. A full run is now 8 failed, 119 passed, 3 skipped, from 7 failed, 105 passed, 18 skipped. The conftest, the README and the pyproject comment all say so; the last of those had been stale at "7 failed, 87 passed" against a conftest saying 105. Signed-off-by: Simon Schrottner --- .../openfeature-provider-flagd/README.md | 11 +- .../openfeature-provider-flagd/pyproject.toml | 11 +- .../tests/tck/conftest.py | 92 ++++++-- .../tests/tck/suite.py | 14 ++ .../tests/tck/test_in_process_conformance.py | 141 ++++++++---- .../tests/tck/test_rpc_conformance.py | 209 +++++++++++++----- 6 files changed, 343 insertions(+), 135 deletions(-) diff --git a/providers/openfeature-provider-flagd/README.md b/providers/openfeature-provider-flagd/README.md index a80adc5e..5b5da386 100644 --- a/providers/openfeature-provider-flagd/README.md +++ b/providers/openfeature-provider-flagd/README.md @@ -203,11 +203,12 @@ with the reason in a comment above them. Why a conformance suite is not a requir Two things that are this provider's rather than the policy's: - **Docker is not what decides it.** `tests/e2e` needs Docker too and does run in the default build. - What decides it is the run: **7 failed, 105 passed, 18 skipped**, being three canonical flags that - flagd-testbed v3.8.0 does not seed yet, failing on each resolver, plus one genuine provider-side - gap in `openfeature-flagd-core`. `tests/tck/conftest.py` accounts for all seven individually, with - the flag or requirement each one turns on, so a reviewer running the suite can tell a new failure - from a known one and the number above is what to expect. + What decides it is the run: **8 failed, 119 passed, 3 skipped**, being three canonical flags that + flagd-testbed v3.8.0 does not seed yet, failing on each resolver, plus two genuine provider-side + gaps — one per resolver, and only one of them carries a known-deviation entry. + `tests/tck/conftest.py` accounts for all eight individually, with the flag or requirement each one + turns on, so a reviewer running the suite can tell a new failure from a known one and the number + above is what to expect. - **The default build still collects the suite** — `poe test` and `poe test-cov` end in `pytest tests/tck --collect-only`, which imports every module and starts no container. An excluded suite that has quietly stopped importing against the harness is worse than one that runs and diff --git a/providers/openfeature-provider-flagd/pyproject.toml b/providers/openfeature-provider-flagd/pyproject.toml index d91cdae4..9bdefd53 100644 --- a/providers/openfeature-provider-flagd/pyproject.toml +++ b/providers/openfeature-provider-flagd/pyproject.toml @@ -130,11 +130,12 @@ strict = true # linked from tools/openfeature-tck/README.md -- and is deliberately not # restated here. # -# What is local to this package: a full run is 7 failed, 87 passed, 18 skipped. -# Six of those failures are three canonical flags flagd-testbed does not seed -# yet, on each of the two resolvers, and the seventh is a real provider-side -# gap; tests/tck/conftest.py accounts for all seven. Docker is not what decides -# the exclusion -- `tests/e2e` needs Docker too and does run here. +# What is local to this package: a full run is 8 failed, 119 passed, 3 skipped. +# Six of those failures are canonical flags flagd-testbed does not seed yet, on +# each of the two resolvers; the other two are real provider-side gaps, one per +# resolver. tests/tck/conftest.py accounts for all eight and for the three +# skips. Docker is not what decides the exclusion -- `tests/e2e` needs Docker +# too and does run here. # # The suite is still *collected* on every default build, so it cannot quietly # stop importing against the harness while nobody runs it. `--ignore` would diff --git a/providers/openfeature-provider-flagd/tests/tck/conftest.py b/providers/openfeature-provider-flagd/tests/tck/conftest.py index 9fc03b79..0d25fdaa 100644 --- a/providers/openfeature-provider-flagd/tests/tck/conftest.py +++ b/providers/openfeature-provider-flagd/tests/tck/conftest.py @@ -35,32 +35,44 @@ (``openfeature/test-harness/version.txt``, and the tag pinned in ``docker-compose.yaml`` beside this file) does not seed: ``large-integer-flag``, ``huge-integer-flag`` and ``integral-float-flag``. Until -open-feature/flagd-testbed catches up, both suites fail these scenarios with -``FLAG_NOT_FOUND``, for every resolver alike: +open-feature/flagd-testbed#392 lands, three scenarios fail with +``FLAG_NOT_FOUND`` on each resolver alike: -* ``A large integer resolves without loss of precision`` -- untagged; -* ``An integer beyond 32 bits resolves without loss of precision`` -- under - ``@large-integers``, which both suites declare; +* ``A large integer resolves without loss of precision`` -- untagged, so it runs + unconditionally and fails on ``large-integer-flag``; * ``The resolved details name the variant``, the ``large-integer-flag`` row of it -- under ``@variants``, which both suites declare. New at spec@26362f85, and the same gap rather than a new one: the row asks for the variant - ``max-int32`` of a flag that is not there, so it fails with the same - ``FLAG_NOT_FOUND`` as the two above. The other seven rows pass on both + ``max-int32`` of a flag that is not there. The other seven rows pass on both resolvers, which is the evidence the capability is declared on -- withholding it would say flagd does not name variants, which is false, and would - attribute a missing flag to a capability the provider has. - -``integral-float-flag`` is asked for only under ``@numeric-coercion``, which -neither suite declares, so its scenario is skipped rather than failed. The -failures are deliberately left as failures: they say something true about the -stack under test, and an ``xfail`` would say the provider is at fault when it is -the backend that is behind. None of them is a ``KnownDeviation`` either: a -deviation is for a behaviour the *provider* is required to have and does not. - -**The seventh failure is the provider's, not the testbed's.** A full run is -``7 failed, 105 passed, 18 skipped`` -- three of the failures above on each -resolver, and one more on in-process alone: ``boolean-flag`` requested as a -Float resolves to ``1.0`` with reason ``STATIC`` and no error code, where the + attribute a missing flag to a capability the provider has; +* ``An integral float requested as an integer is coerced without loss`` -- under + ``@numeric-coercion``, which both suites declare as of this pass. It asks for + ``integral-float-flag``. This one is new to the failure list and is the price + of declaring that tag; the two suites explain why paying it is the honest + report, and the RPC suite's ``KnownDeviation`` summary disclaims it explicitly + so that a reader does not attribute it to flagd. + +``huge-integer-flag`` is asked for only by the single scenario under +``@large-integers``, which neither suite declares any more, so it is skipped +rather than failed -- the backend can put none of that tag's scenarios to the +provider, so a declaration would rest on nothing. ``@numeric-coercion`` is the +opposite case and is declared: two of its three scenarios do reach the provider, +and the two resolvers answer them differently. Both suites state the rule that +decides this. + +The failures are deliberately left as failures: they say something true about +the stack under test, and an ``xfail`` would say the provider is at fault when +it is the backend that is behind. **None of them is a ``KnownDeviation``**: a +deviation is for a behaviour the *provider* is required to have and does not, +and the provider was never given the flag to get wrong. Go and JavaScript both +record their equivalent gaps the same way and say so in the same words. + +**Two failures are the provider's, not the testbed's.** A full run is +``8 failed, 119 passed, 3 skipped``. Six of the eight are the three above on +each resolver. The seventh is on in-process alone: ``boolean-flag`` requested as +a Float resolves to ``1.0`` with reason ``STATIC`` and no error code, where the mandatory wrong-type scenario asks for the caller's default. ``bool`` is a subclass of ``int`` in Python, so the widening at ``flagd_core.py:113-114`` -- ``if isinstance(result.value, int): result.value = float(result.value)`` -- sees @@ -74,6 +86,46 @@ would add nothing a reader cannot already see. It is not a testbed gap and does not go away when the image is bumped. +The eighth is on RPC alone, and it is the one failure here that *does* carry a +``KnownDeviation``: ``float-flag`` (0.5) requested as an Integer comes back as +``0`` with no error code, where ``@numeric-coercion`` requires ``TYPE_MISMATCH`` +and the caller's default. The tag is declared and the scenario left to fail +rather than the tag withheld, because this resolver does attempt the coercion +and gets one direction wrong -- see ``test_rpc_conformance.py``. **The +in-process resolver passes this scenario**, refusing 0.5 locally, so the +deviation is recorded against RPC only. That asymmetry is the most interesting +result in this pair of suites and is the thing a shared declaration would have +hidden: the Java and Go flagd adoptions each record the same defect against both +of their resolvers, correctly, because theirs both narrow; Python is the +language where that would have been false. + +**Declaring ``@lifecycle`` surfaced one thing beyond a pass or a fail, and it is +worth reading before anyone treats the warning as noise.** The in-process +scenario ``Shutting down a provider that cannot reach its backend completes +promptly`` passes -- shutdown does return well inside the bound -- but it leaves +a ``PytestUnhandledThreadExceptionWarning`` behind it: gRPC's connectivity +polling thread raises ``ValueError: Cannot invoke RPC: Channel closed!`` from +``_poll_connectivity`` after ``shutdown`` has closed the channel underneath it. +So the provider closes the channel without first stopping the watcher that is +still using it. Reproducible on every run. + +It is **not** a ``KnownDeviation`` and not a scenario failure: nothing the +specification requires is unmet, the scenario asserts that shutdown completes +promptly and it does, and 2.5.3's double-shutdown scenario passes as well. It is +a shutdown-ordering race in ``openfeature-flagd-core``'s watcher that surfaces +as a stray traceback in a host application's logs during its own shutdown. +Recorded here because it is the one finding the six lifecycle scenarios produced +that neither a pass nor a failure would have carried, and because the six had +never been run against this provider before -- the capability was withheld from +the first pass to the sixth with nothing saying why. + +**The three skips are two scenarios, not three.** ``@large-integers`` gates a +single scenario and is withheld on both resolvers for the backend gap above, so +it skips twice; ``@reinitialization`` gates a single scenario and is declared on +in-process, which passes it, and withheld on RPC, which cannot reuse a closed +channel and is permitted not to by 2.5.2, so it skips once. Neither withholding +is a defect and neither carries a deviation. + ``targeting-key-flag``, new in the canonical set at the same revision, needs no testbed change. It is the flag flagd-testbed's own ``targeting.feature`` already uses -- same key, same ``hit``/``miss`` variants, same uuid -- seeded from diff --git a/providers/openfeature-provider-flagd/tests/tck/suite.py b/providers/openfeature-provider-flagd/tests/tck/suite.py index aba88c5b..f6871b46 100644 --- a/providers/openfeature-provider-flagd/tests/tck/suite.py +++ b/providers/openfeature-provider-flagd/tests/tck/suite.py @@ -24,6 +24,7 @@ from openfeature.contrib.provider.flagd.config import ResolverType from openfeature.contrib.tools.tck import ( Capability, + KnownDeviation, RunningBackend, TckConfig, ) @@ -135,6 +136,18 @@ class ResolverSuite: ready_timeout: float + known_deviations: tuple[KnownDeviation, ...] = () + """Requirements this resolver is known to fail, acknowledged rather than hidden. + + **Per resolver, and that is the whole reason this field is here rather than + shared.** The two resolvers do not deviate alike: RPC narrows ``float-flag`` + to ``0`` where in-process refuses it, so an entry naming + :attr:`~Capability.NUMERIC_COERCION` is true of one and false of the other. + Attaching it to both -- which is what Java and Go each do, correctly, because + *their* two resolvers do behave identically -- would publish a defect against + the resolver that does not have it. + """ + def build_config( suite: ResolverSuite, @@ -188,6 +201,7 @@ def new_unavailable_provider() -> FeatureProvider: new_provider=new_provider, new_unavailable_provider=new_unavailable_provider, capabilities=suite.capabilities, + known_deviations=suite.known_deviations, event_timeout=EVENT_TIMEOUT, ready_timeout=suite.ready_timeout, ) diff --git a/providers/openfeature-provider-flagd/tests/tck/test_in_process_conformance.py b/providers/openfeature-provider-flagd/tests/tck/test_in_process_conformance.py index 44b740aa..3f73ec20 100644 --- a/providers/openfeature-provider-flagd/tests/tck/test_in_process_conformance.py +++ b/providers/openfeature-provider-flagd/tests/tck/test_in_process_conformance.py @@ -83,64 +83,105 @@ # deadline passes without a synced ruleset, which the SDK's registry turns # into PROVIDER_ERROR. # -# LARGE_INTEGERS -# The ruleset arrives as JSON text over the sync stream and FlagdCore parses -# it with `json.loads` (flagd_core.py:73), which gives an unbounded Python -# int for 9007199254740991; nothing between the parser and the SDK routes -# the value through a float or a 32-bit field. -# -# Not declared, and why: +# LIFECYCLE +# Declared on a run, and the run is the point: this capability had been +# withheld here since the first pass with nothing anywhere saying why, so +# the six lifecycle scenarios had never been put to this resolver at all. +# Declaring it and running them settles it -- all six execute and all six +# pass. There was no reason; there was an omission that every later pass +# inherited because the file next door treated it as given. +# +# It is a real question here rather than a formality, which is the test the +# capability's own documentation sets. An SDK synthesises PROVIDER_READY +# around `initialize` for any provider, so the readiness scenario is vacuous +# for a provider that does nothing during initialisation -- a NoOpProvider +# passes it. This resolver syncs the entire ruleset and applies it to the +# evaluator before ready is emitted (grpc_watcher.py:254-262), and +# initialisation can and does fail (grpc_watcher.py:151), so both terminal +# outcomes the feature file asserts are outcomes this provider actually +# reaches. +# +# Java, Go and JavaScript all declare it on both resolvers, and Go's +# adoption records having made and reverted this exact mistake: withholding +# it left that suite blind to six scenarios another language was running +# against the same provider. Python was the last of the four still doing so. # # NUMERIC_COERCION -# Local, and strict in one direction only. flagd_core.py:25 admits only +# Declared here and declared on RPC too, but they are not the same claim, +# and that is the most interesting thing in this pair of suites: **this +# resolver satisfies the lossy half and RPC does not.** Measured, both +# resolvers, same run -- `float-flag` (0.5) requested as an Integer is a +# TYPE_MISMATCH returning the caller's default here, and comes back as `0` +# with no error code at all on RPC. One provider, two resolvers, opposite +# answers to the question the capability exists to ask. +# +# Why it holds here: evaluation is local, so flagd_core.py:25 admits only # `int` for an integer request and `_check_type` (flagd_core.py:228-231) -# raises TypeMismatchError for anything else, so `float-flag`'s 0.5 is a -# mismatch rather than 0 -- the lossy half holds. The float mapping at -# flagd_core.py:26 is the wider `(int, float)`, and `resolve_float_value` -# (flagd_core.py:113-114) widens an int result to a float, so `integer-flag` -# requested as a Float is 10.0 -- that lossless half holds too. Both halves -# were measured by declaring the tag and running it, not read off the -# source, and both pass. -# -# The third scenario fails, and the reason it fails is not the one this note -# used to give. The same `(int,)` rule would reject `integral-float-flag`'s -# 10.0 requested as an Integer where the tag requires 10 -- but that is a -# reading of the source and nothing here can observe it, because -# flagd-testbed seeds no `integral-float-flag`: the scenario fails -# FLAG_NOT_FOUND, the same gap the conftest records for `large-integer-flag` -# and `huge-integer-flag`. So this resolver is two of three with the third -# unmeasured, rather than two of three with a known coercion gap, and a -# declaration is all or nothing either way. flagd's numeric-coercion ADR -# (docs/architecture-decisions/numeric-coercion.md) commits every flagd -# implementation to the lossless rule; when openfeature-flagd-core follows -# it, this is declared again. +# raises TypeMismatchError for anything else. The float mapping at +# flagd_core.py:26 is the wider `(int, float)` and `resolve_float_value` +# (flagd_core.py:113-114) widens an int result, so `integer-flag` requested +# as a Float is 10.0. Both of the scenarios this backend can put to the +# provider pass, and no KnownDeviation is recorded against this resolver -- +# there is no defect here to record. The entry on the RPC suite is +# deliberately not mirrored onto this one. +# +# The tag's third scenario fails, and not for a reason about coercion: +# flagd-testbed seeds no `integral-float-flag`, so it fails FLAG_NOT_FOUND +# (`Flag with key integral-float-flag not present in flag store.`), the same +# backend gap the conftest records for `large-integer-flag`. What this +# resolver would do with a seeded 10.0 is still unmeasured -- the `(int,)` +# rule above says it would refuse it, but that is a reading of the source +# and nothing here observes it. The gap is recorded rather than treated as a +# reason to withhold: two of the three scenarios do reach this provider and +# both pass, and withholding on the strength of the one the backend cannot +# ask would discard the finding that the two resolvers differ. # # REINITIALIZATION # New at spec@fc99d5ac, which gated the scenario "A provider that was shut -# down can be initialized again" that had been untagged before it. Withheld -# here even though this resolver does support reuse, which is the -# interesting half of the story and was measured rather than assumed: -# declaring LIFECYCLE and REINITIALIZATION together locally makes the -# scenario run, and in-process passes it, while RPC fails it against a -# closed channel. The two resolvers genuinely differ. -# -# It stays withheld because declaring it would be vacuous. The scenario -# lives in lifecycle.feature, which carries @lifecycle at the feature level, -# so it inherits that tag and carries both; the gate skips a scenario when -# any capability gating it is undeclared, and this suite does not declare -# LIFECYCLE. Declaring REINITIALIZATION alone would leave the scenario -# skipped on @lifecycle and the claim unexamined -- the same -# declare-what-nothing-exercises error the reserved tag below is kept out -# for. Declaring LIFECYCLE is a separate question from this one and is not -# settled here. +# down can be initialized again" that had been untagged before it. Declared +# here and withheld on RPC, and again the two resolvers genuinely differ: +# this one shuts down and starts again serving `boolean-flag` correctly, +# while RPC evaluates against a closed channel. Measured on the same run. +# +# This was withheld until this pass, on the argument that declaring it would +# be vacuous -- the scenario also carries @lifecycle, which this suite did +# not declare, so it would have skipped on that tag regardless. The argument +# was sound and is now spent: @lifecycle is declared above, the scenario +# runs, and the claim is examined rather than asserted. # # Requirement 2.5.2 says a provider SHOULD revert to its uninitialized # state and that "some providers MAY allow reinitialization", so reuse is -# permitted rather than required and withholding needs no KnownDeviation. +# permitted rather than required -- which is why RPC's withholding needs no +# KnownDeviation, and why declaring it here is a claim worth making rather +# than a mandatory box ticked. # -# Worth recording that this scenario never ran here, at this pin or the one -# before it: it is one of the six @lifecycle skips each resolver reports, -# not a scenario that used to pass. +# Not declared, and why: +# +# LARGE_INTEGERS +# Withheld, and this is a change: it was declared here until this pass and +# failed on every run. Exactly one scenario carries the tag, and it asks for +# `huge-integer-flag`, which flagd-testbed v3.8.0 does not seed -- so the +# declaration was a claim with no evidence behind it either way, and its +# failure (`Flag with key huge-integer-flag not present in flag store.`) +# read as a provider defect while establishing nothing about the provider. +# Python's `int` is unbounded and the ruleset arrives as JSON text parsed +# with `json.loads` (flagd_core.py:73), so nothing here would narrow the +# value; the suite simply cannot show that. +# +# The rule this and NUMERIC_COERCION are both decided by, stated once: +# **declare when at least one scenario gating the tag can actually be put to +# the provider, and record the backend's gap where the others fail; withhold +# when none of them can.** The two look like the same missing-fixture +# problem and are not. @numeric-coercion has three scenarios and this +# backend can ask two of them, which is what makes the declaration mean +# something and the third failure a footnote. @large-integers has one, and +# this backend can ask none of it. +# +# No KnownDeviation for it, in either shape. The gap is in the fixture, and +# an entry would attribute it to the provider. Go and JavaScript withhold it +# for this same reason; Java cannot declare it at all, because its integer +# accessor is 32 bits, which is a third thing again and not this one. +# open-feature/flagd-testbed#392 adds the flag; declare it then. # # STANDARD_REASONS # New at spec@c342461a, which moved every resolution-reason assertion out of @@ -175,8 +216,10 @@ Capability.DISABLED_FLAGS, Capability.TARGETING, Capability.UNAVAILABLE_INIT, - Capability.LARGE_INTEGERS, Capability.STANDARD_REASONS, + Capability.LIFECYCLE, + Capability.NUMERIC_COERCION, + Capability.REINITIALIZATION, } ) diff --git a/providers/openfeature-provider-flagd/tests/tck/test_rpc_conformance.py b/providers/openfeature-provider-flagd/tests/tck/test_rpc_conformance.py index 08f7b5db..7ab82546 100644 --- a/providers/openfeature-provider-flagd/tests/tck/test_rpc_conformance.py +++ b/providers/openfeature-provider-flagd/tests/tck/test_rpc_conformance.py @@ -13,6 +13,7 @@ from openfeature.contrib.provider.flagd.config import ResolverType from openfeature.contrib.tools.tck import ( Capability, + KnownDeviation, RunningBackend, TckConfig, feature_paths, @@ -90,75 +91,127 @@ # passes without a connection, which the SDK's registry turns into # PROVIDER_ERROR. # -# LARGE_INTEGERS -# RPC never narrows an integer. flagd holds every numeric variant as a -# float64 -- Go's encoding/json decodes an untyped number into one -- and -# 2^53 - 1 is exactly the largest integer a float64 represents without -# rounding, which is why the canonical set asks for nothing larger. The -# server casts it to the int64 of ResolveIntResponse.value, and grpc.py:448 -# hands that to the SDK as a Python int, unbounded. Nothing in between is -# 32 bits wide. +# LIFECYCLE +# Declared on a run, and the run is the point: this capability had been +# withheld here since the first pass with nothing anywhere saying why -- the +# @reinitialization note below treated it as given rather than justifying it +# -- so the six lifecycle scenarios had never been put to this resolver at +# all. Declaring it and running them settles it: all six execute, five pass, +# and the sixth is the @reinitialization scenario dealt with below. # -# Not declared, and why: +# It is a real question here rather than a formality. An SDK synthesises +# PROVIDER_READY around `initialize` for any provider, so the readiness +# scenario is vacuous for one that does nothing during initialisation -- a +# NoOpProvider passes it. This resolver blocks until the evaluation stream +# is up and raises ProviderNotReadyError when the deadline passes +# (grpc.py:175, grpc.py:261), so both terminal outcomes the feature file +# asserts are outcomes this provider actually reaches. +# +# Java, Go and JavaScript all declare it on both resolvers, and Go's +# adoption records having made and reverted this exact mistake: withholding +# it left that suite blind to six scenarios another language was running +# against the same provider. Python was the last of the four still doing so. # # NUMERIC_COERCION +# Declared, and failing the lossy scenario -- deliberately, and the failure +# is the report. See KNOWN_DEVIATIONS below. +# # RPC does not type-check locally: grpc.py:444-448 asks flagd for an Int and # passes back whatever the server answers, so the whole decision is flagd's. # flagd's evaluator resolves the variant as a float64 and casts it with a # bare `int64(val)` (core/pkg/evaluator/json.go, ResolveIntValue, at the # v0.16.0 the testbed's `flagd/Dockerfile` builds on), so `float-flag`'s 0.5 # comes back as 0 with reason STATIC and no error code -- silently narrowed, -# which is the one thing the lossy scenario forbids. -# -# Measured by declaring the tag and running it, rather than read off the -# server source: `integer-flag` requested as a Float passes, because a float -# accessor sees the float64 the server already holds. `integral-float-flag` -# requested as an Integer does not, and not for a reason about coercion at -# all -- flagd-testbed seeds no such flag, so it is FLAG_NOT_FOUND, the same -# gap the conftest records for `large-integer-flag`. So two of three fail -# today, one on the narrowing and one on the missing flag, and what the -# server would answer for a seeded 10.0 is unmeasured. A declaration is all -# or nothing either way. -# -# An earlier revision of this file claimed flagd answers INVALID_ARGUMENT -# here. The server source says otherwise: INVALID_ARGUMENT is what -# grpc.py:461-462 would map to TypeMismatchError if it ever arrived, and -# for a float-valued flag it does not. The Java reference adoption recorded -# the same narrowing against the same server. flagd's numeric-coercion ADR -# (docs/architecture-decisions/numeric-coercion.md) commits it to lossless -# coercion, tracked as open-feature/flagd#1996; this is declared again once -# the testbed ships a flagd that implements it. +# which is the one thing the lossy scenario forbids. Measured, not read off +# the server source: the scenario fails with `flag 'float-flag' resolved to +# 0 (int), expected 1 (int)`. +# +# Declared rather than withheld because this resolver *attempts* the +# coercion and gets one direction wrong, which is the case Appendix F's +# guidance is built around: `integer-flag` requested as a Float passes, so +# withdrawing the tag would turn a real, specific defect into a skip +# indistinguishable from a provider that declines to coerce at all. That is +# the failure mode the deviation field exists to prevent. +# +# **The in-process resolver passes this scenario.** It refuses 0.5 as an +# Integer with TYPE_MISMATCH, because it evaluates locally and never asks +# flagd. One provider, two resolvers, opposite answers -- which is why the +# deviation below is on this suite only and is not mirrored onto that one. +# Java and Go both attach their equivalent entry to both of their resolvers, +# correctly, because in those languages both narrow identically; Python is +# the language where that would be false. +# +# Worth flagging for the other adoptions: the Go suite's note currently +# asserts that "the Python one does not [narrow], in either resolver: its +# RPC path asks flagd for an Int and gets INVALID_ARGUMENT for a +# float-valued flag". That is wrong about this resolver, and it is the same +# claim an earlier revision of this file made and retracted -- +# INVALID_ARGUMENT is what grpc.py:461-462 would map to TypeMismatchError if +# it ever arrived, and for a float-valued flag it does not. Python's RPC +# narrows exactly as Go's and Java's do. Only its in-process resolver is the +# exception, and it is the only such resolver in the four languages. +# +# The tag's third scenario fails for a reason that is not flagd's: +# flagd-testbed seeds no `integral-float-flag`, so it is FLAG_NOT_FOUND. The +# conftest records it and the deviation summary disclaims it, which is the +# shape Java uses; it is not a reason to withhold the tag. +# +# Not declared, and why: +# +# LARGE_INTEGERS +# Withheld, and this is a change: it was declared here until this pass and +# failed on every run. Exactly one scenario carries the tag, and it asks for +# `huge-integer-flag`, which flagd-testbed v3.8.0 does not seed -- so the +# declaration was a claim with no evidence behind it either way, and its +# failure read as a provider defect while establishing nothing about the +# provider. Nothing in this path would narrow the value: flagd holds every +# numeric variant as a float64 and 2^53 - 1 is exactly the largest integer a +# float64 represents without rounding, the server casts it to the int64 of +# ResolveIntResponse.value, and grpc.py:448 hands that to the SDK as an +# unbounded Python int. Nothing in between is 32 bits wide. The suite simply +# cannot show that. +# +# The rule this and NUMERIC_COERCION are both decided by, stated once: +# **declare when at least one scenario gating the tag can actually be put to +# the provider, and record the backend's gap where the others fail; withhold +# when none of them can.** The two look like the same missing-fixture +# problem and are not. @numeric-coercion has three scenarios and this +# backend can ask two of them -- and their answers differ between the two +# resolvers, which is the finding a withholding would have buried. +# @large-integers has one, and this backend can ask none of it. +# +# No KnownDeviation for it, in either shape. The gap is in the fixture, and +# an entry would attribute it to the provider. Go and JavaScript withhold it +# for this same reason; Java cannot declare it at all, because its integer +# accessor is 32 bits, which is a third thing again and not this one. +# open-feature/flagd-testbed#392 adds the flag; declare it then. # # REINITIALIZATION # New at spec@fc99d5ac, which gated the scenario "A provider that was shut -# down can be initialized again" that had been untagged before it. Withheld -# for two independent reasons, either of which is sufficient. -# -# First, RPC genuinely does not support reuse, which was measured rather -# than reasoned about: declaring LIFECYCLE and REINITIALIZATION together -# locally makes the scenario run, and it fails with `boolean-flag` resolving -# to the code default because grpc.py:420 raises "Cannot invoke RPC on -# closed channel!". shutdown() closes the channel and the second initialize() -# does not rebuild it, so the provider evaluates against a closed connection -# rather than failing outright -- exactly the shape the specification's own -# note on this capability describes. Requirement 2.5.2 says a provider -# SHOULD revert to its uninitialized state and that "some providers MAY -# allow reinitialization", so reuse is permitted rather than required and -# declining it is a choice the specification offers. Hence no -# KnownDeviation entry: there is no requirement to deviate from. +# down can be initialized again" that had been untagged before it. Withheld, +# and for one reason rather than the two this note used to give. # -# Second, and why this cannot be declared even where reuse does work: the -# scenario lives in lifecycle.feature, which carries @lifecycle at the -# feature level, so it inherits that tag and carries both. The gate skips a -# scenario when any capability gating it is undeclared, and neither resolver -# declares LIFECYCLE. Declaring REINITIALIZATION alone would leave the -# scenario skipped on @lifecycle and the claim unexamined -- a vacuous -# declaration of the kind the reserved tag below is kept out for. +# RPC genuinely does not support reuse, measured rather than reasoned about: +# with LIFECYCLE declared the scenario runs, and it fails with `boolean-flag` +# resolving to the code default because grpc.py:420 raises "Cannot invoke RPC +# on closed channel!". shutdown() closes the channel and the second +# initialize() does not rebuild it, so the provider evaluates against a +# closed connection rather than failing outright -- exactly the shape the +# specification's own note on this capability describes. Requirement 2.5.2 +# says a provider SHOULD revert to its uninitialized state and that "some +# providers MAY allow reinitialization", so reuse is permitted rather than +# required and declining it is a choice the specification offers. Hence no +# KnownDeviation entry: there is no requirement to deviate from. # -# Worth recording that this scenario never ran here, at this pin or the one -# before it: it is one of the six @lifecycle skips each resolver reports, -# not a scenario that used to pass. Reading its absence from the failure -# list as evidence of support is the mistake this note exists to prevent. +# The second reason is gone, and it was the load-bearing one for the wrong +# thing. It ran: the scenario also carries @lifecycle, neither resolver +# declared LIFECYCLE, so declaring REINITIALIZATION alone would leave the +# scenario skipped and the claim unexamined. True at the time, but it rested +# on a withholding that nothing justified, and it is what kept @lifecycle +# unexamined for six passes. LIFECYCLE is declared above now, so the +# scenario runs and this withholding rests on the measurement alone -- which +# is where it should always have rested. The in-process suite declares +# REINITIALIZATION for the same reason in reverse: it runs, and it passes. # # STANDARD_REASONS # New at spec@c342461a, which moved every resolution-reason assertion out of @@ -195,16 +248,60 @@ Capability.DISABLED_FLAGS, Capability.TARGETING, Capability.UNAVAILABLE_INIT, - Capability.LARGE_INTEGERS, Capability.STANDARD_REASONS, + Capability.LIFECYCLE, + Capability.NUMERIC_COERCION, } ) +KNOWN_DEVIATIONS = ( + KnownDeviation.tracked( + capability=Capability.NUMERIC_COERCION, + issue="https://github.com/open-feature/flagd/issues/1996", + summary=( + "The lossy half of the coercion rule is not enforced: evaluating " + "float-flag (0.5) through the integer accessor returns 0 with no " + "error code, rather than TYPE_MISMATCH with the code default, so " + "the fractional part is discarded silently. Lossless coercion is " + "permitted and is not the defect -- this resolver does widen an " + "integer to a float correctly, which is why the capability is " + "declared and the scenario left to fail rather than the capability " + "withheld. The rule is flagd's own accepted numeric-coercion ADR " + "rather than a specification requirement, which does not define " + "numeric coercion at all (open-feature/spec#430), so this is a " + "deviation from a commitment flagd made rather than from the " + "provider contract. Unlike the Java and Go flagd providers, whose " + "two resolvers narrow identically and which therefore place the " + "defect in their shared provider layer, this one is in the server " + "alone: the Python in-process resolver evaluates locally and " + "refuses 0.5 as an integer correctly, so it is not recorded as " + "deviating and carries no equivalent entry. The " + "tag's third scenario also fails, but for an unrelated reason that " + "is not flagd's: integral-float-flag is absent from the pinned " + "flagd-testbed image, open-feature/flagd-testbed#392." + ), + ), +) +"""The one requirement this resolver is known to fail. + +Takes the **declared and failing** shape rather than the withheld-and-skipped +one, which is the shape Appendix F's guidance prefers and this is the case it +prefers it for: flagd does attempt the coercion -- the widening scenario passes +-- and gets the narrowing direction wrong, so withdrawing the capability would +turn a real failure into a skip indistinguishable from a provider that declines +to coerce. The failure stays visible in the results and this entry says it is +known and why. + +Recorded against this resolver only. See the in-process suite, which passes the +scenario this deviates on. +""" + RPC_SUITE = ResolverSuite( name="flagd-rpc", resolver_type=ResolverType.RPC, backend_port=RPC_PORT, capabilities=RPC_CAPABILITIES, + known_deviations=KNOWN_DEVIATIONS, # RPC holds no ruleset of its own: it is ready as soon as the evaluation # stream is up, so it needs less headroom than in-process. ready_timeout=30.0, From ee9fd1de13f86a0d356344af1a927988273594e1 Mon Sep 17 00:00:00 2001 From: Simon Schrottner Date: Sun, 13 Sep 2026 18:48:55 +0200 Subject: [PATCH 15/29] docs(flagd): cite the declaring rule instead of stating it again Both suites carried the rule that decides @numeric-coercion and @large-integers in full, introduced as "stated once" -- which was true when nowhere else stated it. That wording is now Appendix F's sixth declaring rule (spec@4cab0320), taken from these files, so keeping a copy here makes two places for one rule to drift and this effort has already spent several passes undoing exactly that. The rule is named and its two consequences are attributed, and what stays is the part only this suite can say: which scenarios this backend can actually put to each resolver, and what it answered. The second consequence gets acted on rather than quoted. A capability withheld for a backend gap is temporary in a way one withheld by choice is not, so the @large-integers note now says what makes it expire -- flagd-testbed#392 seeding huge-integer-flag -- and what the cost of forgetting is: a withholding that outlives its reason reads as a claim about the provider. The numeric-coercion note pins which revision of the appendix it is following. That paragraph's guidance was reversed at spec@045950ca, and this repository's own TCK README and capability enum taught the old shape until this pass, so a reader who finds one of those in a cached copy can tell which came first. Also drops a stale cross-reference. The Go suite's note about the Python resolvers was wrong when it was written down here and has since been corrected there, so the paragraph keeps the finding -- RPC narrows, in-process does not, and reading the source got it backwards twice in two languages -- without telling a reader to go and check a file that no longer says it. Signed-off-by: Simon Schrottner --- .../tests/tck/test_in_process_conformance.py | 38 +++++++---- .../tests/tck/test_rpc_conformance.py | 64 +++++++++++-------- 2 files changed, 63 insertions(+), 39 deletions(-) diff --git a/providers/openfeature-provider-flagd/tests/tck/test_in_process_conformance.py b/providers/openfeature-provider-flagd/tests/tck/test_in_process_conformance.py index 3f73ec20..b5960e78 100644 --- a/providers/openfeature-provider-flagd/tests/tck/test_in_process_conformance.py +++ b/providers/openfeature-provider-flagd/tests/tck/test_in_process_conformance.py @@ -168,20 +168,30 @@ # with `json.loads` (flagd_core.py:73), so nothing here would narrow the # value; the suite simply cannot show that. # -# The rule this and NUMERIC_COERCION are both decided by, stated once: -# **declare when at least one scenario gating the tag can actually be put to -# the provider, and record the backend's gap where the others fail; withhold -# when none of them can.** The two look like the same missing-fixture -# problem and are not. @numeric-coercion has three scenarios and this -# backend can ask two of them, which is what makes the declaration mean -# something and the third failure a footnote. @large-integers has one, and -# this backend can ask none of it. -# -# No KnownDeviation for it, in either shape. The gap is in the fixture, and -# an entry would attribute it to the provider. Go and JavaScript withhold it -# for this same reason; Java cannot declare it at all, because its integer -# accessor is 32 bits, which is a third thing again and not this one. -# open-feature/flagd-testbed#392 adds the flag; declare it then. +# The rule this and NUMERIC_COERCION are both decided by is **Appendix F's +# sixth declaring rule** -- declare when at least one scenario gating the tag +# can actually be put to the provider, withhold only when none can, the unit +# being the scenario and not the tag. It is cited rather than restated: the +# wording these two suites used last pass is what went into the appendix at +# spec@4cab0320, so the appendix is now where it lives and a copy here would +# be a second place for it to drift. The two gaps look like the same +# missing-fixture problem and are not. @numeric-coercion has three scenarios +# and this backend can ask two of them, which is what makes the declaration +# mean something and the third failure a footnote. @large-integers has one, +# and this backend can ask none of it. +# +# No KnownDeviation for it, in either shape. That is the rule's first +# consequence: a scenario failing because the backend serves no fixture for +# it is not a provider defect, and an entry would attribute the testbed's gap +# to the provider. Go and JavaScript withhold it for this same reason; Java +# cannot declare it at all, because its integer accessor is 32 bits, which is +# a third thing again and not this one. +# +# And the second consequence, which this sentence exists to satisfy: a +# capability withheld for a backend gap is temporary in a way one withheld by +# choice is not. open-feature/flagd-testbed#392 adds `huge-integer-flag`; +# declare the tag when the image carries it, or this withholding outlives its +# reason and starts reading as a claim about the provider. # # STANDARD_REASONS # New at spec@c342461a, which moved every resolution-reason assertion out of diff --git a/providers/openfeature-provider-flagd/tests/tck/test_rpc_conformance.py b/providers/openfeature-provider-flagd/tests/tck/test_rpc_conformance.py index 7ab82546..042a3dc6 100644 --- a/providers/openfeature-provider-flagd/tests/tck/test_rpc_conformance.py +++ b/providers/openfeature-provider-flagd/tests/tck/test_rpc_conformance.py @@ -128,10 +128,14 @@ # # Declared rather than withheld because this resolver *attempts* the # coercion and gets one direction wrong, which is the case Appendix F's -# guidance is built around: `integer-flag` requested as a Float passes, so -# withdrawing the tag would turn a real, specific defect into a skip -# indistinguishable from a provider that declines to coerce at all. That is -# the failure mode the deviation field exists to prevent. +# numeric-coercion note now names outright: a provider in that position +# declares the capability and lets the scenario fail. That note said the +# opposite until spec@045950ca -- withhold, and say which it is -- and two +# of the four adoptions followed it there. `integer-flag` requested as a +# Float passes here, so withdrawing the tag would turn a real, specific +# defect into a skip indistinguishable from a provider that declines to +# coerce at all, which is the failure mode the deviation field exists to +# prevent. # # **The in-process resolver passes this scenario.** It refuses 0.5 as an # Integer with TYPE_MISMATCH, because it evaluates locally and never asks @@ -141,15 +145,15 @@ # correctly, because in those languages both narrow identically; Python is # the language where that would be false. # -# Worth flagging for the other adoptions: the Go suite's note currently -# asserts that "the Python one does not [narrow], in either resolver: its -# RPC path asks flagd for an Int and gets INVALID_ARGUMENT for a -# float-valued flag". That is wrong about this resolver, and it is the same -# claim an earlier revision of this file made and retracted -- -# INVALID_ARGUMENT is what grpc.py:461-462 would map to TypeMismatchError if -# it ever arrived, and for a float-valued flag it does not. Python's RPC -# narrows exactly as Go's and Java's do. Only its in-process resolver is the -# exception, and it is the only such resolver in the four languages. +# Worth keeping because it was got wrong twice, in two languages, by reading +# the source: Python's RPC resolver narrows exactly as Go's and Java's do, +# and only its in-process resolver is the exception -- the only such resolver +# in the four languages. The claim that this path returns INVALID_ARGUMENT +# for a float-valued flag was made and retracted by an earlier revision of +# this file, and asserted about Python by the Go suite until it was corrected +# there. INVALID_ARGUMENT is what grpc.py:461-462 would map to +# TypeMismatchError if it ever arrived; for a float-valued flag it does not. +# A run settled it; neither reading did. # # The tag's third scenario fails for a reason that is not flagd's: # flagd-testbed seeds no `integral-float-flag`, so it is FLAG_NOT_FOUND. The @@ -171,20 +175,30 @@ # unbounded Python int. Nothing in between is 32 bits wide. The suite simply # cannot show that. # -# The rule this and NUMERIC_COERCION are both decided by, stated once: -# **declare when at least one scenario gating the tag can actually be put to -# the provider, and record the backend's gap where the others fail; withhold -# when none of them can.** The two look like the same missing-fixture -# problem and are not. @numeric-coercion has three scenarios and this -# backend can ask two of them -- and their answers differ between the two -# resolvers, which is the finding a withholding would have buried. +# The rule this and NUMERIC_COERCION are both decided by is **Appendix F's +# sixth declaring rule** -- declare when at least one scenario gating the tag +# can actually be put to the provider, withhold only when none can, the unit +# being the scenario and not the tag. It is cited rather than restated: the +# wording these two suites used last pass is what went into the appendix at +# spec@4cab0320, so the appendix is now where it lives and a copy here would +# be a second place for it to drift. The two gaps look like the same +# missing-fixture problem and are not. @numeric-coercion has three scenarios +# and this backend can ask two of them -- and their answers differ between +# the two resolvers, which is the finding a withholding would have buried. # @large-integers has one, and this backend can ask none of it. # -# No KnownDeviation for it, in either shape. The gap is in the fixture, and -# an entry would attribute it to the provider. Go and JavaScript withhold it -# for this same reason; Java cannot declare it at all, because its integer -# accessor is 32 bits, which is a third thing again and not this one. -# open-feature/flagd-testbed#392 adds the flag; declare it then. +# No KnownDeviation for it, in either shape. That is the rule's first +# consequence: a scenario failing because the backend serves no fixture for +# it is not a provider defect, and an entry would attribute the testbed's gap +# to the provider. Go and JavaScript withhold it for this same reason; Java +# cannot declare it at all, because its integer accessor is 32 bits, which is +# a third thing again and not this one. +# +# And the second consequence, which this sentence exists to satisfy: a +# capability withheld for a backend gap is temporary in a way one withheld by +# choice is not. open-feature/flagd-testbed#392 adds `huge-integer-flag`; +# declare the tag when the image carries it, or this withholding outlives its +# reason and starts reading as a claim about the provider. # # REINITIALIZATION # New at spec@fc99d5ac, which gated the scenario "A provider that was shut From 6830da3901789e48e1a559556bcfce5dfaac2ad0 Mon Sep 17 00:00:00 2001 From: Simon Schrottner Date: Sun, 13 Sep 2026 18:49:05 +0200 Subject: [PATCH 16/29] chore(flagd): ignore the flag file every e2e run generates `FlagdContainer` creates `./flags` relative to the package directory and points the launchpad's FLAGS_DIR at it, so every e2e run leaves a 27KB `providers/openfeature-provider-flagd/flags/allFlags.json` behind. The directory is tracked through a `.gitkeep` and no ignore rule matches the file, so the working tree is dirty after any run and a `git add -A` commits a generated fixture. The rule is the one this package already wrote for the same file in the other flag directory: `tests/flags/.gitignore` has contained exactly `allFlags.json` since #121. This is that line, next to the directory that now receives it. Unrelated to the conformance suite, which drives its stack through Compose and writes nothing here -- it is in this branch because this is the branch that touches this package, and it reverts on its own. Signed-off-by: Simon Schrottner --- providers/openfeature-provider-flagd/flags/.gitignore | 1 + 1 file changed, 1 insertion(+) create mode 100644 providers/openfeature-provider-flagd/flags/.gitignore diff --git a/providers/openfeature-provider-flagd/flags/.gitignore b/providers/openfeature-provider-flagd/flags/.gitignore new file mode 100644 index 00000000..f4b13f83 --- /dev/null +++ b/providers/openfeature-provider-flagd/flags/.gitignore @@ -0,0 +1 @@ +allFlags.json From 23d69b93b7bdec78bac2f16b38f318eedf53c30b Mon Sep 17 00:00:00 2001 From: Simon Schrottner Date: Sun, 13 Sep 2026 19:14:15 +0200 Subject: [PATCH 17/29] docs(flagd): cite rule six with the condition it was missing spec@aa2ad24f narrows the rule these two files supplied last pass. Without an opening clause it read as "declare wherever the scenarios are reachable", which forces a declaration even where the specification permits a provider to decline outright -- the Go implementation found it doing exactly that to its own self-tests, whose provider does not coerce by design. The rule now applies once a provider is attempting the capability, and whether an answer is owed at all is the known-deviation rule's question, asked first. Both citations carry the clause and then say why it is satisfied here, which is the part a reader of this file actually needs: this provider attempts the coercion -- correctly in-process, wrongly on RPC -- and nothing in either path declines to resolve a large integer. So what is left for rule six to decide is the fixture gap, which is what it was written for. Signed-off-by: Simon Schrottner --- .../tests/tck/test_in_process_conformance.py | 21 +++++++++++++------ .../tests/tck/test_rpc_conformance.py | 21 +++++++++++++------ 2 files changed, 30 insertions(+), 12 deletions(-) diff --git a/providers/openfeature-provider-flagd/tests/tck/test_in_process_conformance.py b/providers/openfeature-provider-flagd/tests/tck/test_in_process_conformance.py index b5960e78..457a098a 100644 --- a/providers/openfeature-provider-flagd/tests/tck/test_in_process_conformance.py +++ b/providers/openfeature-provider-flagd/tests/tck/test_in_process_conformance.py @@ -169,12 +169,21 @@ # value; the suite simply cannot show that. # # The rule this and NUMERIC_COERCION are both decided by is **Appendix F's -# sixth declaring rule** -- declare when at least one scenario gating the tag -# can actually be put to the provider, withhold only when none can, the unit -# being the scenario and not the tag. It is cited rather than restated: the -# wording these two suites used last pass is what went into the appendix at -# spec@4cab0320, so the appendix is now where it lives and a copy here would -# be a second place for it to drift. The two gaps look like the same +# sixth declaring rule** -- once a provider is attempting a capability, +# declare it when at least one scenario gating it can actually be put to the +# provider and withhold only when none can, the unit being the scenario and +# not the tag. It is cited rather than restated: the wording these two suites +# used last pass is what went into the appendix at spec@4cab0320, so the +# appendix is now where it lives and a copy here would be a second place for +# it to drift. +# +# The opening clause matters and was added at spec@aa2ad24f after the Go +# implementation found the rule forcing declarations it should not: it +# decides whether a question is *askable*, not whether the provider owes an +# answer, and that second question comes first. Both tags clear it here -- +# this resolver does coerce, correctly in both directions it can be asked, +# and nothing about it declines to resolve a large integer -- so what is left +# for rule six to decide is the fixture gap. The two gaps look like the same # missing-fixture problem and are not. @numeric-coercion has three scenarios # and this backend can ask two of them, which is what makes the declaration # mean something and the third failure a footnote. @large-integers has one, diff --git a/providers/openfeature-provider-flagd/tests/tck/test_rpc_conformance.py b/providers/openfeature-provider-flagd/tests/tck/test_rpc_conformance.py index 042a3dc6..f36375c3 100644 --- a/providers/openfeature-provider-flagd/tests/tck/test_rpc_conformance.py +++ b/providers/openfeature-provider-flagd/tests/tck/test_rpc_conformance.py @@ -176,12 +176,21 @@ # cannot show that. # # The rule this and NUMERIC_COERCION are both decided by is **Appendix F's -# sixth declaring rule** -- declare when at least one scenario gating the tag -# can actually be put to the provider, withhold only when none can, the unit -# being the scenario and not the tag. It is cited rather than restated: the -# wording these two suites used last pass is what went into the appendix at -# spec@4cab0320, so the appendix is now where it lives and a copy here would -# be a second place for it to drift. The two gaps look like the same +# sixth declaring rule** -- once a provider is attempting a capability, +# declare it when at least one scenario gating it can actually be put to the +# provider and withhold only when none can, the unit being the scenario and +# not the tag. It is cited rather than restated: the wording these two suites +# used last pass is what went into the appendix at spec@4cab0320, so the +# appendix is now where it lives and a copy here would be a second place for +# it to drift. +# +# The opening clause matters and was added at spec@aa2ad24f after the Go +# implementation found the rule forcing declarations it should not: it +# decides whether a question is *askable*, not whether the provider owes an +# answer, and that second question comes first. Both tags clear it here -- +# this resolver does attempt the coercion, and nothing about it declines to +# resolve a large integer -- so what is left for rule six to decide is the +# fixture gap. The two gaps look like the same # missing-fixture problem and are not. @numeric-coercion has three scenarios # and this backend can ask two of them -- and their answers differ between # the two resolvers, which is the finding a withholding would have buried. From 69f0dcdb6fae086f064501f86014ecfca678623d Mon Sep 17 00:00:00 2001 From: Simon Schrottner Date: Sun, 13 Sep 2026 22:18:32 +0200 Subject: [PATCH 18/29] test(flagd): name the conformance modules for the resolver, not the suite The directory says conformance now that `tests/tck` is a sibling of `tests/e2e` rather than nested inside it, and `poe test-tck` selects it by that path. So `test_rpc_conformance.py` and `test_in_process_conformance.py` were saying it twice, and the half that duplicated the directory is the half that stopped being load-bearing: nothing selects on the module name. The `test_` prefix stays, because pytest still needs it -- that is a framework requirement rather than the naming convention being dropped. Node ids move with the rename. The only hardcoded one in this package's conformance suites is none: the xfail that names a node id lives on the OFREP branch and matches `item.name`, which a module rename does not touch. Same 130 collected, same 8 failed / 119 passed / 3 skipped, same eight scenarios by name. Signed-off-by: Simon Schrottner --- providers/openfeature-provider-flagd/tests/tck/conftest.py | 2 +- providers/openfeature-provider-flagd/tests/tck/suite.py | 4 ++-- .../{test_in_process_conformance.py => test_in_process.py} | 0 .../tests/tck/{test_rpc_conformance.py => test_rpc.py} | 0 4 files changed, 3 insertions(+), 3 deletions(-) rename providers/openfeature-provider-flagd/tests/tck/{test_in_process_conformance.py => test_in_process.py} (100%) rename providers/openfeature-provider-flagd/tests/tck/{test_rpc_conformance.py => test_rpc.py} (100%) diff --git a/providers/openfeature-provider-flagd/tests/tck/conftest.py b/providers/openfeature-provider-flagd/tests/tck/conftest.py index 0d25fdaa..29db1508 100644 --- a/providers/openfeature-provider-flagd/tests/tck/conftest.py +++ b/providers/openfeature-provider-flagd/tests/tck/conftest.py @@ -91,7 +91,7 @@ ``0`` with no error code, where ``@numeric-coercion`` requires ``TYPE_MISMATCH`` and the caller's default. The tag is declared and the scenario left to fail rather than the tag withheld, because this resolver does attempt the coercion -and gets one direction wrong -- see ``test_rpc_conformance.py``. **The +and gets one direction wrong -- see ``test_rpc.py``. **The in-process resolver passes this scenario**, refusing 0.5 locally, so the deviation is recorded against RPC only. That asymmetry is the most interesting result in this pair of suites and is the thing a shared declaration would have diff --git a/providers/openfeature-provider-flagd/tests/tck/suite.py b/providers/openfeature-provider-flagd/tests/tck/suite.py index f6871b46..75796c3e 100644 --- a/providers/openfeature-provider-flagd/tests/tck/suite.py +++ b/providers/openfeature-provider-flagd/tests/tck/suite.py @@ -6,8 +6,8 @@ results is a difference an application would see when it switches resolver, which is exactly the class of thing the conformance suite exists to surface. -Everything they share lives here; everything that differs lives in the two -``test_*_conformance`` modules next to it, where a reader can see the whole of a +Everything they share lives here; everything that differs lives in ``test_rpc.py`` +and ``test_in_process.py`` next to it, where a reader can see the whole of a resolver's declaration in one place. There is no container wiring here any more. The TCK owns the stack -- see diff --git a/providers/openfeature-provider-flagd/tests/tck/test_in_process_conformance.py b/providers/openfeature-provider-flagd/tests/tck/test_in_process.py similarity index 100% rename from providers/openfeature-provider-flagd/tests/tck/test_in_process_conformance.py rename to providers/openfeature-provider-flagd/tests/tck/test_in_process.py diff --git a/providers/openfeature-provider-flagd/tests/tck/test_rpc_conformance.py b/providers/openfeature-provider-flagd/tests/tck/test_rpc.py similarity index 100% rename from providers/openfeature-provider-flagd/tests/tck/test_rpc_conformance.py rename to providers/openfeature-provider-flagd/tests/tck/test_rpc.py From 6fbce2a74f7a743c9165900ea68046cf763d2685 Mon Sep 17 00:00:00 2001 From: Simon Schrottner Date: Sun, 13 Sep 2026 23:05:56 +0200 Subject: [PATCH 19/29] test(flagd): let the collect step run even when the default suite is red `test` and `test-cov` end in `test-tck-collect`, which is the one thing checking that the conformance suite still imports against the harness while `--ignore` keeps it out of the run. poe aborts a sequence at its first failing subtask, and this package's default suite is red on this branch stack -- the TCK requires openfeature-sdk >= 0.10 and this provider is not updated for it yet -- so that check has in fact never run here. `ignore_fail = "return_non_zero"` runs every subtask and still exits non-zero if any failed, so the build's verdict is unchanged and the check is reachable. No count moves: the collect step reports the same 130. Signed-off-by: Simon Schrottner --- providers/openfeature-provider-flagd/pyproject.toml | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/providers/openfeature-provider-flagd/pyproject.toml b/providers/openfeature-provider-flagd/pyproject.toml index 9bdefd53..1c0a338b 100644 --- a/providers/openfeature-provider-flagd/pyproject.toml +++ b/providers/openfeature-provider-flagd/pyproject.toml @@ -141,8 +141,15 @@ strict = true # stop importing against the harness while nobody runs it. `--ignore` would # otherwise leave nothing checking that, since mypy here is configured over # `src` alone. -test = ["test-default", "test-tck-collect"] -test-cov = ["test-cov-default", "test-tck-collect"] +# +# `ignore_fail = "return_non_zero"` is what makes that true rather than +# aspirational: poe aborts a sequence at its first failing subtask, and this +# package's default suite is red on this branch -- the TCK requires +# openfeature-sdk >= 0.10 and this provider is not updated for it yet -- so the +# collect step sat behind a step that never succeeds and never ran. Every +# subtask now runs and a non-zero exit still propagates. +test = { sequence = ["test-default", "test-tck-collect"], ignore_fail = "return_non_zero" } +test-cov = { sequence = ["test-cov-default", "test-tck-collect"], ignore_fail = "return_non_zero" } test-default = "pytest tests --ignore=tests/tck" test-cov-default = "coverage run -m pytest tests --ignore=tests/tck" test-tck = "pytest tests/tck" From ceac0d42f2dbc18fb07d3f22b43004cab570089e Mon Sep 17 00:00:00 2001 From: Simon Schrottner Date: Mon, 14 Sep 2026 07:36:40 +0200 Subject: [PATCH 20/29] test(flagd): drop a gitignore this branch has no reason to add flags/.gitignore ignored allFlags.json, which the launchpad writes into that directory when the pre-existing e2e suite mounts it. The conformance suite mounts no flags directory at all -- it drives the launchpad over HTTP, and the compose file says so -- so nothing this branch adds can produce that file. It is a reasonable fix for the e2e suite and belongs in a PR about the e2e suite. Here it is an unrelated change to a directory this branch otherwise does not touch. Signed-off-by: Simon Schrottner --- providers/openfeature-provider-flagd/flags/.gitignore | 1 - 1 file changed, 1 deletion(-) delete mode 100644 providers/openfeature-provider-flagd/flags/.gitignore diff --git a/providers/openfeature-provider-flagd/flags/.gitignore b/providers/openfeature-provider-flagd/flags/.gitignore deleted file mode 100644 index f4b13f83..00000000 --- a/providers/openfeature-provider-flagd/flags/.gitignore +++ /dev/null @@ -1 +0,0 @@ -allFlags.json From cf717793785dfb42ddd42d4f805517c7d0c1edb3 Mon Sep 17 00:00:00 2001 From: Simon Schrottner Date: Mon, 14 Sep 2026 08:24:35 +0200 Subject: [PATCH 21/29] test(flagd): one compose definition of the backend for the whole repo Stripped of comments, this file and the OFREP adoption's were the same configuration with a different port list: same image, same service name, same absence of host bindings. Two hand-maintained copies of one backend is the mechanism that lets four languages claim to ask the same questions of two different images, which has already happened once across the four. So both now publish 8013, 8015, 8016 and 8080, and the two files are byte-identical. Exposing all four to both suites costs nothing: there are no host bindings, the harness maps container ports to dynamically assigned host ports and looks them up by container port, and it waits only on the ports an adoption declares. This suite never asks for 8016. Two files rather than one shared path, deliberately, and the packaging is the reason. Each provider is published independently, and this package's sdist includes its tests -- measured: `uv build --sdist` here ships tests/tck/docker-compose.yaml, while the OFREP package's ships no tests at all. A file outside the package directory would not travel with the distribution, and pointing one package's conftest at the other's directory is the cross-package coupling the OFREP adoption removed when it stopped reaching into this package's test-harness submodule. Drift is caught by `diff` instead, which both READMEs and the file's own header name. Signed-off-by: Simon Schrottner --- .../openfeature-provider-flagd/README.md | 5 ++++ .../tests/tck/docker-compose.yaml | 29 +++++++------------ 2 files changed, 16 insertions(+), 18 deletions(-) diff --git a/providers/openfeature-provider-flagd/README.md b/providers/openfeature-provider-flagd/README.md index 5b5da386..02c5c78a 100644 --- a/providers/openfeature-provider-flagd/README.md +++ b/providers/openfeature-provider-flagd/README.md @@ -188,6 +188,11 @@ This provider runs the [OpenFeature Provider Conformance Suite][tck] against a f once per resolver, in `tests/tck`. The suite owns the container stack: `tests/tck/conftest.py` declares a Compose file and the two ports the resolvers connect to, and nothing else. +`tests/tck/docker-compose.yaml` is one definition of the backend for the whole repository, and the +OFREP adoption carries a byte-identical copy — each provider package publishes its own distribution +and must not read the other's files, so the two are kept in step by `diff` rather than by sharing a +path. Change one, copy it to the other. + **It is excluded from the default build, and a maintainer runs it by hand before merging a change to it.** diff --git a/providers/openfeature-provider-flagd/tests/tck/docker-compose.yaml b/providers/openfeature-provider-flagd/tests/tck/docker-compose.yaml index 0887be9a..5c27bf27 100644 --- a/providers/openfeature-provider-flagd/tests/tck/docker-compose.yaml +++ b/providers/openfeature-provider-flagd/tests/tck/docker-compose.yaml @@ -1,27 +1,20 @@ -# Backend stack for the OpenFeature TCK, wrapping the unmodified flagd testbed image. +# The backend every conformance suite in this repository runs against: the unmodified +# flagd testbed image, serving flagd and its launchpad control API. # -# The image serves flagd itself and the "launchpad" control API on 8080, whose endpoints the -# TCK's control API contract was derived from. Nothing here is flagd-testbed's own compose -# file: that one stands up envoy as well, for the forbidden-endpoint e2e scenarios, names its -# service `flagd`, and mounts a flags directory from the host. The TCK needs none of that -- -# it drives the launchpad over HTTP -- so this is the smallest stack that serves the suite, -# and it is the same one the Java adoption uses. +# Not flagd-testbed's own compose file, which adds an envoy sidecar for its +# forbidden-endpoint scenarios, names its service `flagd`, and bind-mounts a flags +# directory the launchpad writes into. None of that is needed to drive the launchpad. # -# The service is named `backend` because that is the name every language's TCK defaults to. A -# stack that calls it something else is fine, and says so through -# `ComposeBackend(backend_service=...)`. -# -# Note there are no host port bindings. The TCK requires dynamically mapped ports and -# discovers them after startup -- a pinned host port would make the suite unrunnable in -# parallel and would collide with a developer's local flagd. -# -# The image does not yet serve the whole canonical flag set; tests/tck/conftest.py says which -# flags are missing and which scenarios that fails. Bump the tag when -# open-feature/flagd-testbed#392 lands. +# This file exists twice, once per provider package, and the two are byte-identical: +# providers/openfeature-provider-flagd/tests/tck/docker-compose.yaml +# providers/openfeature-provider-ofrep/tests/tck/docker-compose.yaml +# Each package publishes its own distribution and must not read the other's files, so +# `diff` the two paths after changing either -- that diff is what catches drift. services: backend: image: ghcr.io/open-feature/flagd-testbed:v3.8.0 ports: - 8013 # flagd RPC evaluation (gRPC) - 8015 # flagd in-process sync (gRPC) + - 8016 # flagd's OFREP HTTP API - 8080 # launchpad control API From 8ea66152cb7890dcf04e782ab19f9edb89bbdb05 Mon Sep 17 00:00:00 2001 From: Simon Schrottner Date: Mon, 14 Sep 2026 08:24:50 +0200 Subject: [PATCH 22/29] docs(flagd): keep the evidence, link the rules, delete the retellings These four files were roughly three-quarters prose, and the surplus was not the per-capability reasoning -- it was everything around it: Appendix F's sixth declaring rule quoted at length in both suite modules, the two-shapes rule for a known deviation restated beside the one entry that uses it, the no-container-restart invariant explained twice, and a spec@ revision cited for every tag as though a reader needed the changelog to read the declaration. What is kept is what only this adoption can say. Both resolvers still carry the measurement behind every tag they declare and every tag they withhold, and the numeric-coercion split is kept whole: in-process refuses 0.5 as an Integer with TYPE_MISMATCH where RPC returns 0 with no error code, which is the only such split in the four languages and the reason the deviation is recorded against one resolver and not mirrored onto the other. conftest.py loses most of its length and gains the shape a reader wants: what a full run reports, and what each of the eight failures is. Three causes account for all of them, and the two that are the provider's now cite the issues they were filed as -- python-sdk-contrib#417 for the bool-as-float gap in flagd-core, #419 for the connectivity watcher a clean shutdown leaves running, #420 for the resolver split. The paragraphs explaining which flags flagd-testbed does not seed collapse to one sentence and open-feature/flagd-testbed#392, which names all three and says what each catches; the ones explaining that the canonical set's targeting, disabled and falsy flags need no testbed change collapse to one sentence, because they are flagd-testbed's own flags and that is why the canonical set took their names. Comments and docstrings only. Both suites still report 8 failed, 119 passed, 3 skipped. Signed-off-by: Simon Schrottner --- .../tests/tck/conftest.py | 227 ++++--------- .../tests/tck/suite.py | 32 +- .../tests/tck/test_in_process.py | 256 ++++++-------- .../tests/tck/test_rpc.py | 314 ++++++------------ 4 files changed, 270 insertions(+), 559 deletions(-) diff --git a/providers/openfeature-provider-flagd/tests/tck/conftest.py b/providers/openfeature-provider-flagd/tests/tck/conftest.py index 29db1508..ffa6f550 100644 --- a/providers/openfeature-provider-flagd/tests/tck/conftest.py +++ b/providers/openfeature-provider-flagd/tests/tck/conftest.py @@ -1,153 +1,64 @@ """One testbed stack, declared rather than wired, and shared by both suites. -The whole of the container lifecycle belongs to the TCK now: it starts the -Compose file once per session, discovers the dynamically mapped host ports, -builds the ``HttpControl`` against the launchpad and waits until it accepts -commands, and tears the stack down after the last scenario. What is left here is -the declaration -- which Compose file, which ports the provider connects to -- -and one free port for the scenarios that need a backend that is not there. - -The stack is started once per session and **never restarted** -- container -orchestrators assign host ports dynamically and cannot reliably preserve them -across a restart, so a restarted backend comes back on a different host port, -silently invalidating every provider already pointed at the old one, and the -failure looks like a flaky provider rather than a broken test. Scenario isolation -and every simulated outage go through the control API instead. - -One stack for both suites because one flagd process serves both ports the -resolvers use -- 8013 for RPC and 8015 for sync -- so there is nothing a second -stack would isolate. One ``HttpControl`` with it, which matters and is not merely -tidy: the control tracks whether a disconnect has left the backend down so the -next scenario starts it rather than merely resetting flag state, and two -instances would each hold half of that knowledge. A session-scoped -``tck_backend`` is what makes both true by construction. - -The launchpad registers only ``/start``, ``/restart``, ``/stop`` and ``/change`` -(flagd-testbed ``launchpad/main.go``), so ``/reset`` answers 404 and every -``prepare_scenario`` takes the documented ``/start`` fallback. The probe costs -one 404 for the whole session. It serves no ``/healthz`` either, which the -control API document defines as ready -- so the readiness wait rests on the -control port accepting a connection, which the harness establishes before it -probes. - -**The testbed does not yet serve the whole canonical flag set.** The conformance -assets at spec@89b1519a ask for three flags that flagd-testbed v3.8.0 -(``openfeature/test-harness/version.txt``, and the tag pinned in -``docker-compose.yaml`` beside this file) does not seed: ``large-integer-flag``, -``huge-integer-flag`` and ``integral-float-flag``. Until -open-feature/flagd-testbed#392 lands, three scenarios fail with -``FLAG_NOT_FOUND`` on each resolver alike: - -* ``A large integer resolves without loss of precision`` -- untagged, so it runs - unconditionally and fails on ``large-integer-flag``; -* ``The resolved details name the variant``, the ``large-integer-flag`` row of - it -- under ``@variants``, which both suites declare. New at spec@26362f85, - and the same gap rather than a new one: the row asks for the variant - ``max-int32`` of a flag that is not there. The other seven rows pass on both - resolvers, which is the evidence the capability is declared on -- withholding - it would say flagd does not name variants, which is false, and would - attribute a missing flag to a capability the provider has; -* ``An integral float requested as an integer is coerced without loss`` -- under - ``@numeric-coercion``, which both suites declare as of this pass. It asks for - ``integral-float-flag``. This one is new to the failure list and is the price - of declaring that tag; the two suites explain why paying it is the honest - report, and the RPC suite's ``KnownDeviation`` summary disclaims it explicitly - so that a reader does not attribute it to flagd. - -``huge-integer-flag`` is asked for only by the single scenario under -``@large-integers``, which neither suite declares any more, so it is skipped -rather than failed -- the backend can put none of that tag's scenarios to the -provider, so a declaration would rest on nothing. ``@numeric-coercion`` is the -opposite case and is declared: two of its three scenarios do reach the provider, -and the two resolvers answer them differently. Both suites state the rule that -decides this. - -The failures are deliberately left as failures: they say something true about -the stack under test, and an ``xfail`` would say the provider is at fault when -it is the backend that is behind. **None of them is a ``KnownDeviation``**: a -deviation is for a behaviour the *provider* is required to have and does not, -and the provider was never given the flag to get wrong. Go and JavaScript both -record their equivalent gaps the same way and say so in the same words. - -**Two failures are the provider's, not the testbed's.** A full run is -``8 failed, 119 passed, 3 skipped``. Six of the eight are the three above on -each resolver. The seventh is on in-process alone: ``boolean-flag`` requested as -a Float resolves to ``1.0`` with reason ``STATIC`` and no error code, where the -mandatory wrong-type scenario asks for the caller's default. ``bool`` is a -subclass of ``int`` in Python, so the widening at ``flagd_core.py:113-114`` -- -``if isinstance(result.value, int): result.value = float(result.value)`` -- sees -a boolean as an integer, after ``_resolve`` has already let it through for a -Float request. It is the same shape as the boolean-satisfies-an-Integer finding -the suite's own README records against the in-memory provider, and it is a gap -in ``openfeature-flagd-core`` rather than in either resolver's transport: RPC -passes the row, because the server type-checks it. Recorded here rather than -declared as a ``KnownDeviation`` because the scenario is mandatory and -ungated -- it fails visibly on every run, which is the report, and a deviation -would add nothing a reader cannot already see. It is not a testbed gap and does -not go away when the image is bumped. - -The eighth is on RPC alone, and it is the one failure here that *does* carry a +The container lifecycle belongs to the TCK -- see its README for what +``tck_backend`` does with the declaration below, and Appendix F, "The control +API", for why the stack is started once and never restarted. What is left here +is the declaration and one free port for the scenarios that need a backend that +is not there. + +One stack and one ``HttpControl`` serve both suites, because one flagd process +serves both resolver ports: 8013 for RPC and 8015 for sync. The launchpad +registers no ``/reset`` and no ``/healthz``, so every ``prepare_scenario`` takes +the harness's documented ``/start`` fallback and one 404 is logged per session. + +**What a full run reports, and what each failure is.** ``8 failed, 119 passed, +3 skipped`` over the two resolvers. Read it here rather than counting: three +distinct causes account for all eight, and only two of them are the provider's. + +*Six failures are the backend's flag set.* flagd-testbed v3.8.0 seeds neither +``large-integer-flag`` nor ``integral-float-flag``, so the untagged precision +scenario, the ``max-int32`` row of the ``@variants`` outline and the lossless +``@numeric-coercion`` scenario fail ``FLAG_NOT_FOUND`` on each resolver alike. +open-feature/flagd-testbed#392 seeds all three of the flags the canonical set is +missing and says what each catches; bump the tag in ``docker-compose.yaml`` +beside this file when it lands. Left as failures rather than xfailed, because +they are true about the stack under test, and carrying no ``KnownDeviation`` in +either suite: the provider was never given the flag to get wrong. + +*One failure is ``openfeature-flagd-core``'s*, on in-process alone: +``boolean-flag`` requested as a Float resolves to ``1.0`` with reason ``STATIC`` +and no error code, where the mandatory wrong-type scenario asks for the caller's +default. ``bool`` is a subclass of ``int`` in Python and the int-to-float +widening does not exclude it. Filed as open-feature/python-sdk-contrib#417. RPC +passes the row, because the server type-checks it. No ``KnownDeviation``: the +scenario is mandatory and ungated, so it fails visibly on every run and an entry +would add nothing a reader cannot see. + +*One failure is flagd's*, on RPC alone, and it is the one failure here carrying a ``KnownDeviation``: ``float-flag`` (0.5) requested as an Integer comes back as -``0`` with no error code, where ``@numeric-coercion`` requires ``TYPE_MISMATCH`` -and the caller's default. The tag is declared and the scenario left to fail -rather than the tag withheld, because this resolver does attempt the coercion -and gets one direction wrong -- see ``test_rpc.py``. **The -in-process resolver passes this scenario**, refusing 0.5 locally, so the -deviation is recorded against RPC only. That asymmetry is the most interesting -result in this pair of suites and is the thing a shared declaration would have -hidden: the Java and Go flagd adoptions each record the same defect against both -of their resolvers, correctly, because theirs both narrow; Python is the -language where that would have been false. - -**Declaring ``@lifecycle`` surfaced one thing beyond a pass or a fail, and it is -worth reading before anyone treats the warning as noise.** The in-process -scenario ``Shutting down a provider that cannot reach its backend completes -promptly`` passes -- shutdown does return well inside the bound -- but it leaves -a ``PytestUnhandledThreadExceptionWarning`` behind it: gRPC's connectivity -polling thread raises ``ValueError: Cannot invoke RPC: Channel closed!`` from -``_poll_connectivity`` after ``shutdown`` has closed the channel underneath it. -So the provider closes the channel without first stopping the watcher that is -still using it. Reproducible on every run. - -It is **not** a ``KnownDeviation`` and not a scenario failure: nothing the -specification requires is unmet, the scenario asserts that shutdown completes -promptly and it does, and 2.5.3's double-shutdown scenario passes as well. It is -a shutdown-ordering race in ``openfeature-flagd-core``'s watcher that surfaces -as a stray traceback in a host application's logs during its own shutdown. -Recorded here because it is the one finding the six lifecycle scenarios produced -that neither a pass nor a failure would have carried, and because the six had -never been run against this provider before -- the capability was withheld from -the first pass to the sixth with nothing saying why. - -**The three skips are two scenarios, not three.** ``@large-integers`` gates a -single scenario and is withheld on both resolvers for the backend gap above, so -it skips twice; ``@reinitialization`` gates a single scenario and is declared on -in-process, which passes it, and withheld on RPC, which cannot reuse a closed -channel and is permitted not to by 2.5.2, so it skips once. Neither withholding -is a defect and neither carries a deviation. - -``targeting-key-flag``, new in the canonical set at the same revision, needs no -testbed change. It is the flag flagd-testbed's own ``targeting.feature`` already -uses -- same key, same ``hit``/``miss`` variants, same uuid -- seeded from -``flags/testing-flags.json`` by the launchpad's ``default`` configuration, so -the three ``@targeting`` scenarios pass on both resolvers as they stand. - -The four ``disabled-*`` flags, new in the canonical set at spec@009afe06, need -no testbed change either, and for a sturdier reason than a coincidence of names: -they *are* flagd-testbed's own, from ``flags/disabled-flags.json``, which the -launchpad merges into ``flags/allFlags.json`` along with every other non- -``selector-`` file in ``rawflags`` (``launchpad/pkg/json.go``) and serves under -the ``default`` configuration. So the four ``@disabled-flags`` rows pass on both -resolvers as they stand. The harness also seeds ``disabled-object-flag`` and -``cross-flagset-flag``, which the canonical set deliberately does not ask for -- -an Object resolution would need ``@object`` as well, and a scenario needing two -capability tags cannot be one row of a single outline. - -The falsy flags used to fail the same way and no longer do. ``ba002ce8`` renamed -them to ``boolean-zero-flag``, ``integer-zero-flag`` and ``string-zero-flag``, -which is what ``flags/zero-flags.json`` in the testbed has always called them, -with the same ``zero``/``non-zero`` variants the scenarios assert. Those three -rows were never a gap in the backend, only a disagreement about names. +``0`` with no error code. **The in-process resolver passes that scenario**, which +is why the deviation is recorded against RPC only; ``test_rpc.py`` and +``test_in_process.py`` carry the measurement on each side. + +*The three skips are two scenarios.* ``@large-integers`` gates one and is +withheld on both resolvers, so it skips twice; ``@reinitialization`` gates one, +which in-process declares and passes and RPC withholds, so it skips once. Each +suite gives its own reason beside its declaration. + +**One finding came out of a scenario that passes**, so neither the results nor +the report has anywhere to put it: the in-process *Shutting down a provider that +cannot reach its backend completes promptly* returns well inside its bound and +leaves a ``PytestUnhandledThreadExceptionWarning`` behind it -- gRPC's +connectivity polling thread raising ``ValueError: Cannot invoke RPC: Channel +closed!`` after ``shutdown`` closed the channel underneath it. Reproducible on +every run, and filed as open-feature/python-sdk-contrib#419. Not a deviation: +nothing required is unmet. It is noted here because a reader who sees the warning +should know it is a recorded finding rather than noise. + +The canonical set's ``targeting-key-flag``, four ``disabled-*`` flags and three +falsy flags need no testbed change: they are flagd-testbed's own flags, which is +why the canonical set adopted their names and variants, and the launchpad's +``default`` configuration serves them as they stand. """ from __future__ import annotations @@ -166,13 +77,13 @@ def compose_backend() -> ComposeBackend: """The stack under test, as the TCK's ``tck_backend`` fixture wants it. Both resolver ports are declared even though each suite uses one of them, - because both suites share this stack and the harness checks at startup that - the Compose file publishes everything it was told about. The launchpad's own - 8080 is exposed automatically and must not be listed. + because both suites share this stack. The Compose file publishes flagd's + OFREP port as well, which nothing here asks for: it is the same file the + OFREP adoption uses, and a port nobody declares is neither waited on nor + looked up. - The path is absolute rather than relative to the package directory -- which - is what the harness resolves a relative one against, and where ``poe test`` - runs from -- so that running pytest from the repository root works too. + The path is absolute so that pytest run from the repository root works too; + a relative one resolves against the working directory. """ return ComposeBackend( compose_file=Path(__file__).parent / "docker-compose.yaml", @@ -185,12 +96,10 @@ def closed_port(tck_backend: RunningBackend) -> int: """A port on localhost with nothing listening, for the ``@unavailable`` scenarios. Discovered by binding and releasing rather than hard-coded, because the - testbed's own host ports are mapped dynamically and a hard-coded number could - collide with one. Depending on ``tck_backend`` orders this after the stack has - taken its ports, which is what makes the remaining race negligible. - - Deliberately not a port on the Compose stack: that has to stay up for the - whole session, and simulated outages belong to the control API. + stack's own host ports are mapped dynamically and a fixed number could + collide with one; depending on ``tck_backend`` orders this after the stack + has taken its ports. Deliberately not a port on the stack, which has to stay + up -- simulated outages belong to the control API. """ with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as probe: probe.bind(("127.0.0.1", 0)) diff --git a/providers/openfeature-provider-flagd/tests/tck/suite.py b/providers/openfeature-provider-flagd/tests/tck/suite.py index 75796c3e..30e3d720 100644 --- a/providers/openfeature-provider-flagd/tests/tck/suite.py +++ b/providers/openfeature-provider-flagd/tests/tck/suite.py @@ -8,10 +8,8 @@ Everything they share lives here; everything that differs lives in ``test_rpc.py`` and ``test_in_process.py`` next to it, where a reader can see the whole of a -resolver's declaration in one place. - -There is no container wiring here any more. The TCK owns the stack -- see -``conftest.py`` -- so what is left is the timings, which are flagd's own and +resolver's declaration in one place. The TCK owns the container stack (see +``conftest.py``), so what is left here is the timings, which are flagd's own and interact, and the one function that turns a resolver plus a running endpoint into a ``TckConfig``. """ @@ -127,11 +125,8 @@ class ResolverSuite: capabilities: frozenset[Capability] """What this resolver was run against the suite and seen to satisfy. - Evidence from a run rather than from reading the resolver, which is Appendix - F's rule and not merely a preference: source inspection has been wrong in - both directions here, and the two resolvers differ on reuse in a way that - reading them does not show. See each suite module for the evidence behind - every entry, and behind every omission. + See each suite module for the evidence behind every entry, and behind every + omission. """ ready_timeout: float @@ -142,10 +137,9 @@ class ResolverSuite: **Per resolver, and that is the whole reason this field is here rather than shared.** The two resolvers do not deviate alike: RPC narrows ``float-flag`` to ``0`` where in-process refuses it, so an entry naming - :attr:`~Capability.NUMERIC_COERCION` is true of one and false of the other. - Attaching it to both -- which is what Java and Go each do, correctly, because - *their* two resolvers do behave identically -- would publish a defect against - the resolver that does not have it. + :attr:`~Capability.NUMERIC_COERCION` is true of one and false of the other, + and attaching it to both would publish a defect against the resolver that + does not have it. """ @@ -156,14 +150,10 @@ def build_config( ) -> TckConfig: """Wire one resolver up to the running testbed. - The host and port are read off ``backend.endpoint`` here, inside the factory - and after the stack is up: Compose maps host ports dynamically, so they do - not exist earlier -- and they stay valid for the whole session because - nothing ever restarts a container. Outages are simulated inside the running - stack through ``backend.control`` instead. - - The host comes from the endpoint rather than being written as - ``"localhost"``: with a remote Docker daemon, Docker Desktop on some + The host and port are read off ``backend.endpoint`` inside the factory, which + is why the factory exists: the mapped host port does not exist until the + stack is up. The host comes from the endpoint rather than being written as + ``"localhost"``, because with a remote Docker daemon, Docker Desktop on some platforms or a rootless setup it is neither localhost nor predictable. """ endpoint = backend.endpoint diff --git a/providers/openfeature-provider-flagd/tests/tck/test_in_process.py b/providers/openfeature-provider-flagd/tests/tck/test_in_process.py index 457a098a..0dca555a 100644 --- a/providers/openfeature-provider-flagd/tests/tck/test_in_process.py +++ b/providers/openfeature-provider-flagd/tests/tck/test_in_process.py @@ -23,88 +23,68 @@ from tests.tck.suite import IN_PROCESS_PORT, ResolverSuite, build_config # Every capability below was declared, the suite run, and the scenarios seen to -# pass. The code references say where the behaviour lives, so a reader can check -# the claim -- they are not the evidence for it. -# -# The distinction is Appendix F's, stated there since spec@26362f85 and worth -# repeating here because this file used to get it backwards: source inspection is -# unreliable in both directions, which @reinitialization below shows from the -# other side -- this resolver does support reuse, and reading it does not say so. -# -# EVENTS -# grpc_watcher.py:262 emits PROVIDER_READY once the first sync payload has -# been applied -- note "applied", not "received": the ruleset is written to -# the evaluator at grpc_watcher.py:254 before ready is emitted, so a scenario -# that evaluates immediately after ready cannot race the first sync. -# -# STALE -# grpc_watcher.py:178-190: the channel-connectivity callback emits -# PROVIDER_STALE on TRANSIENT_FAILURE and starts a timer that escalates to -# PROVIDER_ERROR only once retry_grace_period expires. -# +# pass. The code references say where the behaviour lives, so that a reader can +# check the claim -- the run is what it rests on, which is Appendix F's rule and +# not a preference here. @reinitialization below shows why from the other side: +# this resolver does support reuse, and reading it does not say so. +# +# EVENTS grpc_watcher.py:262 emits PROVIDER_READY once the first sync +# payload has been *applied*, not merely received: the ruleset is +# written to the evaluator at grpc_watcher.py:254 first, so a +# scenario evaluating straight after ready cannot race the sync. +# STALE grpc_watcher.py:178-190 emits PROVIDER_STALE on TRANSIENT_FAILURE +# and starts the timer escalating to PROVIDER_ERROR only once +# retry_grace_period expires. # CONFIGURATION_CHANGE -# in_process.py:34 emits PROVIDER_CONFIGURATION_CHANGED naming exactly the -# keys that FlagdCore reports as changed, from every sync payload the watcher -# applies. -# -# OBJECT -# in_process.py:122 resolves structured values from the local ruleset. -# -# VARIANTS -# flagd_core.py returns the variant name it selected with every resolution, -# and in_process.py carries it into the resolution details. The ruleset is -# keyed by variant, so there is always one to report. +# in_process.py:34 emits PROVIDER_CONFIGURATION_CHANGED naming +# exactly the keys FlagdCore reports as changed, from every sync +# payload the watcher applies. +# OBJECT in_process.py:122 resolves structured values from the local +# ruleset. +# VARIANTS flagd_core.py returns the variant it selected with every +# resolution and in_process.py carries it into the details; the +# ruleset is keyed by variant, so there is always one to report. +# TARGETING targeting.py:40-41 puts the evaluation context's targeting key +# into the JSON-logic context under `targetingKey` and +# flagd_core.py:154 evaluates the rule against it, so +# targeting-key-flag selects hit or miss locally. +# UNAVAILABLE_INIT +# grpc_watcher.py:151 raises ProviderNotReadyError once the +# blocking init deadline passes without a synced ruleset, which the +# SDK's registry turns into PROVIDER_ERROR. # # DISABLED_FLAGS -# New at spec@009afe06. All four rows pass, and here that is the -# unsurprising half of the story: evaluation is local, so the resolver has -# the flag's state and the caller's default in the same call. -# flagd_core.py:143-145 returns the caller's `default_value` with reason -# Reason.DISABLED the moment a flag's state is DISABLED, before any -# targeting or variant selection. flagd_core.py:199-200 then skips the type -# check for that reason, which is what stops the substituted default from -# being re-typed against the flag it did not come from. +# All four rows pass, and here that is the unsurprising half of the story: +# evaluation is local, so the resolver has the flag's state and the caller's +# default in the same call. flagd_core.py:143-145 returns the caller's +# `default_value` with reason Reason.DISABLED the moment a flag's state is +# DISABLED, before any targeting or variant selection. flagd_core.py:199-200 +# then skips the type check for that reason, which is what stops the +# substituted default from being re-typed against the flag it did not come +# from. # # Probed directly, each of the four flags resolves to the caller's default # with reason Reason.DISABLED, no variant and no error code -- the SDK's -# enum, where RPC hands back the server's bare 'DISABLED' string. Neither -# is asserted by the scenarios and 2.2.5 requires neither, so the difference -# is recorded rather than acted on. It is the sort of divergence between the -# two resolvers this pair of suites exists to surface. -# -# TARGETING -# targeting.py:40-41 puts the evaluation context's targeting key into the -# JSON-logic context under `targetingKey`, and flagd_core.py:154 evaluates -# the flag's rule against it, so targeting-key-flag selects `hit` or `miss` -# locally. -# -# UNAVAILABLE_INIT -# grpc_watcher.py:151 raises ProviderNotReadyError once the blocking init -# deadline passes without a synced ruleset, which the SDK's registry turns -# into PROVIDER_ERROR. +# enum, where RPC hands back the server's bare 'DISABLED' string. Nothing +# asserts either, so the difference is recorded rather than acted on; it is +# the sort of divergence between the two resolvers this pair of suites exists +# to surface. # # LIFECYCLE -# Declared on a run, and the run is the point: this capability had been -# withheld here since the first pass with nothing anywhere saying why, so -# the six lifecycle scenarios had never been put to this resolver at all. -# Declaring it and running them settles it -- all six execute and all six -# pass. There was no reason; there was an omission that every later pass -# inherited because the file next door treated it as given. +# All six scenarios execute and all six pass. # # It is a real question here rather than a formality, which is the test the -# capability's own documentation sets. An SDK synthesises PROVIDER_READY +# capability's own documentation sets: an SDK synthesises PROVIDER_READY # around `initialize` for any provider, so the readiness scenario is vacuous -# for a provider that does nothing during initialisation -- a NoOpProvider -# passes it. This resolver syncs the entire ruleset and applies it to the -# evaluator before ready is emitted (grpc_watcher.py:254-262), and -# initialisation can and does fail (grpc_watcher.py:151), so both terminal -# outcomes the feature file asserts are outcomes this provider actually -# reaches. +# for a provider that does nothing during initialisation. This resolver syncs +# the entire ruleset and applies it to the evaluator before ready is emitted +# (grpc_watcher.py:254-262), and initialisation can and does fail +# (grpc_watcher.py:151), so both terminal outcomes the feature file asserts +# are outcomes this provider actually reaches. # -# Java, Go and JavaScript all declare it on both resolvers, and Go's -# adoption records having made and reverted this exact mistake: withholding -# it left that suite blind to six scenarios another language was running -# against the same provider. Python was the last of the four still doing so. +# Running them also produced a finding neither a pass nor a fail carries -- +# a shutdown-ordering race that leaves a stray traceback behind a *passing* +# scenario. conftest.py records it; open-feature/python-sdk-contrib#419. # # NUMERIC_COERCION # Declared here and declared on RPC too, but they are not the same claim, @@ -113,7 +93,8 @@ # resolvers, same run -- `float-flag` (0.5) requested as an Integer is a # TYPE_MISMATCH returning the caller's default here, and comes back as `0` # with no error code at all on RPC. One provider, two resolvers, opposite -# answers to the question the capability exists to ask. +# answers to the question the capability exists to ask +# (open-feature/python-sdk-contrib#420). # # Why it holds here: evaluation is local, so flagd_core.py:25 admits only # `int` for an integer request and `_check_type` (flagd_core.py:228-231) @@ -126,105 +107,56 @@ # deliberately not mirrored onto this one. # # The tag's third scenario fails, and not for a reason about coercion: -# flagd-testbed seeds no `integral-float-flag`, so it fails FLAG_NOT_FOUND -# (`Flag with key integral-float-flag not present in flag store.`), the same -# backend gap the conftest records for `large-integer-flag`. What this -# resolver would do with a seeded 10.0 is still unmeasured -- the `(int,)` -# rule above says it would refuse it, but that is a reading of the source -# and nothing here observes it. The gap is recorded rather than treated as a -# reason to withhold: two of the three scenarios do reach this provider and -# both pass, and withholding on the strength of the one the backend cannot -# ask would discard the finding that the two resolvers differ. +# flagd-testbed seeds no `integral-float-flag`, so it fails FLAG_NOT_FOUND. +# What this resolver would do with a seeded 10.0 is still unmeasured -- the +# `(int,)` rule above says it would refuse it, but that is a reading of the +# source and nothing here observes it. Withholding the tag over the one +# scenario this backend cannot ask would discard the finding that the two +# resolvers differ. # # REINITIALIZATION -# New at spec@fc99d5ac, which gated the scenario "A provider that was shut -# down can be initialized again" that had been untagged before it. Declared -# here and withheld on RPC, and again the two resolvers genuinely differ: -# this one shuts down and starts again serving `boolean-flag` correctly, -# while RPC evaluates against a closed channel. Measured on the same run. -# -# This was withheld until this pass, on the argument that declaring it would -# be vacuous -- the scenario also carries @lifecycle, which this suite did -# not declare, so it would have skipped on that tag regardless. The argument -# was sound and is now spent: @lifecycle is declared above, the scenario -# runs, and the claim is examined rather than asserted. -# -# Requirement 2.5.2 says a provider SHOULD revert to its uninitialized -# state and that "some providers MAY allow reinitialization", so reuse is -# permitted rather than required -- which is why RPC's withholding needs no -# KnownDeviation, and why declaring it here is a claim worth making rather -# than a mandatory box ticked. +# Declared here and withheld on RPC, and again the two resolvers genuinely +# differ: this one shuts down and starts again serving `boolean-flag` +# correctly, while RPC evaluates against a closed channel. Measured on the +# same run. Requirement 2.5.2 permits reuse rather than requiring it, which +# is why RPC's withholding needs no KnownDeviation and why declaring it here +# is a claim worth making rather than a box ticked. # # Not declared, and why: # # LARGE_INTEGERS -# Withheld, and this is a change: it was declared here until this pass and -# failed on every run. Exactly one scenario carries the tag, and it asks for -# `huge-integer-flag`, which flagd-testbed v3.8.0 does not seed -- so the -# declaration was a claim with no evidence behind it either way, and its -# failure (`Flag with key huge-integer-flag not present in flag store.`) -# read as a provider defect while establishing nothing about the provider. -# Python's `int` is unbounded and the ruleset arrives as JSON text parsed -# with `json.loads` (flagd_core.py:73), so nothing here would narrow the -# value; the suite simply cannot show that. -# -# The rule this and NUMERIC_COERCION are both decided by is **Appendix F's -# sixth declaring rule** -- once a provider is attempting a capability, -# declare it when at least one scenario gating it can actually be put to the -# provider and withhold only when none can, the unit being the scenario and -# not the tag. It is cited rather than restated: the wording these two suites -# used last pass is what went into the appendix at spec@4cab0320, so the -# appendix is now where it lives and a copy here would be a second place for -# it to drift. -# -# The opening clause matters and was added at spec@aa2ad24f after the Go -# implementation found the rule forcing declarations it should not: it -# decides whether a question is *askable*, not whether the provider owes an -# answer, and that second question comes first. Both tags clear it here -- -# this resolver does coerce, correctly in both directions it can be asked, -# and nothing about it declines to resolve a large integer -- so what is left -# for rule six to decide is the fixture gap. The two gaps look like the same -# missing-fixture problem and are not. @numeric-coercion has three scenarios -# and this backend can ask two of them, which is what makes the declaration -# mean something and the third failure a footnote. @large-integers has one, -# and this backend can ask none of it. -# -# No KnownDeviation for it, in either shape. That is the rule's first -# consequence: a scenario failing because the backend serves no fixture for -# it is not a provider defect, and an entry would attribute the testbed's gap -# to the provider. Go and JavaScript withhold it for this same reason; Java -# cannot declare it at all, because its integer accessor is 32 bits, which is -# a third thing again and not this one. -# -# And the second consequence, which this sentence exists to satisfy: a -# capability withheld for a backend gap is temporary in a way one withheld by -# choice is not. open-feature/flagd-testbed#392 adds `huge-integer-flag`; -# declare the tag when the image carries it, or this withholding outlives its -# reason and starts reading as a claim about the provider. +# Withheld. Exactly one scenario carries the tag, it asks for +# `huge-integer-flag`, and flagd-testbed v3.8.0 seeds no such flag -- so this +# backend can put none of the tag's scenarios to this provider, and Appendix +# F's sixth declaring rule says withhold. Python's `int` is unbounded and the +# ruleset arrives as JSON text parsed with `json.loads` (flagd_core.py:73), +# so nothing here would narrow the value -- and the suite cannot show that, +# which is the point. +# +# Contrast @numeric-coercion above, which is a fixture gap too and is +# declared: two of its three scenarios do reach the provider, and the two +# resolvers answer them differently. The rule counts scenarios, not tags. +# +# No KnownDeviation, in either shape: the gap is the backend's and an entry +# would attribute it to the provider. This withholding is temporary in a way +# the ones above are not -- open-feature/flagd-testbed#392 adds the flag; +# declare the tag when the image carries it, or it outlives its reason and +# starts reading as a claim about the provider. # # STANDARD_REASONS -# New at spec@c342461a, which moved every resolution-reason assertion out of -# the other feature files and into reason.feature, gated as a whole. A claim -# rather than an exemption: 2.2.5 is a SHOULD that permits "some other -# string", so declaring the tag says this provider uses the standard -# vocabulary with the standard meanings. -# -# Declared on a run rather than on the source. All nine scenarios pass -- -# the four rule-less rows as STATIC, an unknown flag and a type mismatch as -# ERROR beside their error codes, TARGETING_MATCH for the matched rule and -# DEFAULT for the miss, DISABLED for a disabled flag. The last three need -# @targeting and @disabled-flags as well, which this suite declares, so none -# of the file is skipped here. -# -# The DISABLED row resolves through Reason.DISABLED here, the SDK's own -# enum, where RPC hands back flagd's bare 'DISABLED' string -- noted under +# Declared on a run rather than on the source. All nine scenarios pass -- the +# four rule-less rows as STATIC, an unknown flag and a type mismatch as ERROR +# beside their error codes, TARGETING_MATCH for the matched rule and DEFAULT +# for the miss, DISABLED for a disabled flag. The last three need @targeting +# and @disabled-flags as well, which this suite declares, so none of the file +# is skipped here. +# +# The DISABLED row resolves through Reason.DISABLED here, the SDK's own enum, +# where RPC hands back flagd's bare 'DISABLED' string -- noted under # DISABLED_FLAGS above. The step compares the reason as text, so both pass. # # CACHING -# Reserved in the Capability enum; no scenario carries the tag. Declaring a -# capability nothing exercises would be a claim with no evidence behind it, -# so it is left out of both suites. @targeting was reserved alongside it -# until spec@26362f85 gave it three scenarios, and is now declared above. +# Reserved, and the harness refuses it: no scenario carries the tag. IN_PROCESS_CAPABILITIES = frozenset( { Capability.EVENTS, @@ -257,10 +189,8 @@ def tck_config(tck_backend: RunningBackend, closed_port: int) -> TckConfig: """The whole of this adoption's wiring. - ``tck_backend`` is the TCK's own session-scoped fixture: it has already - started the Compose file ``tests/tck/conftest.py`` declares, discovered the - dynamically mapped host ports, built the control against the launchpad and - waited for it to accept commands. + ``tck_backend`` is the TCK's own session-scoped fixture: the stack is up and + its control is ready by the time this runs. """ return build_config(IN_PROCESS_SUITE, tck_backend, closed_port) diff --git a/providers/openfeature-provider-flagd/tests/tck/test_rpc.py b/providers/openfeature-provider-flagd/tests/tck/test_rpc.py index f36375c3..f8e80f8f 100644 --- a/providers/openfeature-provider-flagd/tests/tck/test_rpc.py +++ b/providers/openfeature-provider-flagd/tests/tck/test_rpc.py @@ -21,96 +21,61 @@ from tests.tck.suite import RPC_PORT, ResolverSuite, build_config # Every capability below was declared, the suite run, and the scenarios seen to -# pass. The code references say where the behaviour lives, so a reader can check -# the claim -- they are not the evidence for it. -# -# The distinction is Appendix F's, stated there since spec@26362f85 and worth -# repeating here because this file used to get it backwards: source inspection is -# unreliable in both directions. @reinitialization below is the case in point -- -# RPC's shutdown reverts enough of its own state to read as support for reuse, -# and the channel underneath cannot be rebuilt. -# -# EVENTS -# grpc.py:261 emits PROVIDER_READY when the evaluation stream delivers its -# 'provider_ready' message. -# -# STALE -# grpc.py:202-212: the channel-connectivity callback emits PROVIDER_STALE on -# TRANSIENT_FAILURE and only then starts a timer that escalates to -# PROVIDER_ERROR once retry_grace_period expires. -# -# Worth calling out, because the Go provider does NOT do this: its RPC -# resolver sends ProviderError directly on connection loss and never emits -# PROVIDER_STALE, which is filed as go-sdk-contrib#939 and is why the Go -# adoption withholds this capability for RPC. Python has no such asymmetry -- -# both of its resolvers share the same state-change callback shape -- so the -# capability is declared here. -# +# pass. The code references say where the behaviour lives, so that a reader can +# check the claim -- the run is what it rests on, which is Appendix F's rule and +# not a preference here. @reinitialization is the case in point: this resolver's +# shutdown reverts enough of its own state to read as support for reuse, and the +# channel underneath cannot be rebuilt. +# +# EVENTS grpc.py:261 emits PROVIDER_READY when the evaluation stream +# delivers its 'provider_ready' message. +# STALE grpc.py:202-212 emits PROVIDER_STALE on TRANSIENT_FAILURE and +# only then starts the timer escalating to PROVIDER_ERROR once +# retry_grace_period expires. # CONFIGURATION_CHANGE -# grpc.py:302 emits PROVIDER_CONFIGURATION_CHANGED with the changed keys, and -# grpc.py:298-300 evicts exactly those keys from the LRU cache, so the -# re-evaluation the scenario performs afterwards cannot be served a stale -# cached value. -# -# OBJECT -# grpc.py:336 resolves structured values through ResolveObject. -# -# VARIANTS -# grpc.py:449 carries the response's `variant` field into the resolution -# details for every typed call, and flagd names a variant for every flag in -# the testbed's set. +# grpc.py:302 emits PROVIDER_CONFIGURATION_CHANGED with the changed +# keys, and grpc.py:298-300 evicts exactly those keys from the LRU +# cache -- so the re-evaluation the scenario performs afterwards +# cannot be served a stale cached value. +# OBJECT grpc.py:336 resolves structured values through ResolveObject. +# VARIANTS grpc.py:449 carries the response's `variant` into the resolution +# details for every typed call. Seven of the outline's eight rows +# pass; the eighth asks for a flag the backend does not seed. +# TARGETING grpc.py:492 puts the evaluation context's targeting key into the +# request's context struct, so the server evaluates +# targeting-key-flag's rule against it and answers hit or miss. +# UNAVAILABLE_INIT +# grpc.py:175 raises ProviderNotReadyError once the blocking init +# deadline passes without a connection, which the SDK's registry +# turns into PROVIDER_ERROR. # # DISABLED_FLAGS -# New at spec@009afe06. All four rows pass, which is worth saying plainly -# because the appendix's own rationale for gating the tag predicts they -# would not: it reasons that a provider "whose backend decides, such as one -# speaking OFREP", cannot substitute a default the server never saw. RPC is -# a remote evaluator by exactly that description, and it substitutes anyway. -# -# Measured, and then read back to find out how. flagd answers a disabled -# flag with reason DISABLED, no variant, and the zero value of the response -# proto -- ResolveBoolean's `value` field is simply unset -- and -# grpc.py:468-472 replaces that with the caller's `default_value` whenever -# the reason is DEFAULT or DISABLED and no variant came back. So the -# substitution is local even though the evaluation is not: what crosses the -# wire is the signal, and the provider already holds the default. +# All four rows pass, and the measurement is worth keeping because a remote +# evaluator satisfying this is not obvious. flagd answers a disabled flag +# with reason DISABLED, no variant, and the zero value of the response proto +# -- ResolveBoolean's `value` field is simply unset -- and grpc.py:468-472 +# replaces that with the caller's `default_value` whenever the reason is +# DEFAULT or DISABLED and no variant came back. So the substitution is local +# even though the evaluation is not: what crosses the wire is the signal, and +# the provider already holds the default. # # Probed directly, each of the four flags resolves to the caller's default -# with reason 'DISABLED', no variant and no error code. The reason arrives -# as the server's bare string rather than the SDK's Reason enum, which the -# scenarios do not assert and 2.2.5 does not require -- worth noting only -# because the in-process resolver differs there, returning Reason.DISABLED. -# -# TARGETING -# grpc.py:492 puts the evaluation context's targeting key into the request's -# context struct, so the server evaluates targeting-key-flag's rule against -# it and answers `hit` or `miss`. -# -# UNAVAILABLE_INIT -# grpc.py:175 raises ProviderNotReadyError once the blocking init deadline -# passes without a connection, which the SDK's registry turns into -# PROVIDER_ERROR. +# with reason 'DISABLED', no variant and no error code. The reason arrives as +# the server's bare string rather than the SDK's Reason enum, which nothing +# asserts -- noted only because the in-process resolver differs, returning +# Reason.DISABLED. # # LIFECYCLE -# Declared on a run, and the run is the point: this capability had been -# withheld here since the first pass with nothing anywhere saying why -- the -# @reinitialization note below treated it as given rather than justifying it -# -- so the six lifecycle scenarios had never been put to this resolver at -# all. Declaring it and running them settles it: all six execute, five pass, -# and the sixth is the @reinitialization scenario dealt with below. +# All six scenarios execute, five pass, and the sixth is the +# @reinitialization one dealt with below. # -# It is a real question here rather than a formality. An SDK synthesises -# PROVIDER_READY around `initialize` for any provider, so the readiness -# scenario is vacuous for one that does nothing during initialisation -- a -# NoOpProvider passes it. This resolver blocks until the evaluation stream -# is up and raises ProviderNotReadyError when the deadline passes -# (grpc.py:175, grpc.py:261), so both terminal outcomes the feature file -# asserts are outcomes this provider actually reaches. -# -# Java, Go and JavaScript all declare it on both resolvers, and Go's -# adoption records having made and reverted this exact mistake: withholding -# it left that suite blind to six scenarios another language was running -# against the same provider. Python was the last of the four still doing so. +# It is a real question here rather than a formality, which is the test the +# capability's own documentation sets: an SDK synthesises PROVIDER_READY +# around `initialize` for any provider, so the readiness scenario is vacuous +# for one that does nothing during initialisation. This resolver blocks until +# the evaluation stream is up and raises ProviderNotReadyError when the +# deadline passes (grpc.py:175, grpc.py:261), so both terminal outcomes the +# feature file asserts are outcomes this provider actually reaches. # # NUMERIC_COERCION # Declared, and failing the lossy scenario -- deliberately, and the failure @@ -126,141 +91,68 @@ # the server source: the scenario fails with `flag 'float-flag' resolved to # 0 (int), expected 1 (int)`. # -# Declared rather than withheld because this resolver *attempts* the -# coercion and gets one direction wrong, which is the case Appendix F's -# numeric-coercion note now names outright: a provider in that position -# declares the capability and lets the scenario fail. That note said the -# opposite until spec@045950ca -- withhold, and say which it is -- and two -# of the four adoptions followed it there. `integer-flag` requested as a -# Float passes here, so withdrawing the tag would turn a real, specific -# defect into a skip indistinguishable from a provider that declines to -# coerce at all, which is the failure mode the deviation field exists to -# prevent. -# -# **The in-process resolver passes this scenario.** It refuses 0.5 as an +# **The in-process resolver passes that scenario.** It refuses 0.5 as an # Integer with TYPE_MISMATCH, because it evaluates locally and never asks -# flagd. One provider, two resolvers, opposite answers -- which is why the -# deviation below is on this suite only and is not mirrored onto that one. -# Java and Go both attach their equivalent entry to both of their resolvers, -# correctly, because in those languages both narrow identically; Python is -# the language where that would be false. -# -# Worth keeping because it was got wrong twice, in two languages, by reading -# the source: Python's RPC resolver narrows exactly as Go's and Java's do, -# and only its in-process resolver is the exception -- the only such resolver -# in the four languages. The claim that this path returns INVALID_ARGUMENT -# for a float-valued flag was made and retracted by an earlier revision of -# this file, and asserted about Python by the Go suite until it was corrected -# there. INVALID_ARGUMENT is what grpc.py:461-462 would map to -# TypeMismatchError if it ever arrived; for a float-valued flag it does not. -# A run settled it; neither reading did. +# flagd, while `integer-flag` requested as a Float widens correctly on both. +# One provider, two resolvers, opposite answers to the question this +# capability exists to ask -- which is why the deviation below is on this +# suite only and is deliberately not mirrored onto that one, and why the tag +# is declared and left to fail rather than withheld: a skip cannot say "it +# coerces, and one direction is wrong". Filed as +# open-feature/python-sdk-contrib#420. # # The tag's third scenario fails for a reason that is not flagd's: # flagd-testbed seeds no `integral-float-flag`, so it is FLAG_NOT_FOUND. The -# conftest records it and the deviation summary disclaims it, which is the -# shape Java uses; it is not a reason to withhold the tag. +# conftest records it and the deviation summary disclaims it; it is not a +# reason to withhold the tag. # # Not declared, and why: # # LARGE_INTEGERS -# Withheld, and this is a change: it was declared here until this pass and -# failed on every run. Exactly one scenario carries the tag, and it asks for -# `huge-integer-flag`, which flagd-testbed v3.8.0 does not seed -- so the -# declaration was a claim with no evidence behind it either way, and its -# failure read as a provider defect while establishing nothing about the -# provider. Nothing in this path would narrow the value: flagd holds every -# numeric variant as a float64 and 2^53 - 1 is exactly the largest integer a -# float64 represents without rounding, the server casts it to the int64 of -# ResolveIntResponse.value, and grpc.py:448 hands that to the SDK as an -# unbounded Python int. Nothing in between is 32 bits wide. The suite simply -# cannot show that. -# -# The rule this and NUMERIC_COERCION are both decided by is **Appendix F's -# sixth declaring rule** -- once a provider is attempting a capability, -# declare it when at least one scenario gating it can actually be put to the -# provider and withhold only when none can, the unit being the scenario and -# not the tag. It is cited rather than restated: the wording these two suites -# used last pass is what went into the appendix at spec@4cab0320, so the -# appendix is now where it lives and a copy here would be a second place for -# it to drift. -# -# The opening clause matters and was added at spec@aa2ad24f after the Go -# implementation found the rule forcing declarations it should not: it -# decides whether a question is *askable*, not whether the provider owes an -# answer, and that second question comes first. Both tags clear it here -- -# this resolver does attempt the coercion, and nothing about it declines to -# resolve a large integer -- so what is left for rule six to decide is the -# fixture gap. The two gaps look like the same -# missing-fixture problem and are not. @numeric-coercion has three scenarios -# and this backend can ask two of them -- and their answers differ between -# the two resolvers, which is the finding a withholding would have buried. -# @large-integers has one, and this backend can ask none of it. -# -# No KnownDeviation for it, in either shape. That is the rule's first -# consequence: a scenario failing because the backend serves no fixture for -# it is not a provider defect, and an entry would attribute the testbed's gap -# to the provider. Go and JavaScript withhold it for this same reason; Java -# cannot declare it at all, because its integer accessor is 32 bits, which is -# a third thing again and not this one. -# -# And the second consequence, which this sentence exists to satisfy: a -# capability withheld for a backend gap is temporary in a way one withheld by -# choice is not. open-feature/flagd-testbed#392 adds `huge-integer-flag`; -# declare the tag when the image carries it, or this withholding outlives its -# reason and starts reading as a claim about the provider. +# Withheld. Exactly one scenario carries the tag, it asks for +# `huge-integer-flag`, and flagd-testbed v3.8.0 seeds no such flag -- so this +# backend can put none of the tag's scenarios to this provider, and Appendix +# F's sixth declaring rule says withhold. Nothing in this path would narrow +# the value: flagd holds every numeric variant as a float64 and 2^53 - 1 is +# exactly the largest integer a float64 represents without rounding, the +# server casts it to the int64 of ResolveIntResponse.value, and grpc.py:448 +# hands that to the SDK as an unbounded Python int. Nothing in between is 32 +# bits wide -- and the suite cannot show that, which is the point. +# +# Contrast @numeric-coercion above, which is a fixture gap too and is +# declared: two of its three scenarios do reach the provider, and the two +# resolvers answer them differently. The rule counts scenarios, not tags. +# +# No KnownDeviation, in either shape: the gap is the backend's and an entry +# would attribute it to the provider. This withholding is temporary in a way +# the ones above are not -- open-feature/flagd-testbed#392 adds the flag; +# declare the tag when the image carries it, or it outlives its reason and +# starts reading as a claim about the provider. # # REINITIALIZATION -# New at spec@fc99d5ac, which gated the scenario "A provider that was shut -# down can be initialized again" that had been untagged before it. Withheld, -# and for one reason rather than the two this note used to give. -# -# RPC genuinely does not support reuse, measured rather than reasoned about: -# with LIFECYCLE declared the scenario runs, and it fails with `boolean-flag` -# resolving to the code default because grpc.py:420 raises "Cannot invoke RPC -# on closed channel!". shutdown() closes the channel and the second -# initialize() does not rebuild it, so the provider evaluates against a -# closed connection rather than failing outright -- exactly the shape the -# specification's own note on this capability describes. Requirement 2.5.2 -# says a provider SHOULD revert to its uninitialized state and that "some -# providers MAY allow reinitialization", so reuse is permitted rather than -# required and declining it is a choice the specification offers. Hence no -# KnownDeviation entry: there is no requirement to deviate from. -# -# The second reason is gone, and it was the load-bearing one for the wrong -# thing. It ran: the scenario also carries @lifecycle, neither resolver -# declared LIFECYCLE, so declaring REINITIALIZATION alone would leave the -# scenario skipped and the claim unexamined. True at the time, but it rested -# on a withholding that nothing justified, and it is what kept @lifecycle -# unexamined for six passes. LIFECYCLE is declared above now, so the -# scenario runs and this withholding rests on the measurement alone -- which -# is where it should always have rested. The in-process suite declares -# REINITIALIZATION for the same reason in reverse: it runs, and it passes. +# Withheld, on the measurement: with LIFECYCLE declared the scenario runs, +# and it fails with `boolean-flag` resolving to the code default because +# grpc.py:420 raises "Cannot invoke RPC on closed channel!". shutdown() +# closes the channel and the second initialize() does not rebuild it, so the +# provider evaluates against a closed connection rather than failing +# outright. Requirement 2.5.2 permits reuse rather than requiring it, so this +# is a choice the specification offers and there is no requirement to deviate +# from: no KnownDeviation. The in-process suite declares it for the same +# reason in reverse -- it runs, and it passes. # # STANDARD_REASONS -# New at spec@c342461a, which moved every resolution-reason assertion out of -# the other feature files and into reason.feature, gated as a whole. A claim -# rather than an exemption: 2.2.5 is a SHOULD that permits "some other -# string", so declaring the tag says this provider uses the standard -# vocabulary with the standard meanings. +# Declared on a run rather than on the source. All nine scenarios pass -- the +# four rule-less rows as STATIC, an unknown flag and a type mismatch as ERROR +# beside their error codes, TARGETING_MATCH for the matched rule and DEFAULT +# for the miss, DISABLED for a disabled flag. The last three need @targeting +# and @disabled-flags as well, which this suite declares, so none of the file +# is skipped here. # -# Declared on a run rather than on the source. All nine scenarios pass -- -# the four rule-less rows as STATIC, an unknown flag and a type mismatch as -# ERROR beside their error codes, TARGETING_MATCH for the matched rule and -# DEFAULT for the miss, DISABLED for a disabled flag. The last three need -# @targeting and @disabled-flags as well, which this suite declares, so none -# of the file is skipped here. -# -# Worth recording that the DISABLED row passes despite the reason arriving as -# flagd's bare string rather than the SDK's Reason enum, noted under -# DISABLED_FLAGS above: the step compares the reason as text, and "DISABLED" -# is "DISABLED" either way. The in-process resolver returns Reason.DISABLED -# and passes identically. +# The DISABLED row passes despite the reason arriving as flagd's bare string +# rather than the SDK's Reason enum: the step compares the reason as text. # # CACHING -# Reserved in the Capability enum; no scenario carries the tag. Declaring a -# capability nothing exercises would be a claim with no evidence behind it, -# so it is left out of both suites. @targeting was reserved alongside it -# until spec@26362f85 gave it three scenarios, and is now declared above. +# Reserved, and the harness refuses it: no scenario carries the tag. RPC_CAPABILITIES = frozenset( { Capability.EVENTS, @@ -307,16 +199,8 @@ ) """The one requirement this resolver is known to fail. -Takes the **declared and failing** shape rather than the withheld-and-skipped -one, which is the shape Appendix F's guidance prefers and this is the case it -prefers it for: flagd does attempt the coercion -- the widening scenario passes --- and gets the narrowing direction wrong, so withdrawing the capability would -turn a real failure into a skip indistinguishable from a provider that declines -to coerce. The failure stays visible in the results and this entry says it is -known and why. - -Recorded against this resolver only. See the in-process suite, which passes the -scenario this deviates on. +Declared-and-failing rather than withheld-and-skipped, and recorded against this +resolver only: the in-process suite passes the scenario this deviates on. """ RPC_SUITE = ResolverSuite( @@ -335,10 +219,8 @@ def tck_config(tck_backend: RunningBackend, closed_port: int) -> TckConfig: """The whole of this adoption's wiring. - ``tck_backend`` is the TCK's own session-scoped fixture: it has already - started the Compose file ``tests/tck/conftest.py`` declares, discovered the - dynamically mapped host ports, built the control against the launchpad and - waited for it to accept commands. + ``tck_backend`` is the TCK's own session-scoped fixture: the stack is up and + its control is ready by the time this runs. """ return build_config(RPC_SUITE, tck_backend, closed_port) From 6498c09ff483a263a0684bb3bca0261d722bcd8c Mon Sep 17 00:00:00 2001 From: Simon Schrottner Date: Mon, 14 Sep 2026 10:03:48 +0200 Subject: [PATCH 23/29] test(tck): run the conformance suite against flagd-testbed v3.10.1 Four languages pinned this image by hand and had drifted: JavaScript was already on v3.10.1 while Go, Java and Python sat on v3.8.0, so the suites whose results are only comparable if they asked the same backend were asking two. v3.10.1 is the current release, so aligning up rather than down. What this does NOT fix, stated because the tag is easy to mistake for a fix: open-feature/flagd-testbed#392 and #394 are both still open, so v3.10.1 carries neither the three missing precision flags nor the /start readiness fix. The fixture failures and the readiness race are unchanged. The one behavioural change in range is open-feature/flagd-testbed#390, which increases the simulated downtime -- and that is exactly the timing the @stale and @unavailable scenarios depend on. Tallies recorded against v3.8.0 have not been re-measured on this image, including the failure ranges the adoption READMEs cite by tag; those figures stay as they are because they are a record of what v3.8.0 did, and re-taking them is follow-up work rather than a rewrite. Signed-off-by: Simon Schrottner --- .../openfeature-provider-flagd/tests/tck/docker-compose.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/providers/openfeature-provider-flagd/tests/tck/docker-compose.yaml b/providers/openfeature-provider-flagd/tests/tck/docker-compose.yaml index 5c27bf27..9ed476d5 100644 --- a/providers/openfeature-provider-flagd/tests/tck/docker-compose.yaml +++ b/providers/openfeature-provider-flagd/tests/tck/docker-compose.yaml @@ -12,7 +12,7 @@ # `diff` the two paths after changing either -- that diff is what catches drift. services: backend: - image: ghcr.io/open-feature/flagd-testbed:v3.8.0 + image: ghcr.io/open-feature/flagd-testbed:v3.10.1 ports: - 8013 # flagd RPC evaluation (gRPC) - 8015 # flagd in-process sync (gRPC) From feae3bc574170ec8a15b995fb1e9c03765543619 Mon Sep 17 00:00:00 2001 From: Simon Schrottner Date: Mon, 14 Sep 2026 17:29:17 +0200 Subject: [PATCH 24/29] test(tck): stop naming one testbed release as the flag set's boundary Several comments said flagd-testbed "v3.8.0" seeds neither large-integer-flag nor integral-float-flag. The Compose file moved to v3.10.1 and those flags are absent from that release too, so naming one version invited the reader to conclude the other serves them and that the failures were a pin away from going green. They are not: open-feature/flagd-testbed#392 is what adds the flags, and it is open. Now phrased as no released testbed seeding them, which stays true until #392 lands and false the moment it does -- which is when these comments should be revisited anyway. Signed-off-by: Simon Schrottner --- providers/openfeature-provider-flagd/tests/tck/conftest.py | 2 +- .../openfeature-provider-flagd/tests/tck/test_in_process.py | 2 +- providers/openfeature-provider-flagd/tests/tck/test_rpc.py | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/providers/openfeature-provider-flagd/tests/tck/conftest.py b/providers/openfeature-provider-flagd/tests/tck/conftest.py index ffa6f550..b577d4c3 100644 --- a/providers/openfeature-provider-flagd/tests/tck/conftest.py +++ b/providers/openfeature-provider-flagd/tests/tck/conftest.py @@ -15,7 +15,7 @@ 3 skipped`` over the two resolvers. Read it here rather than counting: three distinct causes account for all eight, and only two of them are the provider's. -*Six failures are the backend's flag set.* flagd-testbed v3.8.0 seeds neither +*Six failures are the backend's flag set.* no released flagd-testbed seeds ``large-integer-flag`` nor ``integral-float-flag``, so the untagged precision scenario, the ``max-int32`` row of the ``@variants`` outline and the lossless ``@numeric-coercion`` scenario fail ``FLAG_NOT_FOUND`` on each resolver alike. diff --git a/providers/openfeature-provider-flagd/tests/tck/test_in_process.py b/providers/openfeature-provider-flagd/tests/tck/test_in_process.py index 0dca555a..4ae28f42 100644 --- a/providers/openfeature-provider-flagd/tests/tck/test_in_process.py +++ b/providers/openfeature-provider-flagd/tests/tck/test_in_process.py @@ -126,7 +126,7 @@ # # LARGE_INTEGERS # Withheld. Exactly one scenario carries the tag, it asks for -# `huge-integer-flag`, and flagd-testbed v3.8.0 seeds no such flag -- so this +# `huge-integer-flag`, and no released flagd-testbed seeds such a flag -- so this # backend can put none of the tag's scenarios to this provider, and Appendix # F's sixth declaring rule says withhold. Python's `int` is unbounded and the # ruleset arrives as JSON text parsed with `json.loads` (flagd_core.py:73), diff --git a/providers/openfeature-provider-flagd/tests/tck/test_rpc.py b/providers/openfeature-provider-flagd/tests/tck/test_rpc.py index f8e80f8f..7ba8a45f 100644 --- a/providers/openfeature-provider-flagd/tests/tck/test_rpc.py +++ b/providers/openfeature-provider-flagd/tests/tck/test_rpc.py @@ -110,7 +110,7 @@ # # LARGE_INTEGERS # Withheld. Exactly one scenario carries the tag, it asks for -# `huge-integer-flag`, and flagd-testbed v3.8.0 seeds no such flag -- so this +# `huge-integer-flag`, and no released flagd-testbed seeds such a flag -- so this # backend can put none of the tag's scenarios to this provider, and Appendix # F's sixth declaring rule says withhold. Nothing in this path would narrow # the value: flagd holds every numeric variant as a float64 and 2^53 - 1 is From afd09d8ea21a638e9032916806c68ad6ac5dac42 Mon Sep 17 00:00:00 2001 From: Simon Schrottner Date: Tue, 15 Sep 2026 22:40:49 +0200 Subject: [PATCH 25/29] test(flagd): declare @string-typing on both resolvers, measured The base branch's pin moves four scenarios out of the mandatory mismatch matrix and behind the new `@string-typing` tag: `boolean-flag`, `integer-flag` and `float-flag` asked for through the string accessor, plus `object-flag`. Both resolvers declare it. Measured rather than assumed, and the measurement is cheap to state because those four scenarios were mandatory at the previous pin and passing on both resolvers. The full run is 8 failed, 119 passed, 3 skipped before and after, and the four were run again on their own to see them selected rather than gated away: 8 passed over the two resolvers. The two resolvers answer it for different reasons, which is worth keeping since this is the second capability where that is true. In-process checks the type locally -- `_TYPE_MAP`'s string entry is the narrow `(str,)` and `_check_type` raises TypeMismatchError for anything else, so nothing renders a bool, a number or a structure as text on the way out. RPC does no local check at all: it asks the server for a String, and flagd types each variant from the JSON ruleset, so the mismatch is the server's to raise. Both are the property a string-storing backend lacks, which is the whole reason the capability exists, and neither could have been read off this repository alone. Worth noting on the in-process side: this is the one accessor `bool` does not slip through. The int-to-float widening that makes `boolean-flag` requested as a Float resolve to 1.0 -- python-sdk-contrib#417, still the one ungated failure in that suite -- has no counterpart here, because `str` is not in bool's ancestry. No KnownDeviation on either suite, and nothing withheld: all four scenarios pass, so there is neither a gap to acknowledge nor a question the backend cannot be asked. Signed-off-by: Simon Schrottner --- .../tests/tck/test_in_process.py | 20 +++++++++++++++++ .../tests/tck/test_rpc.py | 22 +++++++++++++++++++ 2 files changed, 42 insertions(+) diff --git a/providers/openfeature-provider-flagd/tests/tck/test_in_process.py b/providers/openfeature-provider-flagd/tests/tck/test_in_process.py index 4ae28f42..75ece7c8 100644 --- a/providers/openfeature-provider-flagd/tests/tck/test_in_process.py +++ b/providers/openfeature-provider-flagd/tests/tck/test_in_process.py @@ -114,6 +114,25 @@ # scenario this backend cannot ask would discard the finding that the two # resolvers differ. # +# STRING_TYPING +# Declared, on a run: all four scenarios pass on this resolver -- three +# Examples rows asking for `boolean-flag`, `integer-flag` and `float-flag` +# through the string accessor, and the `@object` one asking for +# `object-flag`. +# +# Here the type check is local and readable, which is the contrast with RPC. +# `_TYPE_MAP`'s string entry is the narrow `(str,)` (flagd_core.py:23) and +# `_check_type` (flagd_core.py:228-231) raises TypeMismatchError for +# anything else, so nothing renders a bool, a number or a structure as text +# on the way out. Note that this is the one accessor `bool` does not slip +# through: the int-to-float widening that makes `boolean-flag` as a Float +# resolve to 1.0 (python-sdk-contrib#417, the ungated failure below) has no +# counterpart here, because `str` is not in bool's ancestry. +# +# Four scenarios that were mandatory at the previous pin, and passing, so +# the declaration keeps them running and changes no number: the full run is +# 8 failed, 119 passed, 3 skipped before and after. +# # REINITIALIZATION # Declared here and withheld on RPC, and again the two resolvers genuinely # differ: this one shuts down and starts again serving `boolean-flag` @@ -170,6 +189,7 @@ Capability.STANDARD_REASONS, Capability.LIFECYCLE, Capability.NUMERIC_COERCION, + Capability.STRING_TYPING, Capability.REINITIALIZATION, } ) diff --git a/providers/openfeature-provider-flagd/tests/tck/test_rpc.py b/providers/openfeature-provider-flagd/tests/tck/test_rpc.py index 7ba8a45f..0ea08b34 100644 --- a/providers/openfeature-provider-flagd/tests/tck/test_rpc.py +++ b/providers/openfeature-provider-flagd/tests/tck/test_rpc.py @@ -106,6 +106,27 @@ # conftest records it and the deviation summary disclaims it; it is not a # reason to withhold the tag. # +# STRING_TYPING +# Declared, on a run: all four scenarios pass on this resolver -- three +# Examples rows asking for `boolean-flag`, `integer-flag` and `float-flag` +# through the string accessor, and the `@object` one asking for +# `object-flag`. +# +# Declared rather than assumed, because the question is about the backend +# rather than about the provider and RPC is where the backend answers it. +# The decision is flagd's for the same reason the coercion one is: grpc.py +# asks the server for a String and hands back what it answers, and flagd +# types each variant from the JSON ruleset, so a bool or a number reached +# through ResolveString is a TYPE_MISMATCH the server raises rather than one +# the provider notices. That is the property a string-storing backend lacks +# -- see the capability's docstring and Appendix F's `@string-typing` +# section -- and it is not a property of flagd that could be read off this +# repository at all. +# +# Four scenarios that were mandatory at the previous pin, and passing, so +# the declaration keeps them running and changes no number: the full run is +# 8 failed, 119 passed, 3 skipped before and after. +# # Not declared, and why: # # LARGE_INTEGERS @@ -166,6 +187,7 @@ Capability.STANDARD_REASONS, Capability.LIFECYCLE, Capability.NUMERIC_COERCION, + Capability.STRING_TYPING, } ) From 9415a3dee22f949cee6712426fff829675768103 Mon Sep 17 00:00:00 2001 From: Simon Schrottner Date: Wed, 16 Sep 2026 09:46:31 +0200 Subject: [PATCH 26/29] test(flagd): declare @fully-typed-values on both resolvers, measured Spec bda599f1 splits @string-typing: the tag now gates the boolean and integer rows a partially typed backend can still answer, and @fully-typed-values gates the float and structured cases. Both resolvers declare both, because flagd is on the typed side of the line the split draws -- it holds a ruleset rather than a string table and types every variant out of the JSON, so a float variant is a float and a structured one is an object. In-process says the same thing from one table: _TYPE_MAP gives float (int, float) and object (dict, list), and the ruleset is json.loads output all the way down. Measured rather than reasoned. All four scenarios run and pass on each resolver -- two Examples rows plus the float and object scenarios -- and the run is unchanged either side of the split: 8 failed, 119 passed, 3 skipped, being 4 failed, 59 passed, 2 skipped on RPC and 4 failed, 60 passed, 1 skipped in-process. Declaring only @string-typing would have skipped two passing scenarios per resolver and published a gap neither resolver has, which is the outcome the split exists to make visible rather than to cause. Signed-off-by: Simon Schrottner --- .../tests/tck/test_in_process.py | 26 +++++++++++------ .../tests/tck/test_rpc.py | 28 +++++++++++++------ 2 files changed, 38 insertions(+), 16 deletions(-) diff --git a/providers/openfeature-provider-flagd/tests/tck/test_in_process.py b/providers/openfeature-provider-flagd/tests/tck/test_in_process.py index 75ece7c8..f4851847 100644 --- a/providers/openfeature-provider-flagd/tests/tck/test_in_process.py +++ b/providers/openfeature-provider-flagd/tests/tck/test_in_process.py @@ -114,11 +114,11 @@ # scenario this backend cannot ask would discard the finding that the two # resolvers differ. # -# STRING_TYPING -# Declared, on a run: all four scenarios pass on this resolver -- three -# Examples rows asking for `boolean-flag`, `integer-flag` and `float-flag` -# through the string accessor, and the `@object` one asking for -# `object-flag`. +# STRING_TYPING and FULLY_TYPED_VALUES +# Both declared, on a run: all four scenarios pass on this resolver -- two +# Examples rows asking for `boolean-flag` and `integer-flag` through the +# string accessor, and the `float-flag` and `object-flag` scenarios that +# spec bda599f1 moved behind the second tag. # # Here the type check is local and readable, which is the contrast with RPC. # `_TYPE_MAP`'s string entry is the narrow `(str,)` (flagd_core.py:23) and @@ -129,9 +129,18 @@ # resolve to 1.0 (python-sdk-contrib#417, the ungated failure below) has no # counterpart here, because `str` is not in bool's ancestry. # -# Four scenarios that were mandatory at the previous pin, and passing, so -# the declaration keeps them running and changes no number: the full run is -# 8 failed, 119 passed, 3 skipped before and after. +# Both tags rather than one, and here the reason is the same table: +# `_TYPE_MAP` (flagd_core.py:22-28) gives `float` `(int, float)` and +# `object` `(dict, list)`, so this resolver holds a float as a number and a +# structure as a dict or list, exactly as it holds a bool as a bool. There +# is no type the ruleset keeps as text -- it is `json.loads` output all the +# way down (flagd_core.py:73) -- so the question @fully-typed-values asks has +# an answer here. The split is for a backend that types booleans and +# integers and nothing else, and this is not one. +# +# Four scenarios that were mandatory before spec d47a66eb, and passing, so +# the two declarations keep them running and change no number: the full run +# is 8 failed, 119 passed, 3 skipped before and after. # # REINITIALIZATION # Declared here and withheld on RPC, and again the two resolvers genuinely @@ -190,6 +199,7 @@ Capability.LIFECYCLE, Capability.NUMERIC_COERCION, Capability.STRING_TYPING, + Capability.FULLY_TYPED_VALUES, Capability.REINITIALIZATION, } ) diff --git a/providers/openfeature-provider-flagd/tests/tck/test_rpc.py b/providers/openfeature-provider-flagd/tests/tck/test_rpc.py index 0ea08b34..676e905a 100644 --- a/providers/openfeature-provider-flagd/tests/tck/test_rpc.py +++ b/providers/openfeature-provider-flagd/tests/tck/test_rpc.py @@ -106,11 +106,11 @@ # conftest records it and the deviation summary disclaims it; it is not a # reason to withhold the tag. # -# STRING_TYPING -# Declared, on a run: all four scenarios pass on this resolver -- three -# Examples rows asking for `boolean-flag`, `integer-flag` and `float-flag` -# through the string accessor, and the `@object` one asking for -# `object-flag`. +# STRING_TYPING and FULLY_TYPED_VALUES +# Both declared, on a run: all four scenarios pass on this resolver -- two +# Examples rows asking for `boolean-flag` and `integer-flag` through the +# string accessor, and the `float-flag` and `object-flag` scenarios that +# spec bda599f1 moved behind the second tag. # # Declared rather than assumed, because the question is about the backend # rather than about the provider and RPC is where the backend answers it. @@ -123,9 +123,20 @@ # section -- and it is not a property of flagd that could be read off this # repository at all. # -# Four scenarios that were mandatory at the previous pin, and passing, so -# the declaration keeps them running and changes no number: the full run is -# 8 failed, 119 passed, 3 skipped before and after. +# Which is the same answer for the second tag, and why both are declared +# rather than one. flagd holds a ruleset, not a string table, and it types +# every variant out of the JSON: a float variant is a float and a structured +# one is an object, so there is no type it keeps as text and every one of the +# four questions has an answer here. The split exists for a backend that +# records booleans and integers natively and nothing else -- such a provider +# declares @string-typing and withholds @fully-typed-values, and the measured +# reason it exists is that one tag over all four cases published a real +# defect in another language as a permitted absence. flagd is on the other +# side of that line, and declaring both is what says so. +# +# Four scenarios that were mandatory before spec d47a66eb, and passing, so +# the two declarations keep them running and change no number: the full run +# is 8 failed, 119 passed, 3 skipped before and after. # # Not declared, and why: # @@ -188,6 +199,7 @@ Capability.LIFECYCLE, Capability.NUMERIC_COERCION, Capability.STRING_TYPING, + Capability.FULLY_TYPED_VALUES, } ) From 90fbb0f1357cb939e0a71308eb14ad35edd4fc9e Mon Sep 17 00:00:00 2001 From: Simon Schrottner Date: Wed, 16 Sep 2026 09:46:44 +0200 Subject: [PATCH 27/29] docs(flagd): the standard-reasons note sits with the declarations it describes Both suites declare STANDARD_REASONS and both explained it under the "Not declared, and why" header, where a reader scanning for what the suite withholds finds a capability it claims -- and the note's own first line says "declared on a run", so the header and the text contradict each other. The withholdings around it are LARGE_INTEGERS, REINITIALIZATION on RPC, and the reserved CACHING; this is not one of them. Moved up into the declared block, unchanged otherwise. Signed-off-by: Simon Schrottner --- .../tests/tck/test_in_process.py | 24 +++++++++---------- .../tests/tck/test_rpc.py | 22 ++++++++--------- 2 files changed, 23 insertions(+), 23 deletions(-) diff --git a/providers/openfeature-provider-flagd/tests/tck/test_in_process.py b/providers/openfeature-provider-flagd/tests/tck/test_in_process.py index f4851847..43360bff 100644 --- a/providers/openfeature-provider-flagd/tests/tck/test_in_process.py +++ b/providers/openfeature-provider-flagd/tests/tck/test_in_process.py @@ -150,6 +150,18 @@ # is why RPC's withholding needs no KnownDeviation and why declaring it here # is a claim worth making rather than a box ticked. # +# STANDARD_REASONS +# Declared on a run rather than on the source. All nine scenarios pass -- the +# four rule-less rows as STATIC, an unknown flag and a type mismatch as ERROR +# beside their error codes, TARGETING_MATCH for the matched rule and DEFAULT +# for the miss, DISABLED for a disabled flag. The last three need @targeting +# and @disabled-flags as well, which this suite declares, so none of the file +# is skipped here. +# +# The DISABLED row resolves through Reason.DISABLED here, the SDK's own enum, +# where RPC hands back flagd's bare 'DISABLED' string -- noted under +# DISABLED_FLAGS above. The step compares the reason as text, so both pass. +# # Not declared, and why: # # LARGE_INTEGERS @@ -171,18 +183,6 @@ # declare the tag when the image carries it, or it outlives its reason and # starts reading as a claim about the provider. # -# STANDARD_REASONS -# Declared on a run rather than on the source. All nine scenarios pass -- the -# four rule-less rows as STATIC, an unknown flag and a type mismatch as ERROR -# beside their error codes, TARGETING_MATCH for the matched rule and DEFAULT -# for the miss, DISABLED for a disabled flag. The last three need @targeting -# and @disabled-flags as well, which this suite declares, so none of the file -# is skipped here. -# -# The DISABLED row resolves through Reason.DISABLED here, the SDK's own enum, -# where RPC hands back flagd's bare 'DISABLED' string -- noted under -# DISABLED_FLAGS above. The step compares the reason as text, so both pass. -# # CACHING # Reserved, and the harness refuses it: no scenario carries the tag. IN_PROCESS_CAPABILITIES = frozenset( diff --git a/providers/openfeature-provider-flagd/tests/tck/test_rpc.py b/providers/openfeature-provider-flagd/tests/tck/test_rpc.py index 676e905a..49f4903f 100644 --- a/providers/openfeature-provider-flagd/tests/tck/test_rpc.py +++ b/providers/openfeature-provider-flagd/tests/tck/test_rpc.py @@ -138,6 +138,17 @@ # the two declarations keep them running and change no number: the full run # is 8 failed, 119 passed, 3 skipped before and after. # +# STANDARD_REASONS +# Declared on a run rather than on the source. All nine scenarios pass -- the +# four rule-less rows as STATIC, an unknown flag and a type mismatch as ERROR +# beside their error codes, TARGETING_MATCH for the matched rule and DEFAULT +# for the miss, DISABLED for a disabled flag. The last three need @targeting +# and @disabled-flags as well, which this suite declares, so none of the file +# is skipped here. +# +# The DISABLED row passes despite the reason arriving as flagd's bare string +# rather than the SDK's Reason enum: the step compares the reason as text. +# # Not declared, and why: # # LARGE_INTEGERS @@ -172,17 +183,6 @@ # from: no KnownDeviation. The in-process suite declares it for the same # reason in reverse -- it runs, and it passes. # -# STANDARD_REASONS -# Declared on a run rather than on the source. All nine scenarios pass -- the -# four rule-less rows as STATIC, an unknown flag and a type mismatch as ERROR -# beside their error codes, TARGETING_MATCH for the matched rule and DEFAULT -# for the miss, DISABLED for a disabled flag. The last three need @targeting -# and @disabled-flags as well, which this suite declares, so none of the file -# is skipped here. -# -# The DISABLED row passes despite the reason arriving as flagd's bare string -# rather than the SDK's Reason enum: the step compares the reason as text. -# # CACHING # Reserved, and the harness refuses it: no scenario carries the tag. RPC_CAPABILITIES = frozenset( From c590b4023ab3237ae4605c1f510125fdb1086d64 Mon Sep 17 00:00:00 2001 From: Simon Schrottner Date: Wed, 16 Sep 2026 09:46:56 +0200 Subject: [PATCH 28/29] test(flagd): sync the conformance assets where the scenarios execute Appendix F now requires the revision check to be in force where the scenarios run, not only in the TCK implementation's own tests, and `test-tck` was bare pytest while the TCK package's own `test` syncs first. That gap is not theoretical here: this is a uv workspace, so openfeature-tck is installed editable and the scenarios this suite collects are the gitignored copies under tools/openfeature-tck/src. A rebase moves the submodule pin and not the submodule's working tree, so a run from this package could ask the previous revision's questions and report plausible numbers, which is exactly how an adoption in another language published a stale pass. `hatch_build_sync.py` checks the submodule out at the pin, refuses to copy when it cannot, and imports nothing but the standard library, so the task is one line and adds no dependency. Not wired onto `test-tck-collect`, deliberately. That runs on every default build, executes no scenario and publishes no result, so the rule does not reach it and the default build stays free of a submodule operation. Signed-off-by: Simon Schrottner --- .../openfeature-provider-flagd/pyproject.toml | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) diff --git a/providers/openfeature-provider-flagd/pyproject.toml b/providers/openfeature-provider-flagd/pyproject.toml index 1c0a338b..46ba6681 100644 --- a/providers/openfeature-provider-flagd/pyproject.toml +++ b/providers/openfeature-provider-flagd/pyproject.toml @@ -152,7 +152,23 @@ test = { sequence = ["test-default", "test-tck-collect"], ignore_fail = "return_ test-cov = { sequence = ["test-cov-default", "test-tck-collect"], ignore_fail = "return_non_zero" } test-default = "pytest tests --ignore=tests/tck" test-cov-default = "coverage run -m pytest tests --ignore=tests/tck" -test-tck = "pytest tests/tck" +# The sync is part of the run rather than something a maintainer remembers, +# because Appendix F requires the revision check to be in force **where the +# scenarios execute** and not only in the TCK implementation's own tests. This +# is a uv workspace, so `openfeature-tck` is installed editable and the +# canonical assets this suite collects are the gitignored copies under +# `tools/openfeature-tck/src`: a rebase that moves the submodule pin leaves them +# from the previous revision, and a suite run from here would neither notice nor +# say so. That is exactly how an adoption in another language published a full +# pass against stale scenarios. `hatch_build_sync.py` checks the submodule out +# at the pin, refuses to copy if it cannot, and needs nothing but the standard +# library. +# +# Not on `test-tck-collect`, deliberately: that runs on every default build, it +# executes no scenario, and it publishes no result, so the rule does not reach +# it and the default build stays free of a submodule operation. +sync-spec-assets = "python ../../tools/openfeature-tck/hatch_build_sync.py" +test-tck = ["sync-spec-assets", {cmd = "pytest tests/tck"}] test-tck-collect = "pytest tests/tck --collect-only -q" cov-report = "coverage xml" cov = [ From 83192d33f78132060e9095e64821feb88ea938cf Mon Sep 17 00:00:00 2001 From: Simon Schrottner Date: Wed, 16 Sep 2026 12:36:56 +0200 Subject: [PATCH 29/29] docs(flagd): stop naming a testbed release in the README tally The commit that stopped naming one release as the flag set's boundary changed the conftest prose and missed this line, because the check afterwards looked only at .py files. The flags are absent from every released flagd-testbed, not from one. Signed-off-by: Simon Schrottner --- providers/openfeature-provider-flagd/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/providers/openfeature-provider-flagd/README.md b/providers/openfeature-provider-flagd/README.md index 02c5c78a..3d382009 100644 --- a/providers/openfeature-provider-flagd/README.md +++ b/providers/openfeature-provider-flagd/README.md @@ -209,7 +209,7 @@ Two things that are this provider's rather than the policy's: - **Docker is not what decides it.** `tests/e2e` needs Docker too and does run in the default build. What decides it is the run: **8 failed, 119 passed, 3 skipped**, being three canonical flags that - flagd-testbed v3.8.0 does not seed yet, failing on each resolver, plus two genuine provider-side + no released flagd-testbed seeds yet, failing on each resolver, plus two genuine provider-side gaps — one per resolver, and only one of them carries a known-deviation entry. `tests/tck/conftest.py` accounts for all eight individually, with the flag or requirement each one turns on, so a reviewer running the suite can tell a new failure from a known one and the number