Guard eager OpenFeature init and add detailed /ffe/evaluate for Go agentless configuration contract

manifests/golang.yml

@@ -1327,7 +1327,7 @@ manifest:
   tests/debugger/test_debugger_symdb.py::Test_Debugger_SymDb::test_event_metadata: missing_feature (extended event schema not yet shipped)
   tests/debugger/test_debugger_telemetry.py::Test_Debugger_Telemetry: missing_feature
   tests/docker_ssi/test_docker_ssi_appsec.py::TestDockerSSIAppsecFeatures::test_telemetry_source_ssi: v2.0.0
-  tests/ffe/test_agentless_configuration.py: missing_feature (FFL-2695 tracks Go agentless configuration-source implementation; FFL-2731 tracks the system-tests contract)
+  tests/ffe/test_agentless_configuration.py: v2.11.0-dev
   tests/ffe/test_dynamic_evaluation.py::Test_FFE_Flag_Parse_Error_Isolation::test_valid_flag_unaffected:  # TODO: a lower version might be supported
     - declaration: bug (FFL-2184)
       component_version: <2.10.0
@@ -1597,28 +1597,7 @@ manifest:
   tests/parametric/test_extract_behavior.py::Test_ExtractBehavior_Ignore: incomplete_test_app (The parametric test app does not apply DD_TRACE_PROPAGATION_BEHAVIOR_EXTRACT=ignore)
   tests/parametric/test_extract_behavior.py::Test_ExtractBehavior_Restart: incomplete_test_app (The parametric test app does not emit restart span links or preserve baggage)
   tests/parametric/test_extract_behavior.py::Test_ExtractBehavior_Restart_With_Extract_First: missing_feature (DD_TRACE_PROPAGATION_EXTRACT_FIRST=true is unsupported; dd-trace-go panics when extraction fails)
-  tests/parametric/test_ffe/test_configuration_sources.py:  # TODO: a lower version might be supported
-    - declaration: missing_feature (FFL-2695 tracks Go agentless configuration-source implementation; FFL-2731 tracks system-tests configuration-source contract)
-      component_version: <2.10.0
-  tests/parametric/test_ffe/test_configuration_sources.py::Test_Feature_Flag_Configuration_Source_Cold_Failure_And_Recovery: missing_feature (FFL-2695 tracks Go agentless configuration-source implementation; FFL-2731 tracks system-tests configuration-source contract)
-  tests/parametric/test_ffe/test_configuration_sources.py::Test_Feature_Flag_Configuration_Source_Poller_Concurrency: missing_feature (FFL-2695 tracks Go agentless configuration-source implementation; FFL-2731 tracks system-tests configuration-source contract)
-  ? tests/parametric/test_ffe/test_configuration_sources.py::Test_Feature_Flag_Configuration_Source_Selection::test_default_agentless_positive
-  : missing_feature (FFL-2695 tracks Go agentless configuration-source implementation; FFL-2731 tracks system-tests configuration-source contract)
-  ? tests/parametric/test_ffe/test_configuration_sources.py::Test_Feature_Flag_Configuration_Source_Selection::test_explicit_agentless_wins_over_legacy_true
-  : missing_feature (FFL-2695 tracks Go agentless configuration-source implementation; FFL-2731 tracks system-tests configuration-source contract)
-  ? tests/parametric/test_ffe/test_configuration_sources.py::Test_Feature_Flag_Configuration_Source_Selection::test_explicit_remote_config_wins_over_legacy_false
-  : missing_feature (FFL-2695 tracks Go agentless configuration-source implementation; FFL-2731 tracks system-tests configuration-source contract)
-  ? tests/parametric/test_ffe/test_configuration_sources.py::Test_Feature_Flag_Configuration_Source_Selection::test_legacy_true_preserves_remote_config
-  : missing_feature (FFL-2695 tracks Go agentless configuration-source implementation; FFL-2731 tracks system-tests configuration-source contract)
-  ? tests/parametric/test_ffe/test_configuration_sources.py::Test_Feature_Flag_Configuration_Source_Selection::test_provider_kill_switch_overrides_legacy_true
-  : missing_feature (FFL-2695 tracks Go agentless configuration-source implementation; FFL-2731 tracks system-tests configuration-source contract)
-  ? tests/parametric/test_ffe/test_configuration_sources.py::Test_Feature_Flag_Configuration_Source_Selection::test_remote_config_positive_ignores_agentless_env
-  : missing_feature (FFL-2695 tracks Go agentless configuration-source implementation; FFL-2731 tracks system-tests configuration-source contract)
-  ? tests/parametric/test_ffe/test_configuration_sources.py::Test_Feature_Flag_Configuration_Source_Selection::test_remote_config_without_rc_does_not_fallback_to_agentless
-  : missing_feature (FFL-2695 tracks Go agentless configuration-source implementation; FFL-2731 tracks system-tests configuration-source contract)
-  ? tests/parametric/test_ffe/test_configuration_sources.py::Test_Feature_Flag_Configuration_Source_Selection::test_stable_true_preserves_default_agentless
-  : missing_feature (FFL-2695 tracks Go agentless configuration-source implementation; FFL-2731 tracks system-tests configuration-source contract)
-  tests/parametric/test_ffe/test_configuration_sources.py::Test_Feature_Flag_Configuration_Source_Warm_State_Preservation: missing_feature (FFL-2695 tracks Go agentless configuration-source implementation; FFL-2731 tracks system-tests configuration-source contract)
+  tests/parametric/test_ffe/test_configuration_sources.py: v2.11.0-dev
   tests/parametric/test_ffe/test_dynamic_evaluation.py::Test_Feature_Flag_Dynamic_Evaluation: v2.5.0-dev
   tests/parametric/test_ffe/test_span_enrichment.py: '>=2.10.0'  # TODO: a lower version might be supported
   tests/parametric/test_ffe/test_span_enrichment.py::Test_Span_Enrichment_Child_Span_Propagation: missing_feature

tests/parametric/test_ffe/test_configuration_sources.py

@@ -151,8 +151,11 @@ def _assert_no_mock_requests(mock_ffe_agentless_backend: MockFFEAgentlessBackend
 
 
 def _remote_config_products(test_agent: TestAgentAPI) -> set[str]:
+    # Not post_only: golang polls /v0.7/config with GET, every other library with
+    # POST. The verb says nothing about which products the body advertises, and
+    # the capability assertion next to this one is already unfiltered.
     products: set[str] = set()
-    for request in test_agent.rc_requests(post_only=True):
+    for request in test_agent.rc_requests():
         client = request["body"].get("client", {})
         products.update(client.get("products", []))
     return products

utils/build/docker/golang/parametric/ffe.go

@@ -1,14 +1,76 @@
 package main
 
 import (
+	"context"
 	"encoding/json"
+	"errors"
 	"net/http"
+	"sync"
+	"time"
 
+	ddof "github.com/DataDog/dd-trace-go/v2/openfeature"
 	of "github.com/open-feature/go-sdk/openfeature"
 )
 
-func (s *apmClientServer) ffeStart(http.ResponseWriter, *http.Request) {
-	return
+// ffeStartTimeout bounds the wait for the first configuration. Above the 10s
+// DD_EXPERIMENTAL_FLAGGING_PROVIDER_INITIALIZATION_TIMEOUT_MS default, so the
+// tracer's own timeout governs whenever it applies one.
+const ffeStartTimeout = 15 * time.Second
+
+// ffeStartMu guards ffeStarted, which records a *successful* start. sync.Once
+// would burn its one shot on a failed attempt and then report 200 with no
+// client, so a retry could never recover.
+var (
+	ffeStartMu sync.Mutex
+	ffeStarted bool
+)
+
+func (s *apmClientServer) ffeStart(writer http.ResponseWriter, request *http.Request) {
+	startErr := func() error {
+		ffeStartMu.Lock()
+		defer ffeStartMu.Unlock()
+		if ffeStarted {
+			return nil
+		}
+
+		provider, err := ddof.NewDatadogProvider(ddof.ProviderConfig{})
+		if err != nil {
+			return err
+		}
+
+		// Wait for Init: plain SetProvider returns before it, so /ffe/start would
+		// answer 200 with no configuration and the next evaluation gets the
+		// default. Other SDKs block on initialize inside set_provider.
+		//
+		// The deadline is ours rather than SetProviderAndWait's background
+		// context, so tracers that only bound Init still cannot hang the suite.
+		//
+		// PROVIDER_NOT_READY is not a start failure: the provider is registered
+		// and evaluations return defaults until configuration arrives.
+		ctx, cancel := context.WithTimeout(context.Background(), ffeStartTimeout)
+		defer cancel()
+
+		if err := of.SetProviderWithContextAndWait(ctx, provider); err != nil {
+			var initErr *of.ProviderInitError
+			if !errors.As(err, &initErr) || initErr.ErrorCode != of.ProviderNotReadyCode {
+				return err
+			}
+		}
+
+		s.ddProvider = provider
+		s.ofClient = of.NewClient("system-tests-weblog-client")
+		ffeStarted = true
+		return nil
+	}()
+
+	if startErr != nil {
+		writer.WriteHeader(http.StatusInternalServerError)
+		_ = json.NewEncoder(writer).Encode(map[string]string{"error": startErr.Error()})
+		return
+	}
+
+	writer.WriteHeader(http.StatusOK)
+	_ = json.NewEncoder(writer).Encode(map[string]any{})
 }
 
 func (s *apmClientServer) ffeEval(writer http.ResponseWriter, request *http.Request) {
@@ -24,21 +86,109 @@ func (s *apmClientServer) ffeEval(writer http.ResponseWriter, request *http.Requ
 		return
 	}
 
-	ctx := of.NewEvaluationContext(body.TargetingKey, body.Attributes)
+	if s.ofClient == nil {
+		writer.WriteHeader(http.StatusInternalServerError)
+		_ = json.NewEncoder(writer).Encode(map[string]string{"error": "FFE provider not initialized"})
+		return
+	}
 
-	if initer, ok := s.ddProvider.(of.StateHandler); ok {
-		initer.Init(ctx)
+	switch body.VariationType {
+	case "BOOLEAN", "STRING", "INTEGER", "NUMERIC", "JSON":
+	default:
+		http.Error(writer, "unknown variation type: "+body.VariationType, http.StatusBadRequest)
+		return
 	}
 
-	val := s.ofClient.Object(request.Context(), body.Flag, body.DefaultValue, ctx)
+	ctx := of.NewEvaluationContext(body.TargetingKey, body.Attributes)
+
+	value := body.DefaultValue
+	reason := string(of.DefaultReason)
+	var errorCode string
 
-	writer.WriteHeader(http.StatusOK)
+	evalCtx := request.Context()
+
+	func() {
+		defer func() {
+			if r := recover(); r != nil {
+				value = body.DefaultValue
+				reason = "ERROR"
+			}
+		}()
+
+		// The SDK returns evaluation details alongside an error, and the retry
+		// helper keys on errorCode PROVIDER_NOT_READY, so the details have to
+		// survive the error path rather than collapsing to a bare "ERROR".
+		var (
+			details of.EvaluationDetails
+			err     error
+		)
+
+		switch body.VariationType {
+		case "BOOLEAN":
+			defaultValue, _ := body.DefaultValue.(bool)
+			d, e := s.ofClient.BooleanValueDetails(evalCtx, body.Flag, defaultValue, ctx)
+			value, details, err = d.Value, d.EvaluationDetails, e
+		case "STRING":
+			defaultValue, _ := body.DefaultValue.(string)
+			d, e := s.ofClient.StringValueDetails(evalCtx, body.Flag, defaultValue, ctx)
+			value, details, err = d.Value, d.EvaluationDetails, e
+		case "INTEGER":
+			defaultValue, _ := toInt64(body.DefaultValue)
+			d, e := s.ofClient.IntValueDetails(evalCtx, body.Flag, defaultValue, ctx)
+			value, details, err = d.Value, d.EvaluationDetails, e
+		case "NUMERIC":
+			defaultValue, _ := toFloat64(body.DefaultValue)
+			d, e := s.ofClient.FloatValueDetails(evalCtx, body.Flag, defaultValue, ctx)
+			value, details, err = d.Value, d.EvaluationDetails, e
+		case "JSON":
+			d, e := s.ofClient.ObjectValueDetails(evalCtx, body.Flag, body.DefaultValue, ctx)
+			value, details, err = d.Value, d.EvaluationDetails, e
+		}
+
+		reason = string(details.Reason)
+		errorCode = string(details.ErrorCode)
+		if err != nil {
+			value = body.DefaultValue
+			if reason == "" {
+				reason = "ERROR"
+			}
+		}
+	}()
 
+	writer.WriteHeader(http.StatusOK)
 	response := struct {
-		Value any `json:"value"`
-	}{val}
+		Value     any    `json:"value"`
+		Reason    string `json:"reason"`
+		ErrorCode string `json:"errorCode"`
+	}{value, reason, errorCode}
 
 	if err := json.NewEncoder(writer).Encode(response); err != nil {
 		http.Error(writer, "failed to encode response: "+err.Error(), http.StatusInternalServerError)
 	}
 }
+
+func toInt64(v any) (int64, bool) {
+	switch n := v.(type) {
+	case int64:
+		return n, true
+	case int:
+		return int64(n), true
+	case float64:
+		return int64(n), true
+	default:
+		return 0, false
+	}
+}
+
+func toFloat64(v any) (float64, bool) {
+	switch n := v.(type) {
+	case float64:
+		return n, true
+	case int:
+		return float64(n), true
+	case int64:
+		return float64(n), true
+	default:
+		return 0, false
+	}
+}

utils/build/docker/golang/parametric/main.go

@@ -58,19 +58,48 @@ func newServer() *apmClientServer {
 		instruments:  make(map[string]interface{}),
 	}
 
-	s.ddProvider, err = ddof.NewDatadogProvider(ddof.ProviderConfig{})
-	if err != nil {
-		log.Fatalf("failed to create Datadog OpenFeature provider: %v", err)
-	}
+	// The configuration-source contract requires lazy activation: no configuration
+	// delivery may happen before the provider is accessed through /ffe/start. When any
+	// Feature Flagging configuration variable is set, skip this eager initialization and
+	// leave provider setup to /ffe/start. Tests that predate that contract keep the
+	// original eager behavior.
+	if !ffeConfigurationEnvVarsSet() {
+		s.ddProvider, err = ddof.NewDatadogProvider(ddof.ProviderConfig{})
+		if err != nil {
+			log.Fatalf("failed to create Datadog OpenFeature provider: %v", err)
+		}
 
-	if err := of.SetProvider(s.ddProvider); err != nil {
-		log.Fatalf("failed to set Datadog OpenFeature provider and wait for initialization: %v", err)
+		// Async on purpose, unlike /ffe/start: this runs for every parametric
+		// test that sets no Feature Flagging variable, and waiting would add the
+		// 10s DD_EXPERIMENTAL_FLAGGING_PROVIDER_INITIALIZATION_TIMEOUT_MS to each
+		// container start.
+		if err := of.SetProvider(s.ddProvider); err != nil {
+			log.Fatalf("failed to set Datadog OpenFeature provider: %v", err)
+		}
+
+		s.ofClient = of.NewClient("system-tests-weblog-client")
 	}
 
-	s.ofClient = of.NewClient("system-tests-weblog-client")
 	return s
 }
 
+var ffeConfigurationEnvVars = []string{
+	"DD_FEATURE_FLAGS_ENABLED",
+	"DD_FEATURE_FLAGS_CONFIGURATION_SOURCE",
+	"DD_FEATURE_FLAGS_CONFIGURATION_SOURCE_AGENTLESS_BASE_URL",
+	"DD_FEATURE_FLAGS_CONFIGURATION_SOURCE_AGENTLESS_POLL_INTERVAL_SECONDS",
+	"DD_FEATURE_FLAGS_CONFIGURATION_SOURCE_AGENTLESS_REQUEST_TIMEOUT_SECONDS",
+}
+
+func ffeConfigurationEnvVarsSet() bool {
+	for _, name := range ffeConfigurationEnvVars {
+		if _, ok := os.LookupEnv(name); ok {
+			return true
+		}
+	}
+	return false
+}
+
 func main() {
 	flag.String("Darg1", "", "Argument 1")
 	flag.Parse()