Enhanced feature flags

azureappconfiguration/app_configuration_client.go

@@ -0,0 +1,75 @@
+// Copyright (c) Microsoft Corporation.
+// Licensed under the MIT License.
+
+package azureappconfiguration
+
+import (
+	"context"
+
+	"github.com/Azure/azure-sdk-for-go/sdk/azcore"
+	"github.com/Azure/azure-sdk-for-go/sdk/azcore/runtime"
+	"github.com/Azure/azure-sdk-for-go/sdk/data/azappconfig/v2"
+)
+
+// appConfigClient abstracts the Azure App Configuration operations used by the provider.
+type appConfigClient interface {
+	// Key-value operations.
+	NewListSettingsPager(selector azappconfig.SettingSelector, options *azappconfig.ListSettingsOptions) *runtime.Pager[azappconfig.ListSettingsPageResponse]
+	GetSetting(ctx context.Context, key string, options *azappconfig.GetSettingOptions) (azappconfig.GetSettingResponse, error)
+	GetSnapshot(ctx context.Context, snapshotName string, options *azappconfig.GetSnapshotOptions) (azappconfig.GetSnapshotResponse, error)
+	NewListSettingsForSnapshotPager(snapshotName string, options *azappconfig.ListSettingsForSnapshotOptions) *runtime.Pager[azappconfig.ListSettingsForSnapshotResponse]
+
+	// Feature flag operations.
+	NewListFeatureFlagsPager(selector azappconfig.FeatureFlagSelector, options *azappconfig.ListFeatureFlagsOptions) *runtime.Pager[azappconfig.ListFeatureFlagsPageResponse]
+}
+
+// appConfigurationClient is the default appConfigClient implementation.
+// The feature flag client is derived from the configuration client so both share the same pipeline and sync-token cache.
+type appConfigurationClient struct {
+	configurationClient *azappconfig.Client
+	featureFlagClient   *azappconfig.FeatureFlagClient
+}
+
+func newAppConfigurationClient(endpoint string, credential azcore.TokenCredential, options *azappconfig.ClientOptions) (appConfigClient, error) {
+	configurationClient, err := azappconfig.NewClient(endpoint, credential, options)
+	if err != nil {
+		return nil, err
+	}
+
+	return &appConfigurationClient{
+		configurationClient: configurationClient,
+		featureFlagClient:   configurationClient.NewFeatureFlagClient(),
+	}, nil
+}
+
+func newAppConfigurationClientFromConnectionString(connectionString string, options *azappconfig.ClientOptions) (appConfigClient, error) {
+	configurationClient, err := azappconfig.NewClientFromConnectionString(connectionString, options)
+	if err != nil {
+		return nil, err
+	}
+
+	return &appConfigurationClient{
+		configurationClient: configurationClient,
+		featureFlagClient:   configurationClient.NewFeatureFlagClient(),
+	}, nil
+}
+
+func (c *appConfigurationClient) NewListSettingsPager(selector azappconfig.SettingSelector, options *azappconfig.ListSettingsOptions) *runtime.Pager[azappconfig.ListSettingsPageResponse] {
+	return c.configurationClient.NewListSettingsPager(selector, options)
+}
+
+func (c *appConfigurationClient) GetSetting(ctx context.Context, key string, options *azappconfig.GetSettingOptions) (azappconfig.GetSettingResponse, error) {
+	return c.configurationClient.GetSetting(ctx, key, options)
+}
+
+func (c *appConfigurationClient) GetSnapshot(ctx context.Context, snapshotName string, options *azappconfig.GetSnapshotOptions) (azappconfig.GetSnapshotResponse, error) {
+	return c.configurationClient.GetSnapshot(ctx, snapshotName, options)
+}
+
+func (c *appConfigurationClient) NewListSettingsForSnapshotPager(snapshotName string, options *azappconfig.ListSettingsForSnapshotOptions) *runtime.Pager[azappconfig.ListSettingsForSnapshotResponse] {
+	return c.configurationClient.NewListSettingsForSnapshotPager(snapshotName, options)
+}
+
+func (c *appConfigurationClient) NewListFeatureFlagsPager(selector azappconfig.FeatureFlagSelector, options *azappconfig.ListFeatureFlagsOptions) *runtime.Pager[azappconfig.ListFeatureFlagsPageResponse] {
+	return c.featureFlagClient.NewListFeatureFlagsPager(selector, options)
+}

azureappconfiguration/azureappconfiguration.go

@@ -48,6 +48,7 @@ type AzureAppConfiguration struct {
 	kvSelectors          []Selector
 	ffEnabled            bool
 	ffSelectors          []Selector
+	enhancedFFSelectors  []Selector
 	trimPrefixes         []string
 	watchedSettings      []WatchedSetting
 	loadBalancingEnabled bool
@@ -57,6 +58,7 @@ type AzureAppConfiguration struct {
 	watchAll               bool
 	kvETags                map[comparableSelector][]*azcore.ETag
 	ffETags                map[comparableSelector][]*azcore.ETag
+	enhancedFFETags        map[comparableSelector][]*azcore.ETag
 	keyVaultRefs           map[string]string // unversioned Key Vault references
 	kvRefreshTimer         refresh.Condition
 	secretRefreshTimer     refresh.Condition
@@ -135,9 +137,11 @@ func Load(ctx context.Context, authentication AuthenticationOptions, options *Op
 
 	if azappcfg.ffEnabled {
 		azappcfg.ffSelectors = getFeatureFlagSelectors(deduplicateSelectors(options.FeatureFlagOptions.Selectors))
+		azappcfg.enhancedFFSelectors = deduplicateSelectors(options.FeatureFlagOptions.Selectors)
 		if options.FeatureFlagOptions.RefreshOptions.Enabled {
 			azappcfg.ffRefreshTimer = refresh.NewTimer(options.FeatureFlagOptions.RefreshOptions.Interval)
 			azappcfg.ffETags = make(map[comparableSelector][]*azcore.ETag)
+			azappcfg.enhancedFFETags = make(map[comparableSelector][]*azcore.ETag)
 		}
 	}
 
@@ -246,7 +250,7 @@ func (azappcfg *AzureAppConfiguration) Refresh(ctx context.Context) error {
 
 	var keyValueRefreshed, featureFlagRefreshed bool
 	var err error
-	refreshTask := func(client *azappconfig.Client) error {
+	refreshTask := func(client appConfigClient) error {
 		eg, egCtx := errgroup.WithContext(ctx)
 		eg.Go(func() error {
 			if keyValueRefreshed, err = azappcfg.refreshKeyValues(egCtx, azappcfg.newKeyValueRefreshClient(client)); err != nil {
@@ -309,7 +313,7 @@ func (azappcfg *AzureAppConfiguration) OnRefreshSuccess(callback func()) {
 }
 
 func (azappcfg *AzureAppConfiguration) load(ctx context.Context) error {
-	loadTask := func(client *azappconfig.Client) error {
+	loadTask := func(client appConfigClient) error {
 		eg, egCtx := errgroup.WithContext(ctx)
 		eg.Go(func() error {
 			keyValuesClient := &selectorSettingsClient{
@@ -338,7 +342,12 @@ func (azappcfg *AzureAppConfiguration) load(ctx context.Context) error {
 					client:         client,
 					tracingOptions: azappcfg.tracingOptions,
 				}
-				return azappcfg.loadFeatureFlags(egCtx, ffClient)
+				enhancedFFClient := &enhFFSettingsClient{
+					selectors:      azappcfg.enhancedFFSelectors,
+					client:         client,
+					tracingOptions: azappcfg.tracingOptions,
+				}
+				return azappcfg.loadFeatureFlags(egCtx, ffClient, enhancedFFClient)
 			})
 		}
 
@@ -564,34 +573,19 @@ func (azappcfg *AzureAppConfiguration) loadKeyVaultSecrets(ctx context.Context,
 	return secrets, nil
 }
 
-func (azappcfg *AzureAppConfiguration) loadFeatureFlags(ctx context.Context, settingsClient settingsClient) error {
-	settingsResponse, err := settingsClient.getSettings(ctx)
+func (azappcfg *AzureAppConfiguration) loadFeatureFlags(ctx context.Context, kvClient settingsClient, ffClient settingsClient) error {
+	ffRsp, err := kvClient.getSettings(ctx)
 	if err != nil {
 		return err
 	}
 
-	dedupFeatureFlags := make(map[string]any, len(settingsResponse.settings))
-	for _, setting := range settingsResponse.settings {
-		// Skip non-feature flag settings
-		if setting.ContentType == nil || *setting.ContentType != featureFlagContentType {
-			continue
-		}
-
-		if setting.Key != nil {
-			var v map[string]any
-			if err := json.Unmarshal([]byte(*setting.Value), &v); err != nil {
-				log.Printf("Invalid feature flag setting: key=%s, error=%s, just ignore", *setting.Key, err.Error())
-				continue
-			}
-			azappcfg.updateFeatureFlagTracing(v)
-			dedupFeatureFlags[*setting.Key] = v
-		}
+	enhFFRsp, err := ffClient.getSettings(ctx)
+	if err != nil {
+		return err
 	}
 
-	featureFlags := make([]any, 0, len(dedupFeatureFlags))
-	for _, v := range dedupFeatureFlags {
-		featureFlags = append(featureFlags, v)
-	}
+	azappcfg.tracingOptions.UseEnhancedFeatureFlag = len(enhFFRsp.featureFlags) > 0
+	featureFlags := azappcfg.processFeatureFlags(ffRsp.settings, enhFFRsp.featureFlags)
 
 	// "feature_management": {"feature_flags": [{...}, {...}]}
 	ffSettings := map[string]any{
@@ -600,7 +594,8 @@ func (azappcfg *AzureAppConfiguration) loadFeatureFlags(ctx context.Context, set
 		},
 	}
 
-	azappcfg.ffETags = settingsResponse.pageETags
+	azappcfg.ffETags = ffRsp.pageETags
+	azappcfg.enhancedFFETags = enhFFRsp.pageETags
 	azappcfg.featureFlags = ffSettings
 
 	return nil
@@ -696,27 +691,28 @@ func (azappcfg *AzureAppConfiguration) refreshFeatureFlags(ctx context.Context,
 		return false, nil
 	}
 
-	// Check if any ETags have changed
-	eTagChanged, err := refreshClient.monitor.checkIfETagChanged(ctx)
+	ffChanged, err := refreshClient.monitor.checkIfETagChanged(ctx)
 	if err != nil {
 		log.Printf("Failed to check if feature flag settings have changed: %s", err.Error())
 		return false, err
 	}
 
-	if !eTagChanged {
+	if !ffChanged {
+		enhFFChanged, err := refreshClient.enhFFMonitor.checkIfETagChanged(ctx)
+		if err != nil {
+			log.Printf("Failed to check if enhanced feature flags have changed: %s", err.Error())
+			return false, err
+		}
+		ffChanged = enhFFChanged
+	}
+
+	if !ffChanged {
 		// No changes detected, reset timer and return
 		azappcfg.ffRefreshTimer.Reset()
 		return false, nil
 	}
 
-	// Reload feature flags
-	eg, egCtx := errgroup.WithContext(ctx)
-	eg.Go(func() error {
-		settingsClient := refreshClient.loader
-		return azappcfg.loadFeatureFlags(egCtx, settingsClient)
-	})
-
-	if err := eg.Wait(); err != nil {
+	if err := azappcfg.loadFeatureFlags(ctx, refreshClient.loader, refreshClient.enhFFLoader); err != nil {
 		log.Printf("Failed to reload feature flag configuration: %s", err.Error())
 		// Don't reset the timer if reload failed
 		return false, err
@@ -727,7 +723,7 @@ func (azappcfg *AzureAppConfiguration) refreshFeatureFlags(ctx context.Context,
 	return true, nil
 }
 
-func (azappcfg *AzureAppConfiguration) executeFailoverPolicy(ctx context.Context, operation func(*azappconfig.Client) error) error {
+func (azappcfg *AzureAppConfiguration) executeFailoverPolicy(ctx context.Context, operation func(appConfigClient) error) error {
 	clients, err := azappcfg.clientManager.getClients(ctx)
 	if err != nil {
 		return err
@@ -742,7 +738,7 @@ func (azappcfg *AzureAppConfiguration) executeFailoverPolicy(ctx context.Context
 		rotateClientsToNextEndpoint(clients, azappcfg.lastSuccessfulEndpoint)
 	}
 
-	if manager, ok := azappcfg.clientManager.(*configurationClientManager); ok {
+	if manager, ok := azappcfg.clientManager.(*appConfigClientManager); ok {
 		azappcfg.tracingOptions.ReplicaCount = len(manager.dynamicClients)
 	}
 
@@ -945,7 +941,7 @@ func normalizedWatchedSettings(s []WatchedSetting) []WatchedSetting {
 	return result
 }
 
-func (azappcfg *AzureAppConfiguration) newKeyValueRefreshClient(client *azappconfig.Client) refreshClient {
+func (azappcfg *AzureAppConfiguration) newKeyValueRefreshClient(client appConfigClient) refreshClient {
 	var monitor eTagsClient
 	if azappcfg.watchAll {
 		monitor = &pageETagsClient{
@@ -976,7 +972,7 @@ func (azappcfg *AzureAppConfiguration) newKeyValueRefreshClient(client *azappcon
 	}
 }
 
-func (azappcfg *AzureAppConfiguration) newFeatureFlagRefreshClient(client *azappconfig.Client) refreshClient {
+func (azappcfg *AzureAppConfiguration) newFeatureFlagRefreshClient(client appConfigClient) refreshClient {
 	return refreshClient{
 		loader: &selectorSettingsClient{
 			selectors:      azappcfg.ffSelectors,
@@ -988,6 +984,16 @@ func (azappcfg *AzureAppConfiguration) newFeatureFlagRefreshClient(client *azapp
 			tracingOptions: azappcfg.tracingOptions,
 			pageETags:      azappcfg.ffETags,
 		},
+		enhFFLoader: &enhFFSettingsClient{
+			selectors:      azappcfg.enhancedFFSelectors,
+			client:         client,
+			tracingOptions: azappcfg.tracingOptions,
+		},
+		enhFFMonitor: &enhFFETagsClient{
+			client:         client,
+			tracingOptions: azappcfg.tracingOptions,
+			pageETags:      azappcfg.enhancedFFETags,
+		},
 	}
 }
 
@@ -1029,7 +1035,7 @@ func (azappcfg *AzureAppConfiguration) updateFeatureFlagTracing(featureFlag map[
 	}
 }
 
-func rotateClientsToNextEndpoint(clients []*configurationClientWrapper, lastSuccessfulEndpoint string) {
+func rotateClientsToNextEndpoint(clients []*appConfigClientWrapper, lastSuccessfulEndpoint string) {
 	if len(clients) <= 1 {
 		return
 	}