diff --git a/internal/controller/appconfigurationprovider_controller.go b/internal/controller/appconfigurationprovider_controller.go index bfe923b..1621ae2 100644 --- a/internal/controller/appconfigurationprovider_controller.go +++ b/internal/controller/appconfigurationprovider_controller.go @@ -56,6 +56,7 @@ type ReconciliationState struct { SentinelETags map[acpv1.Sentinel]*azcore.ETag KeyValueETags map[acpv1.ComparableSelector][]*azcore.ETag FeatureFlagETags map[acpv1.ComparableSelector][]*azcore.ETag + EnhancedFeatureFlagETags map[acpv1.ComparableSelector][]*azcore.ETag ExistingK8sSecrets map[string]*loader.TargetK8sSecretMetadata NextKeyValueRefreshReconcileTime metav1.Time NextSecretReferenceRefreshReconcileTime metav1.Time @@ -149,6 +150,7 @@ func (reconciler *AzureAppConfigurationProviderReconciler) Reconcile(ctx context SentinelETags: make(map[acpv1.Sentinel]*azcore.ETag), KeyValueETags: make(map[acpv1.ComparableSelector][]*azcore.ETag), FeatureFlagETags: make(map[acpv1.ComparableSelector][]*azcore.ETag), + EnhancedFeatureFlagETags: make(map[acpv1.ComparableSelector][]*azcore.ETag), ExistingK8sSecrets: make(map[string]*loader.TargetK8sSecretMetadata), ClientManager: nil, } diff --git a/internal/controller/processor.go b/internal/controller/processor.go index faef321..bc5cca5 100644 --- a/internal/controller/processor.go +++ b/internal/controller/processor.go @@ -30,18 +30,20 @@ type AppConfigurationProviderProcessor struct { } type RefreshOptions struct { - keyValueRefreshEnabled bool - secretReferenceRefreshEnabled bool - secretReferenceRefreshNeeded bool - featureFlagRefreshEnabled bool - featureFlagRefreshNeeded bool - ConfigMapSettingPopulated bool - SecretSettingPopulated bool - sentinelChanged bool - keyValuePageETagsChanged bool - updatedSentinelETags map[acpv1.Sentinel]*azcore.ETag - updatedKeyValueETags map[acpv1.ComparableSelector][]*azcore.ETag - updatedFeatureFlagETags map[acpv1.ComparableSelector][]*azcore.ETag + keyValueRefreshEnabled bool + secretReferenceRefreshEnabled bool + secretReferenceRefreshNeeded bool + featureFlagRefreshEnabled bool + featureFlagRefreshNeeded bool + enhancedFeatureFlagRefreshNeeded bool + ConfigMapSettingPopulated bool + SecretSettingPopulated bool + sentinelChanged bool + keyValuePageETagsChanged bool + updatedSentinelETags map[acpv1.Sentinel]*azcore.ETag + updatedKeyValueETags map[acpv1.ComparableSelector][]*azcore.ETag + updatedFeatureFlagETags map[acpv1.ComparableSelector][]*azcore.ETag + updatedEnhancedFeatureFlagETags map[acpv1.ComparableSelector][]*azcore.ETag } func (processor *AppConfigurationProviderProcessor) PopulateSettings(existingConfigMap *corev1.ConfigMap, existingSecrets map[string]corev1.Secret) error { @@ -75,6 +77,7 @@ func (processor *AppConfigurationProviderProcessor) processFullReconciliation() processor.RefreshOptions.ConfigMapSettingPopulated = true processor.RefreshOptions.updatedKeyValueETags = updatedSettings.KeyValueETags processor.RefreshOptions.updatedFeatureFlagETags = updatedSettings.FeatureFlagETags + processor.RefreshOptions.updatedEnhancedFeatureFlagETags = updatedSettings.EnhancedFeatureFlagETags processor.RefreshOptions.updatedSentinelETags = updatedSettings.SentinelETags if processor.Provider.Spec.Secret != nil { processor.RefreshOptions.SecretSettingPopulated = true @@ -112,7 +115,11 @@ func (processor *AppConfigurationProviderProcessor) processFeatureFlagRefresh(ex return err } - if !processor.RefreshOptions.featureFlagRefreshNeeded { + if processor.RefreshOptions.enhancedFeatureFlagRefreshNeeded, err = (processor.Retriever).CheckIfEnhancedFeatureFlagsChanged(processor.Context, reconcileState.EnhancedFeatureFlagETags); err != nil { + return err + } + + if !(processor.RefreshOptions.featureFlagRefreshNeeded || processor.RefreshOptions.enhancedFeatureFlagRefreshNeeded) { reconcileState.NextFeatureFlagRefreshReconcileTime = nextFeatureFlagRefreshReconcileTime return nil } @@ -123,6 +130,7 @@ func (processor *AppConfigurationProviderProcessor) processFeatureFlagRefresh(ex } processor.RefreshOptions.updatedFeatureFlagETags = featureFlagRefreshedSettings.FeatureFlagETags + processor.RefreshOptions.updatedEnhancedFeatureFlagETags = featureFlagRefreshedSettings.EnhancedFeatureFlagETags processor.Settings = featureFlagRefreshedSettings processor.RefreshOptions.ConfigMapSettingPopulated = true // Update next refresh time only if settings updated successfully @@ -323,6 +331,10 @@ func (processor *AppConfigurationProviderProcessor) Finish() (ctrl.Result, error processor.ReconciliationState.FeatureFlagETags = processor.RefreshOptions.updatedFeatureFlagETags } + if processor.RefreshOptions.updatedEnhancedFeatureFlagETags != nil { + processor.ReconciliationState.EnhancedFeatureFlagETags = processor.RefreshOptions.updatedEnhancedFeatureFlagETags + } + if processor.ShouldReconcile { processor.ReconciliationState.SentinelETags = processor.RefreshOptions.updatedSentinelETags } @@ -348,7 +360,7 @@ func (processor *AppConfigurationProviderProcessor) Finish() (ctrl.Result, error processor.Provider.Status.RefreshStatus.LastKeyVaultReferenceRefreshTime = processor.CurrentTime } // Update provider last feature flag refresh time - if processor.RefreshOptions.featureFlagRefreshNeeded { + if processor.RefreshOptions.featureFlagRefreshNeeded || processor.RefreshOptions.enhancedFeatureFlagRefreshNeeded { processor.Provider.Status.RefreshStatus.LastFeatureFlagRefreshTime = processor.CurrentTime } // At least one dynamic feature is enabled, requeueAfterInterval need be recalculated @@ -361,16 +373,17 @@ func (processor *AppConfigurationProviderProcessor) Finish() (ctrl.Result, error func NewRefreshOptions() *RefreshOptions { return &RefreshOptions{ - keyValueRefreshEnabled: false, - secretReferenceRefreshEnabled: false, - secretReferenceRefreshNeeded: false, - featureFlagRefreshEnabled: false, - featureFlagRefreshNeeded: false, - ConfigMapSettingPopulated: false, - SecretSettingPopulated: false, - sentinelChanged: false, - keyValuePageETagsChanged: false, - updatedSentinelETags: make(map[acpv1.Sentinel]*azcore.ETag), + keyValueRefreshEnabled: false, + secretReferenceRefreshEnabled: false, + secretReferenceRefreshNeeded: false, + featureFlagRefreshEnabled: false, + featureFlagRefreshNeeded: false, + enhancedFeatureFlagRefreshNeeded: false, + ConfigMapSettingPopulated: false, + SecretSettingPopulated: false, + sentinelChanged: false, + keyValuePageETagsChanged: false, + updatedSentinelETags: make(map[acpv1.Sentinel]*azcore.ETag), } } diff --git a/internal/controller/processor_test.go b/internal/controller/processor_test.go index 173f590..1931393 100644 --- a/internal/controller/processor_test.go +++ b/internal/controller/processor_test.go @@ -36,6 +36,9 @@ var _ = Describe("AppConfiguationProvider processor", func() { BeforeEach(func() { mockCtrl = gomock.NewController(GinkgoT()) mockConfigurationSettings = mocks.NewMockConfigurationSettingsRetriever(mockCtrl) + // The dedicated feature flag endpoint is checked whenever the classic feature flag page ETags + // are unchanged; default to reporting no change so existing scenarios are unaffected. + mockConfigurationSettings.EXPECT().CheckIfEnhancedFeatureFlagsChanged(gomock.Any(), gomock.Any()).Return(false, nil).AnyTimes() }) AfterEach(func() { diff --git a/internal/controller/suite_test.go b/internal/controller/suite_test.go index 9abde0b..84c2cfb 100644 --- a/internal/controller/suite_test.go +++ b/internal/controller/suite_test.go @@ -88,6 +88,9 @@ var _ = BeforeSuite(func() { mockCtrl = gomock.NewController(GinkgoT()) mockConfigurationSettings = mocks.NewMockConfigurationSettingsRetriever(mockCtrl) + // The dedicated feature flag endpoint is checked whenever the classic feature flag page ETags + // are unchanged; default to reporting no change so existing scenarios are unaffected. + mockConfigurationSettings.EXPECT().CheckIfEnhancedFeatureFlagsChanged(gomock.Any(), gomock.Any()).Return(false, nil).AnyTimes() err = (&AzureAppConfigurationProviderReconciler{ Client: k8sManager.GetClient(), diff --git a/internal/loader/app_configuration_client.go b/internal/loader/app_configuration_client.go new file mode 100644 index 0000000..38703ea --- /dev/null +++ b/internal/loader/app_configuration_client.go @@ -0,0 +1,92 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +package loader + +import ( + "context" + + "github.com/Azure/azure-sdk-for-go/sdk/azcore" + "github.com/Azure/azure-sdk-for-go/sdk/azcore/runtime" + azappconfig "github.com/Azure/azure-sdk-for-go/sdk/data/azappconfig/v2" +) + +// AppConfigurationClient abstracts the Azure App Configuration operations used by the provider. +type AppConfigurationClient interface { + // Key-value configuration operations. + NewListSettingsPager(selector azappconfig.SettingSelector, options *azappconfig.ListSettingsOptions) *runtime.Pager[azappconfig.ListSettingsPageResponse] + GetSetting(ctx context.Context, key string, options *azappconfig.GetSettingOptions) (azappconfig.GetSettingResponse, error) + GetSnapshot(ctx context.Context, snapshotName string, options *azappconfig.GetSnapshotOptions) (azappconfig.GetSnapshotResponse, error) + NewListSettingsForSnapshotPager(snapshotName string, options *azappconfig.ListSettingsForSnapshotOptions) *runtime.Pager[azappconfig.ListSettingsForSnapshotResponse] + + // Feature flag operations served by the dedicated feature flag endpoint. + NewListFeatureFlagsPager(selector azappconfig.FeatureFlagSelector, options *azappconfig.ListFeatureFlagsOptions) *runtime.Pager[azappconfig.ListFeatureFlagsPageResponse] +} + +type appConfigurationClient struct { + configurationClient *azappconfig.Client + featureFlagClient *azappconfig.FeatureFlagClient +} + +func NewAppConfigurationClient(endpoint string, credential azcore.TokenCredential, options *azappconfig.ClientOptions) (AppConfigurationClient, error) { + configurationClient, err := azappconfig.NewClient(endpoint, credential, options) + if err != nil { + return nil, err + } + + featureFlagClient, err := azappconfig.NewFeatureFlagClient(endpoint, credential, featureFlagClientOptions(options)) + if err != nil { + return nil, err + } + + return &appConfigurationClient{ + configurationClient: configurationClient, + featureFlagClient: featureFlagClient, + }, nil +} + +func NewAppConfigurationClientFromConnectionString(connectionString string, options *azappconfig.ClientOptions) (AppConfigurationClient, error) { + configurationClient, err := azappconfig.NewClientFromConnectionString(connectionString, options) + if err != nil { + return nil, err + } + + featureFlagClient, err := azappconfig.NewFeatureFlagClientFromConnectionString(connectionString, featureFlagClientOptions(options)) + if err != nil { + return nil, err + } + + return &appConfigurationClient{ + configurationClient: configurationClient, + featureFlagClient: featureFlagClient, + }, nil +} + +// featureFlagClientOptions mirrors the configuration client options onto feature flag client options +func featureFlagClientOptions(options *azappconfig.ClientOptions) *azappconfig.FeatureFlagClientOptions { + if options == nil { + return nil + } + + return &azappconfig.FeatureFlagClientOptions{ClientOptions: options.ClientOptions} +} + +func (c *appConfigurationClient) NewListSettingsPager(selector azappconfig.SettingSelector, options *azappconfig.ListSettingsOptions) *runtime.Pager[azappconfig.ListSettingsPageResponse] { + return c.configurationClient.NewListSettingsPager(selector, options) +} + +func (c *appConfigurationClient) GetSetting(ctx context.Context, key string, options *azappconfig.GetSettingOptions) (azappconfig.GetSettingResponse, error) { + return c.configurationClient.GetSetting(ctx, key, options) +} + +func (c *appConfigurationClient) GetSnapshot(ctx context.Context, snapshotName string, options *azappconfig.GetSnapshotOptions) (azappconfig.GetSnapshotResponse, error) { + return c.configurationClient.GetSnapshot(ctx, snapshotName, options) +} + +func (c *appConfigurationClient) NewListSettingsForSnapshotPager(snapshotName string, options *azappconfig.ListSettingsForSnapshotOptions) *runtime.Pager[azappconfig.ListSettingsForSnapshotResponse] { + return c.configurationClient.NewListSettingsForSnapshotPager(snapshotName, options) +} + +func (c *appConfigurationClient) NewListFeatureFlagsPager(selector azappconfig.FeatureFlagSelector, options *azappconfig.ListFeatureFlagsOptions) *runtime.Pager[azappconfig.ListFeatureFlagsPageResponse] { + return c.featureFlagClient.NewListFeatureFlagsPager(selector, options) +} diff --git a/internal/loader/configuraiton_setting_loader_test.go b/internal/loader/configuraiton_setting_loader_test.go index 9b00ace..d0cb491 100644 --- a/internal/loader/configuraiton_setting_loader_test.go +++ b/internal/loader/configuraiton_setting_loader_test.go @@ -41,7 +41,7 @@ var ( mockCtrl *gomock.Controller mockCongiurationClientManager *MockClientManager endpointName string = "https://fake-endpoint" - fakeClientWrapper = ConfigurationClientWrapper{ + fakeClientWrapper = AppConfigurationClientWrapper{ Client: nil, Endpoint: endpointName, BackOffEndTime: metav1.Time{}, @@ -244,10 +244,10 @@ func (m *MockClientManager) EXPECT() *MockClientManagerMockRecorder { } // GetClients mocks base method. -func (m *MockClientManager) GetClients(arg0 context.Context) ([]*ConfigurationClientWrapper, error) { +func (m *MockClientManager) GetClients(arg0 context.Context) ([]*AppConfigurationClientWrapper, error) { m.ctrl.T.Helper() ret := m.ctrl.Call(m, "GetClients", arg0) - ret0, _ := ret[0].([]*ConfigurationClientWrapper) + ret0, _ := ret[0].([]*AppConfigurationClientWrapper) ret1, _ := ret[1].(error) return ret0, ret1 } @@ -294,7 +294,7 @@ func (m *MockSettingsClient) EXPECT() *MockSettingsClientMockRecorder { } // GetSettings mocks base method. -func (m *MockSettingsClient) GetSettings(arg0 context.Context, arg1 *azappconfig.Client) (*SettingsResponse, error) { +func (m *MockSettingsClient) GetSettings(arg0 context.Context, arg1 AppConfigurationClient) (*SettingsResponse, error) { m.ctrl.T.Helper() ret := m.ctrl.Call(m, "GetSettings", arg0, arg1) ret0, _ := ret[0].(*SettingsResponse) @@ -377,7 +377,7 @@ var _ = Describe("AppConfiguationProvider Get All Settings", func() { Spec: testSpec, } - mockCongiurationClientManager.EXPECT().GetClients(gomock.Any()).Return([]*ConfigurationClientWrapper{&fakeClientWrapper}, nil) + mockCongiurationClientManager.EXPECT().GetClients(gomock.Any()).Return([]*AppConfigurationClientWrapper{&fakeClientWrapper}, nil) configurationProvider, _ := NewConfigurationSettingLoader(testProvider, mockCongiurationClientManager, mockSettingsClient) secretValue := "fakeSecretValue" secret1 := azsecrets.GetSecretResponse{ @@ -444,7 +444,7 @@ var _ = Describe("AppConfiguationProvider Get All Settings", func() { Spec: testSpec, } - mockCongiurationClientManager.EXPECT().GetClients(gomock.Any()).Return([]*ConfigurationClientWrapper{&fakeClientWrapper}, nil) + mockCongiurationClientManager.EXPECT().GetClients(gomock.Any()).Return([]*AppConfigurationClientWrapper{&fakeClientWrapper}, nil) configurationProvider, _ := NewConfigurationSettingLoader(testProvider, mockCongiurationClientManager, mockSettingsClient) secretValue := "fakeSecretValue" secret1 := azsecrets.GetSecretResponse{ @@ -513,7 +513,7 @@ var _ = Describe("AppConfiguationProvider Get All Settings", func() { Spec: testSpec, } - mockCongiurationClientManager.EXPECT().GetClients(gomock.Any()).Return([]*ConfigurationClientWrapper{&fakeClientWrapper}, nil) + mockCongiurationClientManager.EXPECT().GetClients(gomock.Any()).Return([]*AppConfigurationClientWrapper{&fakeClientWrapper}, nil) configurationProvider, _ := NewConfigurationSettingLoader(testProvider, mockCongiurationClientManager, mockSettingsClient) secretValue := "fakeSecretValue" secret1 := azsecrets.GetSecretResponse{ @@ -573,7 +573,7 @@ var _ = Describe("AppConfiguationProvider Get All Settings", func() { Etags: keyValueEtags, } mockSettingsClient.EXPECT().GetSettings(gomock.Any(), gomock.Any()).Return(settingsResponse, nil) - mockCongiurationClientManager.EXPECT().GetClients(gomock.Any()).Return([]*ConfigurationClientWrapper{&fakeClientWrapper}, nil) + mockCongiurationClientManager.EXPECT().GetClients(gomock.Any()).Return([]*AppConfigurationClientWrapper{&fakeClientWrapper}, nil) configurationProvider, _ := NewConfigurationSettingLoader(testProvider, mockCongiurationClientManager, mockSettingsClient) allSettings, err := configurationProvider.CreateTargetSettings(context.Background(), mockResolveSecretReference) @@ -1064,7 +1064,7 @@ var _ = Describe("AppConfiguationProvider Get All Settings", func() { Etags: keyValueEtags, } mockSettingsClient.EXPECT().GetSettings(gomock.Any(), gomock.Any()).Return(settingsResponse, nil) - mockCongiurationClientManager.EXPECT().GetClients(gomock.Any()).Return([]*ConfigurationClientWrapper{&fakeClientWrapper}, nil) + mockCongiurationClientManager.EXPECT().GetClients(gomock.Any()).Return([]*AppConfigurationClientWrapper{&fakeClientWrapper}, nil) configurationProvider, _ := NewConfigurationSettingLoader(testProvider, mockCongiurationClientManager, mockSettingsClient) allSettings, err := configurationProvider.CreateTargetSettings(context.Background(), mockResolveSecretReference) @@ -1112,7 +1112,7 @@ var _ = Describe("AppConfiguationProvider Get All Settings", func() { Etags: keyValueEtags, } mockSettingsClient.EXPECT().GetSettings(gomock.Any(), gomock.Any()).Return(settingsResponse, nil) - mockCongiurationClientManager.EXPECT().GetClients(gomock.Any()).Return([]*ConfigurationClientWrapper{&fakeClientWrapper}, nil) + mockCongiurationClientManager.EXPECT().GetClients(gomock.Any()).Return([]*AppConfigurationClientWrapper{&fakeClientWrapper}, nil) configurationProvider, _ := NewConfigurationSettingLoader(testProvider, mockCongiurationClientManager, mockSettingsClient) allSettings, err := configurationProvider.CreateTargetSettings(context.Background(), mockResolveSecretReference) @@ -1154,7 +1154,7 @@ var _ = Describe("AppConfiguationProvider Get All Settings", func() { Etags: keyValueEtags, } mockSettingsClient.EXPECT().GetSettings(gomock.Any(), gomock.Any()).Return(settingsResponse, nil) - mockCongiurationClientManager.EXPECT().GetClients(gomock.Any()).Return([]*ConfigurationClientWrapper{&fakeClientWrapper}, nil) + mockCongiurationClientManager.EXPECT().GetClients(gomock.Any()).Return([]*AppConfigurationClientWrapper{&fakeClientWrapper}, nil) configurationProvider, _ := NewConfigurationSettingLoader(testProvider, mockCongiurationClientManager, mockSettingsClient) allSettings, err := configurationProvider.CreateTargetSettings(context.Background(), mockResolveSecretReference) @@ -1198,7 +1198,7 @@ var _ = Describe("AppConfiguationProvider Get All Settings", func() { Etags: keyValueEtags, } mockSettingsClient.EXPECT().GetSettings(gomock.Any(), gomock.Any()).Return(settingsResponse, nil) - mockCongiurationClientManager.EXPECT().GetClients(gomock.Any()).Return([]*ConfigurationClientWrapper{&fakeClientWrapper}, nil) + mockCongiurationClientManager.EXPECT().GetClients(gomock.Any()).Return([]*AppConfigurationClientWrapper{&fakeClientWrapper}, nil) configurationProvider, _ := NewConfigurationSettingLoader(testProvider, mockCongiurationClientManager, mockSettingsClient) allSettings, err := configurationProvider.CreateTargetSettings(context.Background(), mockResolveSecretReference) @@ -1250,8 +1250,8 @@ var _ = Describe("AppConfiguationProvider Get All Settings", func() { Settings: featureFlagsToReturn, Etags: featureFlagEtags, } - mockSettingsClient.EXPECT().GetSettings(gomock.Any(), gomock.Any()).Return(settingsResponse, nil).Times(2) - mockCongiurationClientManager.EXPECT().GetClients(gomock.Any()).Return([]*ConfigurationClientWrapper{&fakeClientWrapper}, nil).Times(2) + mockSettingsClient.EXPECT().GetSettings(gomock.Any(), gomock.Any()).Return(settingsResponse, nil).Times(3) + mockCongiurationClientManager.EXPECT().GetClients(gomock.Any()).Return([]*AppConfigurationClientWrapper{&fakeClientWrapper}, nil).Times(3) configurationProvider, _ := NewConfigurationSettingLoader(testProvider, mockCongiurationClientManager, mockSettingsClient) allSettings, err := configurationProvider.CreateTargetSettings(context.Background(), mockResolveSecretReference) @@ -1300,8 +1300,8 @@ var _ = Describe("AppConfiguationProvider Get All Settings", func() { Settings: featureFlagsToReturn, Etags: featureFlagEtags, } - mockSettingsClient.EXPECT().GetSettings(gomock.Any(), gomock.Any()).Return(settingsResponse, nil).Times(2) - mockCongiurationClientManager.EXPECT().GetClients(gomock.Any()).Return([]*ConfigurationClientWrapper{&fakeClientWrapper}, nil).Times(2) + mockSettingsClient.EXPECT().GetSettings(gomock.Any(), gomock.Any()).Return(settingsResponse, nil).Times(3) + mockCongiurationClientManager.EXPECT().GetClients(gomock.Any()).Return([]*AppConfigurationClientWrapper{&fakeClientWrapper}, nil).Times(3) configurationProvider, _ := NewConfigurationSettingLoader(testProvider, mockCongiurationClientManager, mockSettingsClient) allSettings, err := configurationProvider.CreateTargetSettings(context.Background(), mockResolveSecretReference) @@ -1351,8 +1351,8 @@ var _ = Describe("AppConfiguationProvider Get All Settings", func() { Settings: featureFlagsToReturn, Etags: featureFlagEtags, } - mockSettingsClient.EXPECT().GetSettings(gomock.Any(), gomock.Any()).Return(settingsResponse, nil).Times(2) - mockCongiurationClientManager.EXPECT().GetClients(gomock.Any()).Return([]*ConfigurationClientWrapper{&fakeClientWrapper}, nil).Times(2) + mockSettingsClient.EXPECT().GetSettings(gomock.Any(), gomock.Any()).Return(settingsResponse, nil).Times(3) + mockCongiurationClientManager.EXPECT().GetClients(gomock.Any()).Return([]*AppConfigurationClientWrapper{&fakeClientWrapper}, nil).Times(3) configurationProvider, _ := NewConfigurationSettingLoader(testProvider, mockCongiurationClientManager, mockSettingsClient) allSettings, err := configurationProvider.CreateTargetSettings(context.Background(), mockResolveSecretReference) @@ -1385,7 +1385,7 @@ var _ = Describe("AppConfiguationProvider Get All Settings", func() { err := errors.New("fake error") mockSettingsClient.EXPECT().GetSettings(gomock.Any(), gomock.Any()).Return(nil, err) - mockCongiurationClientManager.EXPECT().GetClients(gomock.Any()).Return([]*ConfigurationClientWrapper{&fakeClientWrapper}, nil) + mockCongiurationClientManager.EXPECT().GetClients(gomock.Any()).Return([]*AppConfigurationClientWrapper{&fakeClientWrapper}, nil) configurationProvider, _ := NewConfigurationSettingLoader(testProvider, mockCongiurationClientManager, mockSettingsClient) allSettings, err := configurationProvider.CreateTargetSettings(context.Background(), mockResolveSecretReference) @@ -1418,14 +1418,14 @@ var _ = Describe("AppConfiguationProvider Get All Settings", func() { netErr := &net.OpError{Err: errors.New("fake network error")} settingsToReturn := mockConfigurationSettings() - failedClient := ConfigurationClientWrapper{ + failedClient := AppConfigurationClientWrapper{ Client: nil, Endpoint: endpointName, BackOffEndTime: metav1.Time{}, FailedAttempts: 0, } - succeededClient := ConfigurationClientWrapper{ + succeededClient := AppConfigurationClientWrapper{ Client: nil, Endpoint: endpointName, BackOffEndTime: metav1.Time{}, @@ -1440,7 +1440,7 @@ var _ = Describe("AppConfiguationProvider Get All Settings", func() { } mockSettingsClient.EXPECT().GetSettings(gomock.Any(), gomock.Any()).Return(nil, netErr).Times(1) mockSettingsClient.EXPECT().GetSettings(gomock.Any(), gomock.Any()).Return(settingsResponse, nil).Times(1) - mockCongiurationClientManager.EXPECT().GetClients(gomock.Any()).Return([]*ConfigurationClientWrapper{&failedClient, &succeededClient}, nil) + mockCongiurationClientManager.EXPECT().GetClients(gomock.Any()).Return([]*AppConfigurationClientWrapper{&failedClient, &succeededClient}, nil) configurationProvider, _ := NewConfigurationSettingLoader(testProvider, mockCongiurationClientManager, mockSettingsClient) allSettings, err := configurationProvider.CreateTargetSettings(context.Background(), mockResolveSecretReference) @@ -1499,7 +1499,7 @@ var _ = Describe("TagFilters Support", func() { Spec: testSpec, } - mockCongiurationClientManager.EXPECT().GetClients(gomock.Any()).Return([]*ConfigurationClientWrapper{&fakeClientWrapper}, nil) + mockCongiurationClientManager.EXPECT().GetClients(gomock.Any()).Return([]*AppConfigurationClientWrapper{&fakeClientWrapper}, nil) configurationProvider, _ := NewConfigurationSettingLoader(testProvider, mockCongiurationClientManager, mockSettingsClient) settingsToReturn := mockConfigurationSettings() @@ -2301,8 +2301,8 @@ func TestSnapshotReferenceInCreateKeyValueSettings(t *testing.T) { // First GetClients call: for ExecuteFailoverPolicy (initial key-value loading) - returns valid wrapper // Second GetClients call: for resolveSnapshotReferences - returns empty to trigger error gomock.InOrder( - mockClientManager.EXPECT().GetClients(gomock.Any()).Return([]*ConfigurationClientWrapper{&fakeClientWrapper}, nil), - mockClientManager.EXPECT().GetClients(gomock.Any()).Return([]*ConfigurationClientWrapper{}, nil), + mockClientManager.EXPECT().GetClients(gomock.Any()).Return([]*AppConfigurationClientWrapper{&fakeClientWrapper}, nil), + mockClientManager.EXPECT().GetClients(gomock.Any()).Return([]*AppConfigurationClientWrapper{}, nil), ) configurationProvider, _ := NewConfigurationSettingLoader(testProvider, mockClientManager, mockSettingsClient) @@ -2354,7 +2354,7 @@ func TestSnapshotReferenceInCreateKeyValueSettings(t *testing.T) { } mockSettingsClient.EXPECT().GetSettings(gomock.Any(), gomock.Any()).Return(settingsResponse, nil) - mockClientManager.EXPECT().GetClients(gomock.Any()).Return([]*ConfigurationClientWrapper{&fakeClientWrapper}, nil).AnyTimes() + mockClientManager.EXPECT().GetClients(gomock.Any()).Return([]*AppConfigurationClientWrapper{&fakeClientWrapper}, nil).AnyTimes() configurationProvider, _ := NewConfigurationSettingLoader(testProvider, mockClientManager, mockSettingsClient) rawSettings, err := configurationProvider.CreateKeyValueSettings(context.Background(), nil) @@ -2413,11 +2413,11 @@ func TestSnapshotReferenceInCreateKeyValueSettings(t *testing.T) { } mockSettingsClient.EXPECT().GetSettings(gomock.Any(), gomock.Any()).Return(settingsResponse, nil) - fakeClient := &ConfigurationClientWrapper{ + fakeClient := &AppConfigurationClientWrapper{ Client: nil, Endpoint: EndpointName, } - mockClientManager.EXPECT().GetClients(gomock.Any()).Return([]*ConfigurationClientWrapper{fakeClient}, nil).AnyTimes() + mockClientManager.EXPECT().GetClients(gomock.Any()).Return([]*AppConfigurationClientWrapper{fakeClient}, nil).AnyTimes() configurationProvider, _ := NewConfigurationSettingLoader(testProvider, mockClientManager, mockSettingsClient) _, err := configurationProvider.CreateKeyValueSettings(context.Background(), nil) diff --git a/internal/loader/configuration_client_manager.go b/internal/loader/configuration_client_manager.go index ddc05f4..fa7cc98 100644 --- a/internal/loader/configuration_client_manager.go +++ b/internal/loader/configuration_client_manager.go @@ -39,8 +39,8 @@ import ( type ConfigurationClientManager struct { ReplicaDiscoveryEnabled bool LoadBalancingEnabled bool - StaticClientWrappers []*ConfigurationClientWrapper - DynamicClientWrappers []*ConfigurationClientWrapper + StaticClientWrappers []*AppConfigurationClientWrapper + DynamicClientWrappers []*AppConfigurationClientWrapper validDomain string endpoint string credential azcore.TokenCredential @@ -51,15 +51,15 @@ type ConfigurationClientManager struct { lastSuccessfulEndpoint string } -type ConfigurationClientWrapper struct { +type AppConfigurationClientWrapper struct { Endpoint string - Client *azappconfig.Client + Client AppConfigurationClient BackOffEndTime metav1.Time FailedAttempts int } type ClientManager interface { - GetClients(ctx context.Context) ([]*ConfigurationClientWrapper, error) + GetClients(ctx context.Context) ([]*AppConfigurationClientWrapper, error) RefreshClients(ctx context.Context) } @@ -114,7 +114,7 @@ func NewConfigurationClientManager(ctx context.Context, provider acpv1.AzureAppC } var err error - var staticClient *azappconfig.Client + var staticClient AppConfigurationClient if provider.Spec.ConnectionStringReference != nil { connectionString, err := getConnectionStringParameter(ctx, types.NamespacedName{Namespace: provider.Namespace, Name: *provider.Spec.ConnectionStringReference}) if err != nil { @@ -132,21 +132,21 @@ func NewConfigurationClientManager(ctx context.Context, provider acpv1.AzureAppC if manager.id, err = parseConnectionString(connectionString, IdSection); err != nil { return nil, err } - if staticClient, err = azappconfig.NewClientFromConnectionString(connectionString, newClientOptions()); err != nil { + if staticClient, err = NewAppConfigurationClientFromConnectionString(connectionString, newClientOptions()); err != nil { return nil, err } } else { if manager.credential, err = CreateTokenCredential(ctx, provider.Spec.Auth, provider.Namespace); err != nil { return nil, err } - if staticClient, err = azappconfig.NewClient(*provider.Spec.Endpoint, manager.credential, newClientOptions()); err != nil { + if staticClient, err = NewAppConfigurationClient(*provider.Spec.Endpoint, manager.credential, newClientOptions()); err != nil { return nil, err } manager.endpoint = *provider.Spec.Endpoint } manager.validDomain = getValidDomain(manager.endpoint) - manager.StaticClientWrappers = []*ConfigurationClientWrapper{{ + manager.StaticClientWrappers = []*AppConfigurationClientWrapper{{ Endpoint: manager.endpoint, Client: staticClient, BackOffEndTime: metav1.Time{}, @@ -156,9 +156,9 @@ func NewConfigurationClientManager(ctx context.Context, provider acpv1.AzureAppC return manager, nil } -func (manager *ConfigurationClientManager) GetClients(ctx context.Context) ([]*ConfigurationClientWrapper, error) { +func (manager *ConfigurationClientManager) GetClients(ctx context.Context) ([]*AppConfigurationClientWrapper, error) { currentTime := metav1.Now() - clients := make([]*ConfigurationClientWrapper, 0) + clients := make([]*AppConfigurationClientWrapper, 0) for _, clientWrapper := range manager.StaticClientWrappers { if currentTime.After(clientWrapper.BackOffEndTime.Time) { clients = append(clients, clientWrapper) @@ -227,7 +227,7 @@ func (manager *ConfigurationClientManager) DiscoverFallbackClients(ctx context.C srvTargetHosts[i], srvTargetHosts[j] = srvTargetHosts[j], srvTargetHosts[i] } - newDynamicClients := make([]*ConfigurationClientWrapper, 0) + newDynamicClients := make([]*AppConfigurationClientWrapper, 0) for _, host := range srvTargetHosts { if isValidEndpoint(host, manager.validDomain) { targetEndpoint := "https://" + host @@ -239,7 +239,7 @@ func (manager *ConfigurationClientManager) DiscoverFallbackClients(ctx context.C klog.Warningf("build fallback clients failed, %s", err.Error()) return } - newDynamicClients = append(newDynamicClients, &ConfigurationClientWrapper{ + newDynamicClients = append(newDynamicClients, &AppConfigurationClientWrapper{ Endpoint: targetEndpoint, Client: client, BackOffEndTime: metav1.Time{}, @@ -298,9 +298,9 @@ func QuerySrvTargetHost(ctx context.Context, host string) ([]string, error) { return results, nil } -func (manager *ConfigurationClientManager) newConfigurationClient(endpoint string) (*azappconfig.Client, error) { +func (manager *ConfigurationClientManager) newConfigurationClient(endpoint string) (AppConfigurationClient, error) { if manager.credential != nil { - return azappconfig.NewClient(endpoint, manager.credential, newClientOptions()) + return NewAppConfigurationClient(endpoint, manager.credential, newClientOptions()) } connectionStr := buildConnectionString(endpoint, manager.secret, manager.id) @@ -308,7 +308,7 @@ func (manager *ConfigurationClientManager) newConfigurationClient(endpoint strin return nil, fmt.Errorf("failed to build connection string for fallback client") } - return azappconfig.NewClientFromConnectionString(connectionStr, newClientOptions()) + return NewAppConfigurationClientFromConnectionString(connectionStr, newClientOptions()) } func isValidEndpoint(host string, validDomain string) bool { diff --git a/internal/loader/configuration_setting_loader.go b/internal/loader/configuration_setting_loader.go index d43cee1..411f3ba 100644 --- a/internal/loader/configuration_setting_loader.go +++ b/internal/loader/configuration_setting_loader.go @@ -46,11 +46,12 @@ type ConfigurationSettingLoader struct { type TargetKeyValueSettings struct { ConfigMapSettings map[string]string // Multiple secrets could be managed - SecretSettings map[string]corev1.Secret - K8sSecrets map[string]*TargetK8sSecretMetadata - KeyValueETags map[acpv1.ComparableSelector][]*azcore.ETag - FeatureFlagETags map[acpv1.ComparableSelector][]*azcore.ETag - SentinelETags map[acpv1.Sentinel]*azcore.ETag + SecretSettings map[string]corev1.Secret + K8sSecrets map[string]*TargetK8sSecretMetadata + KeyValueETags map[acpv1.ComparableSelector][]*azcore.ETag + SentinelETags map[acpv1.Sentinel]*azcore.ETag + FeatureFlagETags map[acpv1.ComparableSelector][]*azcore.ETag + EnhancedFeatureFlagETags map[acpv1.ComparableSelector][]*azcore.ETag } type TargetK8sSecretMetadata struct { @@ -60,19 +61,21 @@ type TargetK8sSecretMetadata struct { } type RawSettings struct { - KeyValueSettings map[string]*string - IsJsonContentTypeMap map[string]bool - FeatureFlagSettings map[string]interface{} - SecretSettings map[string]corev1.Secret - K8sSecrets map[string]*TargetK8sSecretMetadata - KeyValueETags map[acpv1.ComparableSelector][]*azcore.ETag - FeatureFlagETags map[acpv1.ComparableSelector][]*azcore.ETag + KeyValueSettings map[string]*string + IsJsonContentTypeMap map[string]bool + FeatureFlagSettings map[string]interface{} + SecretSettings map[string]corev1.Secret + K8sSecrets map[string]*TargetK8sSecretMetadata + KeyValueETags map[acpv1.ComparableSelector][]*azcore.ETag + FeatureFlagETags map[acpv1.ComparableSelector][]*azcore.ETag + EnhancedFeatureFlagETags map[acpv1.ComparableSelector][]*azcore.ETag } type ConfigurationSettingsRetriever interface { CreateTargetSettings(ctx context.Context, resolveSecretReference SecretReferenceResolver) (*TargetKeyValueSettings, error) CheckAndRefreshSentinels(ctx context.Context, provider *acpv1.AzureAppConfigurationProvider, eTags map[acpv1.Sentinel]*azcore.ETag) (bool, map[acpv1.Sentinel]*azcore.ETag, error) CheckPageETags(ctx context.Context, eTags map[acpv1.ComparableSelector][]*azcore.ETag) (bool, error) + CheckIfEnhancedFeatureFlagsChanged(ctx context.Context, eTags map[acpv1.ComparableSelector][]*azcore.ETag) (bool, error) RefreshKeyValueSettings(ctx context.Context, existingConfigMapSettings *map[string]string, resolveSecretReference SecretReferenceResolver) (*TargetKeyValueSettings, error) RefreshFeatureFlagSettings(ctx context.Context, existingConfigMapSettings *map[string]string) (*TargetKeyValueSettings, error) ResolveSecretReferences(ctx context.Context, kvReferencesToResolve map[string]*TargetK8sSecretMetadata, kvResolver SecretReferenceResolver) (*TargetKeyValueSettings, error) @@ -95,6 +98,7 @@ const ( FeatureFlagKeyPrefix string = ".appconfig.featureflag/" FeatureFlagSectionName string = "feature_flags" FeatureManagementSectionName string = "feature_management" + FeatureFlagIdKey string = "id" PreservedSecretTypeTag string = ".kubernetes.secret.type" CertTypePem string = "application/x-pem-file" CertTypePfx string = "application/x-pkcs12" @@ -133,9 +137,24 @@ func (csl *ConfigurationSettingLoader) CreateTargetSettings(ctx context.Context, } if csl.Spec.FeatureFlag != nil { - if rawSettings.FeatureFlagSettings, rawSettings.FeatureFlagETags, err = csl.getFeatureFlagSettings(ctx); err != nil { + featureFlags, featureFlagETags, err := csl.loadFeatureFlags(ctx) + if err != nil { + return nil, err + } + + enhancedFeatureFlags, enhancedFeatureFlagETags, err := csl.loadEnhancedFeatureFlags(ctx) + if err != nil { return nil, err } + + deduplicatedFeatureFlags, err := csl.ProcessFeatureFlags(featureFlags, enhancedFeatureFlags) + if err != nil { + return nil, err + } + + rawSettings.FeatureFlagETags = featureFlagETags + rawSettings.EnhancedFeatureFlagETags = enhancedFeatureFlagETags + rawSettings.FeatureFlagSettings = deduplicatedFeatureFlags } typedSettings, err := createTypedSettings(rawSettings, csl.Spec.Target.ConfigMapData) @@ -144,12 +163,13 @@ func (csl *ConfigurationSettingLoader) CreateTargetSettings(ctx context.Context, } return &TargetKeyValueSettings{ - ConfigMapSettings: typedSettings, - SecretSettings: rawSettings.SecretSettings, - K8sSecrets: rawSettings.K8sSecrets, - KeyValueETags: rawSettings.KeyValueETags, - FeatureFlagETags: rawSettings.FeatureFlagETags, - SentinelETags: initializedSentinelETags, + ConfigMapSettings: typedSettings, + SecretSettings: rawSettings.SecretSettings, + K8sSecrets: rawSettings.K8sSecrets, + KeyValueETags: rawSettings.KeyValueETags, + EnhancedFeatureFlagETags: rawSettings.EnhancedFeatureFlagETags, + FeatureFlagETags: rawSettings.FeatureFlagETags, + SentinelETags: initializedSentinelETags, }, nil } @@ -180,7 +200,17 @@ func (csl *ConfigurationSettingLoader) RefreshKeyValueSettings(ctx context.Conte } func (csl *ConfigurationSettingLoader) RefreshFeatureFlagSettings(ctx context.Context, existingConfigMapSetting *map[string]string) (*TargetKeyValueSettings, error) { - latestFeatureFlagSettings, latestFeatureFlagETags, err := csl.getFeatureFlagSettings(ctx) + featureFlags, featureFlagETags, err := csl.loadFeatureFlags(ctx) + if err != nil { + return nil, err + } + + enhancedFeatureFlags, enhancedFeatureFlagETags, err := csl.loadEnhancedFeatureFlags(ctx) + if err != nil { + return nil, err + } + + latestFeatureFlags, err := csl.ProcessFeatureFlags(featureFlags, enhancedFeatureFlags) if err != nil { return nil, err } @@ -190,7 +220,7 @@ func (csl *ConfigurationSettingLoader) RefreshFeatureFlagSettings(ctx context.Co return nil, err } - existingSettings[FeatureManagementSectionName] = latestFeatureFlagSettings + existingSettings[FeatureManagementSectionName] = latestFeatureFlags typedStr, err := marshalJsonYaml(existingSettings, csl.Spec.Target.ConfigMapData) if err != nil { return nil, err @@ -200,7 +230,8 @@ func (csl *ConfigurationSettingLoader) RefreshFeatureFlagSettings(ctx context.Co ConfigMapSettings: map[string]string{ csl.Spec.Target.ConfigMapData.Key: typedStr, }, - FeatureFlagETags: latestFeatureFlagETags, + EnhancedFeatureFlagETags: enhancedFeatureFlagETags, + FeatureFlagETags: featureFlagETags, }, nil } @@ -208,8 +239,8 @@ func (csl *ConfigurationSettingLoader) RefreshFeatureFlagSettings(ctx context.Co type settingProcessContext struct { rawSettings *RawSettings resolver *SecretReferenceResolver - allowSnapshotRef bool // false inside a snapshot's resolved settings to prevent nested resolution - snapshotClient *azappconfig.Client // lazily initialized when the first snapshot reference is resolved + allowSnapshotRef bool // false inside a snapshot's resolved settings to prevent nested resolution + snapshotClient AppConfigurationClient // lazily initialized when the first snapshot reference is resolved useAIConfiguration bool useAIChatCompletionConfiguration bool } @@ -383,8 +414,6 @@ func (csl *ConfigurationSettingLoader) processSettings(ctx context.Context, sett if err := csl.processSettings(ctx, snapshotSettings, nestedCtx); err != nil { return err } - processCtx.useAIConfiguration = processCtx.useAIConfiguration || nestedCtx.useAIConfiguration - processCtx.useAIChatCompletionConfiguration = processCtx.useAIChatCompletionConfiguration || nestedCtx.useAIChatCompletionConfiguration default: processCtx.rawSettings.KeyValueSettings[trimmedKey] = setting.Value processCtx.rawSettings.IsJsonContentTypeMap[trimmedKey] = isJsonContentType(setting.ContentType) @@ -468,7 +497,24 @@ func (csl *ConfigurationSettingLoader) CheckPageETags(ctx context.Context, eTags return settingsResponse.Etags != nil, nil } -func (csl *ConfigurationSettingLoader) getFeatureFlagSettings(ctx context.Context) (map[string]interface{}, map[acpv1.ComparableSelector][]*azcore.ETag, error) { +func (csl *ConfigurationSettingLoader) CheckIfEnhancedFeatureFlagsChanged(ctx context.Context, eTags map[acpv1.ComparableSelector][]*azcore.ETag) (bool, error) { + settingsClient := csl.SettingsClient + if settingsClient == nil { + settingsClient = &EnhancedFeatureFlagEtagsClient{ + etags: eTags, + } + } + + settingsResponse, err := csl.ExecuteFailoverPolicy(ctx, settingsClient) + if err != nil { + return false, err + } + + // a non-nil Etags map signals that the feature flag endpoint page ETags changed + return settingsResponse.Etags != nil, nil +} + +func (csl *ConfigurationSettingLoader) loadFeatureFlags(ctx context.Context) ([]azappconfig.Setting, map[acpv1.ComparableSelector][]*azcore.ETag, error) { featureFlagFilters := GetFeatureFlagFilters(csl.Spec) settingsClient := csl.SettingsClient if settingsClient == nil { @@ -476,47 +522,101 @@ func (csl *ConfigurationSettingLoader) getFeatureFlagSettings(ctx context.Contex selectors: featureFlagFilters, } } + + settingsResponse, err := csl.ExecuteFailoverPolicy(ctx, settingsClient) + if err != nil { + return nil, nil, err + } + + return settingsResponse.Settings, settingsResponse.Etags, nil +} + +func (csl *ConfigurationSettingLoader) loadEnhancedFeatureFlags(ctx context.Context) ([]azappconfig.FeatureFlag, map[acpv1.ComparableSelector][]*azcore.ETag, error) { + settingsClient := csl.SettingsClient + if settingsClient == nil { + settingsClient = &EnhancedFeatureFlagSettingsClient{ + enhancedFeatureFlagSelectors: GetEnhancedFeatureFlagFilters(csl.Spec), + } + } + settingsResponse, err := csl.ExecuteFailoverPolicy(ctx, settingsClient) if err != nil { return nil, nil, err } - settingsLength := len(settingsResponse.Settings) - featureFlagExist := make(map[string]bool, settingsLength) - deduplicatedFeatureFlags := make([]interface{}, 0) + csl.TracingFeatures.UseEnhancedFeatureFlag = len(settingsResponse.EnhancedFeatureFlags) > 0 + return settingsResponse.EnhancedFeatureFlags, settingsResponse.Etags, nil +} + +func (csl *ConfigurationSettingLoader) ProcessFeatureFlags(featureFlags []azappconfig.Setting, enhancedFeatureFlags []azappconfig.FeatureFlag) (map[string]interface{}, error) { clientEndpoint := "" if manager, ok := csl.ClientManager.(*ConfigurationClientManager); ok { // use primary client endpoint in feature flag reference clientEndpoint = manager.StaticClientWrappers[0].Endpoint } - // if settings returned like this: [{"id": "Beta"...}, {"id": "Alpha"...}, {"id": "Beta"...}], we need to deduplicate it to [{"id": "Alpha"...}, {"id": "Beta"...}], the last one wins - for i := settingsLength - 1; i >= 0; i-- { - key := *settingsResponse.Settings[i].Key - if featureFlagExist[key] { + mergedFeatureFlags := make([]map[string]interface{}, 0, len(featureFlags)+len(enhancedFeatureFlags)) + for _, setting := range featureFlags { + if setting.Key == nil || setting.Value == nil { continue } - featureFlagExist[key] = true - var out map[string]interface{} - err := json.Unmarshal([]byte(*settingsResponse.Settings[i].Value), &out) - if err != nil { - return nil, nil, fmt.Errorf("failed to unmarshal feature flag settings: %s", err.Error()) + var ff map[string]interface{} + if err := json.Unmarshal([]byte(*setting.Value), &ff); err != nil { + return nil, fmt.Errorf("failed to unmarshal feature flag settings: %s", err.Error()) + } + + featureFlagReference := fmt.Sprintf("%s/kv/%s", clientEndpoint, *setting.Key) + if setting.Label != nil && strings.TrimSpace(*setting.Label) != "" { + featureFlagReference += fmt.Sprintf("?label=%s", *setting.Label) + } + + populateTelemetryMetadata(ff, setting.ETag, featureFlagReference) + mergedFeatureFlags = append(mergedFeatureFlags, ff) + } + + for _, featureFlag := range enhancedFeatureFlags { + if featureFlag.Name == nil { + continue + } + + featureFlagReference := fmt.Sprintf("%s/ff/%s", clientEndpoint, *featureFlag.Name) + if featureFlag.Label != nil && strings.TrimSpace(*featureFlag.Label) != "" { + featureFlagReference += fmt.Sprintf("?label=%s", *featureFlag.Label) + } + + convertedFF := convertToMicrosoftSchema(featureFlag) + populateTelemetryMetadata(convertedFF, featureFlag.ETag, featureFlagReference) + mergedFeatureFlags = append(mergedFeatureFlags, convertedFF) + } + + // Deduplicate by id keeping the last occurrence so enhanced feature flags supersede classic ones. + return deduplicateFeatureFlags(mergedFeatureFlags), nil +} + +func deduplicateFeatureFlags(featureFlags []map[string]interface{}) map[string]interface{} { + seen := make(map[string]bool, len(featureFlags)) + deduplicated := make([]interface{}, 0, len(featureFlags)) + + for i := len(featureFlags) - 1; i >= 0; i-- { + id, _ := featureFlags[i][FeatureFlagIdKey].(string) + if seen[id] { + continue } - populateTelemetryMetadata(out, settingsResponse.Settings[i], clientEndpoint) - deduplicatedFeatureFlags = append(deduplicatedFeatureFlags, out) + seen[id] = true + deduplicated = append(deduplicated, featureFlags[i]) } - // reverse the deduplicateFeatureFlags to keep the order - for i, j := 0, len(deduplicatedFeatureFlags)-1; i < j; i, j = i+1, j-1 { - deduplicatedFeatureFlags[i], deduplicatedFeatureFlags[j] = deduplicatedFeatureFlags[j], deduplicatedFeatureFlags[i] + // reverse to restore the original order + for i, j := 0, len(deduplicated)-1; i < j; i, j = i+1, j-1 { + deduplicated[i], deduplicated[j] = deduplicated[j], deduplicated[i] } // featureFlagSection = {"feature_flags": [{...}, {...}]} - var featureFlagSection = map[string]interface{}{ - FeatureFlagSectionName: deduplicatedFeatureFlags, + featureFlagSection := map[string]interface{}{ + FeatureFlagSectionName: deduplicated, } - return featureFlagSection, settingsResponse.Etags, nil + return featureFlagSection } func (csl *ConfigurationSettingLoader) ResolveSecretReferences( @@ -700,7 +800,7 @@ func (csl *ConfigurationSettingLoader) ExecuteFailoverPolicy(ctx context.Context return nil, fmt.Errorf("all app configuration clients failed to get settings: %v", errors) } -func updateClientBackoffStatus(clientWrapper *ConfigurationClientWrapper, successful bool) { +func updateClientBackoffStatus(clientWrapper *AppConfigurationClientWrapper, successful bool) { if successful { clientWrapper.BackOffEndTime = metav1.Time{} // Reset FailedAttempts when client succeeded @@ -774,16 +874,20 @@ func GetKeyValueFilters(acpSpec acpv1.AzureAppConfigurationProviderSpec) []acpv1 return deduplicateFilters(normalizeFilter(acpSpec.Configuration.Selectors)) } +func GetEnhancedFeatureFlagFilters(acpSpec acpv1.AzureAppConfigurationProviderSpec) []acpv1.Selector { + if acpSpec.FeatureFlag == nil { + return make([]acpv1.Selector, 0) + } + + return deduplicateFilters(normalizeFilter(acpSpec.FeatureFlag.Selectors)) +} + func GetFeatureFlagFilters(acpSpec acpv1.AzureAppConfigurationProviderSpec) []acpv1.Selector { - featureFlagFilters := make([]acpv1.Selector, 0) - - if acpSpec.FeatureFlag != nil { - featureFlagFilters = deduplicateFilters(normalizeFilter(acpSpec.FeatureFlag.Selectors)) - for i := 0; i < len(featureFlagFilters); i++ { - if featureFlagFilters[i].KeyFilter != nil { - prefixedFeatureFlagFilter := FeatureFlagKeyPrefix + *featureFlagFilters[i].KeyFilter - featureFlagFilters[i].KeyFilter = &prefixedFeatureFlagFilter - } + featureFlagFilters := GetEnhancedFeatureFlagFilters(acpSpec) + for i := 0; i < len(featureFlagFilters); i++ { + if featureFlagFilters[i].KeyFilter != nil { + prefixedFeatureFlagFilter := FeatureFlagKeyPrefix + *featureFlagFilters[i].KeyFilter + featureFlagFilters[i].KeyFilter = &prefixedFeatureFlagFilter } } @@ -1058,7 +1162,7 @@ func MergeSecret(secret map[string]corev1.Secret, newSecret map[string]corev1.Se } // rotates the slice to the left by k positions -func rotate(clients []*ConfigurationClientWrapper, k int) { +func rotate(clients []*AppConfigurationClientWrapper, k int) { n := len(clients) k = k % n if k == 0 { @@ -1072,7 +1176,7 @@ func rotate(clients []*ConfigurationClientWrapper, k int) { reverseClients(clients, n-k, n-1) } -func reverseClients(clients []*ConfigurationClientWrapper, start, end int) { +func reverseClients(clients []*AppConfigurationClientWrapper, start, end int) { for start < end { clients[start], clients[end] = clients[end], clients[start] start++ @@ -1080,18 +1184,7 @@ func reverseClients(clients []*ConfigurationClientWrapper, start, end int) { } } -func generateFeatureFlagReference(setting azappconfig.Setting, endpoint string) string { - featureFlagReference := fmt.Sprintf("%s/kv/%s", endpoint, *setting.Key) - - // Check if the label is present and not empty - if setting.Label != nil && strings.TrimSpace(*setting.Label) != "" { - featureFlagReference += fmt.Sprintf("?label=%s", *setting.Label) - } - - return featureFlagReference -} - -func populateTelemetryMetadata(featureFlag map[string]interface{}, setting azappconfig.Setting, endpoint string) { +func populateTelemetryMetadata(featureFlag map[string]interface{}, eTag *azcore.ETag, featureFlagRef string) { if telemetry, ok := featureFlag[TelemetryKey].(map[string]interface{}); ok { if enabled, ok := telemetry[EnabledKey].(bool); ok && enabled { metadata, _ := telemetry[MetadataKey].(map[string]interface{}) @@ -1100,8 +1193,10 @@ func populateTelemetryMetadata(featureFlag map[string]interface{}, setting azapp } // Set the new metadata - metadata[ETagKey] = *setting.ETag - metadata[FeatureFlagReferenceKey] = generateFeatureFlagReference(setting, endpoint) + if eTag != nil { + metadata[ETagKey] = *eTag + } + metadata[FeatureFlagReferenceKey] = featureFlagRef telemetry[MetadataKey] = metadata } } diff --git a/internal/loader/feature_flag_converter.go b/internal/loader/feature_flag_converter.go new file mode 100644 index 0000000..29f26ed --- /dev/null +++ b/internal/loader/feature_flag_converter.go @@ -0,0 +1,182 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +package loader + +import ( + "encoding/json" + + azappconfig "github.com/Azure/azure-sdk-for-go/sdk/data/azappconfig/v2" +) + +// convertToMicrosoftSchema converts an enhanced FeatureFlag returned by new feature flag +// endpoint into the Microsoft Feature Management schema object (snake_case) used within the +// `feature_management.feature_flags` array. +func convertToMicrosoftSchema(featureFlag azappconfig.FeatureFlag) map[string]interface{} { + result := make(map[string]interface{}) + + if featureFlag.Name != nil { + result["id"] = *featureFlag.Name + } + + if featureFlag.Enabled != nil { + result["enabled"] = *featureFlag.Enabled + } else { + result["enabled"] = false + } + + if featureFlag.Description != nil { + result["description"] = *featureFlag.Description + } + + // conditions: filters -> client_filters, requirementType -> requirement_type + conditions := make(map[string]interface{}) + clientFilters := make([]interface{}, 0) + if featureFlag.Conditions != nil { + for _, filter := range featureFlag.Conditions.Filters { + clientFilter := make(map[string]interface{}) + if filter.Name != nil { + clientFilter["name"] = *filter.Name + } + if filter.Parameters != nil { + parameters := make(map[string]interface{}, len(filter.Parameters)) + for key, value := range filter.Parameters { + parameters[key] = parseFeatureFlagValue(value) + } + clientFilter["parameters"] = parameters + } + clientFilters = append(clientFilters, clientFilter) + } + } + conditions["client_filters"] = clientFilters + if featureFlag.Conditions != nil && featureFlag.Conditions.RequirementType != nil { + conditions["requirement_type"] = string(*featureFlag.Conditions.RequirementType) + } + result["conditions"] = conditions + + // variants: value -> configuration_value, statusOverride -> status_override + if featureFlag.Variants != nil { + variants := make([]interface{}, 0, len(featureFlag.Variants)) + for _, variant := range featureFlag.Variants { + variantMap := make(map[string]interface{}) + if variant.Name != nil { + variantMap["name"] = *variant.Name + } + if variant.Value != nil { + variantMap["configuration_value"] = parseFeatureFlagValue(variant.Value) + } + if variant.StatusOverride != nil { + variantMap["status_override"] = string(*variant.StatusOverride) + } + variants = append(variants, variantMap) + } + result["variants"] = variants + } + + // allocation: camelCase -> snake_case + if featureFlag.Allocation != nil { + allocation := make(map[string]interface{}) + source := featureFlag.Allocation + if source.DefaultWhenDisabled != nil { + allocation["default_when_disabled"] = *source.DefaultWhenDisabled + } + if source.DefaultWhenEnabled != nil { + allocation["default_when_enabled"] = *source.DefaultWhenEnabled + } + if source.Percentile != nil { + percentiles := make([]interface{}, 0, len(source.Percentile)) + for _, percentile := range source.Percentile { + percentileMap := make(map[string]interface{}) + if percentile.Variant != nil { + percentileMap["variant"] = *percentile.Variant + } + if percentile.From != nil { + percentileMap["from"] = *percentile.From + } + if percentile.To != nil { + percentileMap["to"] = *percentile.To + } + percentiles = append(percentiles, percentileMap) + } + allocation["percentile"] = percentiles + } + if source.Group != nil { + groups := make([]interface{}, 0, len(source.Group)) + for _, group := range source.Group { + groupMap := make(map[string]interface{}) + if group.Variant != nil { + groupMap["variant"] = *group.Variant + } + if group.Groups != nil { + groupMap["groups"] = toInterfaceSlice(group.Groups) + } + groups = append(groups, groupMap) + } + allocation["group"] = groups + } + if source.User != nil { + users := make([]interface{}, 0, len(source.User)) + for _, user := range source.User { + userMap := make(map[string]interface{}) + if user.Variant != nil { + userMap["variant"] = *user.Variant + } + if user.Users != nil { + userMap["users"] = toInterfaceSlice(user.Users) + } + users = append(users, userMap) + } + allocation["user"] = users + } + if source.Seed != nil { + allocation["seed"] = *source.Seed + } + result["allocation"] = allocation + } + + // telemetry: metadata is (re)populated later by populateTelemetryMetadata with ETag/FeatureFlagReference + if featureFlag.Telemetry != nil { + telemetry := make(map[string]interface{}) + if featureFlag.Telemetry.Enabled != nil { + telemetry["enabled"] = *featureFlag.Telemetry.Enabled + } else { + telemetry["enabled"] = false + } + if featureFlag.Telemetry.Metadata != nil { + metadata := make(map[string]interface{}, len(featureFlag.Telemetry.Metadata)) + for key, value := range featureFlag.Telemetry.Metadata { + if value != nil { + metadata[key] = *value + } + } + telemetry["metadata"] = metadata + } + result["telemetry"] = telemetry + } + + return result +} + +// Attempting to parse the string as JSON recovers booleans, numbers, and nested objects; non-JSON strings are returned as-is. +func parseFeatureFlagValue(raw *string) interface{} { + if raw == nil { + return nil + } + + var parsed interface{} + if err := json.Unmarshal([]byte(*raw), &parsed); err == nil { + return parsed + } + + return *raw +} + +// toInterfaceSlice converts a slice of strings into a slice of interface{} for inclusion in the +// generic map that is marshaled into the feature management schema. +func toInterfaceSlice(values []string) []interface{} { + result := make([]interface{}, len(values)) + for i, value := range values { + result[i] = value + } + return result +} diff --git a/internal/loader/feature_flag_converter_test.go b/internal/loader/feature_flag_converter_test.go new file mode 100644 index 0000000..2a7409e --- /dev/null +++ b/internal/loader/feature_flag_converter_test.go @@ -0,0 +1,201 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +package loader + +import ( + acpv1 "azappconfig/provider/api/v1" + "context" + "encoding/json" + "fmt" + "testing" + + "github.com/Azure/azure-sdk-for-go/sdk/azcore" + "github.com/Azure/azure-sdk-for-go/sdk/azcore/runtime" + azappconfig "github.com/Azure/azure-sdk-for-go/sdk/data/azappconfig/v2" +) + +// fakeAppConfigurationClient is a test double for AppConfigurationClient that serves the provided +// pages from in-memory slices. Only the list operations exercised by the feature flag loading path +// are backed by data; the remaining methods return empty results. +type fakeAppConfigurationClient struct { + keyValuePages [][]azappconfig.Setting + featureFlagPages [][]azappconfig.FeatureFlag +} + +func (c *fakeAppConfigurationClient) NewListSettingsPager(_ azappconfig.SettingSelector, _ *azappconfig.ListSettingsOptions) *runtime.Pager[azappconfig.ListSettingsPageResponse] { + pages := c.keyValuePages + if len(pages) == 0 { + pages = [][]azappconfig.Setting{{}} + } + index := 0 + return runtime.NewPager(runtime.PagingHandler[azappconfig.ListSettingsPageResponse]{ + More: func(azappconfig.ListSettingsPageResponse) bool { return index < len(pages) }, + Fetcher: func(context.Context, *azappconfig.ListSettingsPageResponse) (azappconfig.ListSettingsPageResponse, error) { + page := pages[index] + index++ + etag := azcore.ETag(fmt.Sprintf("kv-page-%d", index)) + return azappconfig.ListSettingsPageResponse{Settings: page, ETag: &etag}, nil + }, + }) +} + +func (c *fakeAppConfigurationClient) GetSetting(context.Context, string, *azappconfig.GetSettingOptions) (azappconfig.GetSettingResponse, error) { + return azappconfig.GetSettingResponse{}, nil +} + +func (c *fakeAppConfigurationClient) GetSnapshot(context.Context, string, *azappconfig.GetSnapshotOptions) (azappconfig.GetSnapshotResponse, error) { + return azappconfig.GetSnapshotResponse{}, nil +} + +func (c *fakeAppConfigurationClient) NewListSettingsForSnapshotPager(_ string, _ *azappconfig.ListSettingsForSnapshotOptions) *runtime.Pager[azappconfig.ListSettingsForSnapshotResponse] { + return runtime.NewPager(runtime.PagingHandler[azappconfig.ListSettingsForSnapshotResponse]{ + More: func(azappconfig.ListSettingsForSnapshotResponse) bool { return false }, + Fetcher: func(context.Context, *azappconfig.ListSettingsForSnapshotResponse) (azappconfig.ListSettingsForSnapshotResponse, error) { + return azappconfig.ListSettingsForSnapshotResponse{}, nil + }, + }) +} + +func (c *fakeAppConfigurationClient) NewListFeatureFlagsPager(_ azappconfig.FeatureFlagSelector, _ *azappconfig.ListFeatureFlagsOptions) *runtime.Pager[azappconfig.ListFeatureFlagsPageResponse] { + pages := c.featureFlagPages + if len(pages) == 0 { + pages = [][]azappconfig.FeatureFlag{{}} + } + index := 0 + return runtime.NewPager(runtime.PagingHandler[azappconfig.ListFeatureFlagsPageResponse]{ + More: func(azappconfig.ListFeatureFlagsPageResponse) bool { return index < len(pages) }, + Fetcher: func(context.Context, *azappconfig.ListFeatureFlagsPageResponse) (azappconfig.ListFeatureFlagsPageResponse, error) { + page := pages[index] + index++ + etag := azcore.ETag(fmt.Sprintf("ff-page-%d", index)) + return azappconfig.ListFeatureFlagsPageResponse{FeatureFlags: page, ETag: &etag}, nil + }, + }) +} + +func newTypedFeatureFlag(name string, enabled bool) azappconfig.FeatureFlag { + etag := azcore.ETag("etag-" + name) + return azappconfig.FeatureFlag{ + Name: &name, + Enabled: &enabled, + ETag: &etag, + Conditions: &azappconfig.FeatureFlagConditions{Filters: []azappconfig.FeatureFlagFilter{}}, + } +} + +func TestConvertFeatureFlagToMap(t *testing.T) { + name := "Variant" + enabled := true + filterName := "Microsoft.TimeWindow" + startParam := "Mon, 01 Jan 2024 00:00:00 GMT" + requirementType := azappconfig.RequirementTypeAll + offName, offValue := "Off", "false" + onName, onValue := "On", "true" + statusOverride := azappconfig.StatusOverrideDisabled + defaultVariant := "Off" + percentileVariant := "On" + from, to := 0.0, 50.0 + seed := "seed-value" + telemetryEnabled := true + + featureFlag := azappconfig.FeatureFlag{ + Name: &name, + Enabled: &enabled, + Conditions: &azappconfig.FeatureFlagConditions{ + RequirementType: &requirementType, + Filters: []azappconfig.FeatureFlagFilter{ + {Name: &filterName, Parameters: map[string]*string{"Start": &startParam}}, + }, + }, + Variants: []azappconfig.FeatureFlagVariantDefinition{ + {Name: &offName, Value: &offValue, StatusOverride: &statusOverride}, + {Name: &onName, Value: &onValue}, + }, + Allocation: &azappconfig.FeatureFlagAllocation{ + DefaultWhenEnabled: &defaultVariant, + DefaultWhenDisabled: &defaultVariant, + Percentile: []azappconfig.PercentileAllocation{{Variant: &percentileVariant, From: &from, To: &to}}, + Seed: &seed, + }, + Telemetry: &azappconfig.FeatureFlagTelemetryConfiguration{Enabled: &telemetryEnabled}, + } + + actual, err := json.Marshal(convertToMicrosoftSchema(featureFlag)) + if err != nil { + t.Fatalf("failed to marshal converted feature flag: %s", err) + } + + expected := `{"allocation":{"default_when_disabled":"Off","default_when_enabled":"Off","percentile":[{"from":0,"to":50,"variant":"On"}],"seed":"seed-value"},"conditions":{"client_filters":[{"name":"Microsoft.TimeWindow","parameters":{"Start":"Mon, 01 Jan 2024 00:00:00 GMT"}}],"requirement_type":"All"},"enabled":true,"id":"Variant","telemetry":{"enabled":true},"variants":[{"configuration_value":false,"name":"Off","status_override":"Disabled"},{"configuration_value":true,"name":"On"}]}` + if string(actual) != expected { + t.Errorf("unexpected converted feature flag.\n got: %s\nwant: %s", actual, expected) + } +} + +func TestEnhancedFeatureFlagSettingsClientLoadsEnhancedFlags(t *testing.T) { + endpointNameFilter := "*" + nullLabel := "\x00" + + client := &fakeAppConfigurationClient{ + featureFlagPages: [][]azappconfig.FeatureFlag{{ + newTypedFeatureFlag("Shared", false), + newTypedFeatureFlag("EnhancedOnly", true), + }}, + } + + settingsClient := &EnhancedFeatureFlagSettingsClient{ + enhancedFeatureFlagSelectors: []acpv1.Selector{{KeyFilter: &endpointNameFilter, LabelFilter: &nullLabel}}, + } + + response, err := settingsClient.GetSettings(context.Background(), client) + if err != nil { + t.Fatalf("GetSettings returned error: %s", err) + } + + // The enhanced client only loads flags from the dedicated feature flag endpoint; merging with + // classic feature flags happens later in ProcessFeatureFlags. + if len(response.EnhancedFeatureFlags) != 2 { + t.Fatalf("expected 2 enhanced feature flags, got %d", len(response.EnhancedFeatureFlags)) + } + if response.EnhancedFeatureFlags[0].Name == nil || *response.EnhancedFeatureFlags[0].Name != "Shared" { + t.Errorf("expected first enhanced flag to be 'Shared', got %v", response.EnhancedFeatureFlags[0].Name) + } + if len(response.Settings) != 0 { + t.Errorf("expected no classic settings from the enhanced client, got %d", len(response.Settings)) + } +} + +func TestFeatureFlagEndpointEtagSettingsClientDetectsChanges(t *testing.T) { + nameFilter := "*" + nullLabel := "\x00" + comparable := acpv1.MakeComparable(acpv1.Selector{KeyFilter: &nameFilter, LabelFilter: &nullLabel}) + + client := &fakeAppConfigurationClient{ + featureFlagPages: [][]azappconfig.FeatureFlag{{newTypedFeatureFlag("Beta", true)}}, + } + + // The fake client assigns the first page the ETag "ff-page-1". + unchangedETag := azcore.ETag("ff-page-1") + unchangedClient := &EnhancedFeatureFlagEtagsClient{ + etags: map[acpv1.ComparableSelector][]*azcore.ETag{comparable: {&unchangedETag}}, + } + unchangedResponse, err := unchangedClient.GetSettings(context.Background(), client) + if err != nil { + t.Fatalf("GetSettings returned error: %s", err) + } + if unchangedResponse.Etags != nil { + t.Errorf("expected no change to be detected when page ETags match") + } + + staleETag := azcore.ETag("stale-etag") + changedClient := &EnhancedFeatureFlagEtagsClient{ + etags: map[acpv1.ComparableSelector][]*azcore.ETag{comparable: {&staleETag}}, + } + changedResponse, err := changedClient.GetSettings(context.Background(), client) + if err != nil { + t.Fatalf("GetSettings returned error: %s", err) + } + if changedResponse.Etags == nil { + t.Errorf("expected a change to be detected when page ETags differ") + } +} diff --git a/internal/loader/mocks/mock_configuration_settings_retriever.go b/internal/loader/mocks/mock_configuration_settings_retriever.go index 446f7f6..00c6a34 100644 --- a/internal/loader/mocks/mock_configuration_settings_retriever.go +++ b/internal/loader/mocks/mock_configuration_settings_retriever.go @@ -53,6 +53,21 @@ func (mr *MockConfigurationSettingsRetrieverMockRecorder) CheckAndRefreshSentine return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "CheckAndRefreshSentinels", reflect.TypeOf((*MockConfigurationSettingsRetriever)(nil).CheckAndRefreshSentinels), arg0, arg1, arg2) } +// CheckIfEnhancedFeatureFlagsChanged mocks base method. +func (m *MockConfigurationSettingsRetriever) CheckIfEnhancedFeatureFlagsChanged(arg0 context.Context, arg1 map[v1.ComparableSelector][]*azcore.ETag) (bool, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "CheckIfEnhancedFeatureFlagsChanged", arg0, arg1) + ret0, _ := ret[0].(bool) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// CheckIfEnhancedFeatureFlagsChanged indicates an expected call of CheckIfEnhancedFeatureFlagsChanged. +func (mr *MockConfigurationSettingsRetrieverMockRecorder) CheckIfEnhancedFeatureFlagsChanged(arg0, arg1 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "CheckIfEnhancedFeatureFlagsChanged", reflect.TypeOf((*MockConfigurationSettingsRetriever)(nil).CheckIfEnhancedFeatureFlagsChanged), arg0, arg1) +} + // CheckPageETags mocks base method. func (m *MockConfigurationSettingsRetriever) CheckPageETags(arg0 context.Context, arg1 map[v1.ComparableSelector][]*azcore.ETag) (bool, error) { m.ctrl.T.Helper() diff --git a/internal/loader/request_tracing.go b/internal/loader/request_tracing.go index 710b29e..e1eee40 100644 --- a/internal/loader/request_tracing.go +++ b/internal/loader/request_tracing.go @@ -24,6 +24,7 @@ type TracingFeatures struct { UseAIConfiguration bool UseAIChatCompletionConfiguration bool UseSnapshotReference bool + UseEnhancedFeatureFlag bool } // Feature flag telemetry @@ -49,6 +50,7 @@ const ( AIConfigurationKey string = "AI" AIChatCompletionKey string = "AICC" SnapshotReferenceKey string = "SnapshotRef" + EnhancedFeatureFlagKey string = "EnhFF" ) func createCorrelationContextHeader(ctx context.Context, provider acpv1.AzureAppConfigurationProvider, tracingFeatures TracingFeatures) http.Header { @@ -103,6 +105,10 @@ func createCorrelationContextHeader(ctx context.Context, provider acpv1.AzureApp features = append(features, SnapshotReferenceKey) } + if tracingFeatures.UseEnhancedFeatureFlag { + features = append(features, EnhancedFeatureFlagKey) + } + if len(features) > 0 { featureStr := "Features=" + strings.Join(features, TracingFeatureDelimiterKey) output = append(output, featureStr) diff --git a/internal/loader/settings_client.go b/internal/loader/settings_client.go index adeb575..f336692 100644 --- a/internal/loader/settings_client.go +++ b/internal/loader/settings_client.go @@ -17,8 +17,9 @@ import ( //go:generate mockgen -destination=mocks/mock_settings_client.go -package mocks . SettingsClient type SettingsResponse struct { - Settings []azappconfig.Setting - Etags map[acpv1.ComparableSelector][]*azcore.ETag + Settings []azappconfig.Setting + Etags map[acpv1.ComparableSelector][]*azcore.ETag + EnhancedFeatureFlags []azappconfig.FeatureFlag } type EtagSettingsClient struct { @@ -35,11 +36,21 @@ type SelectorSettingsClient struct { selectors []acpv1.Selector } +// EnhancedFeatureFlagEtagsClient is used to check if the enhanced feature flags have changed +type EnhancedFeatureFlagEtagsClient struct { + etags map[acpv1.ComparableSelector][]*azcore.ETag +} + +// EnhancedFeatureFlagSettingsClient loads enhanced feature flags +type EnhancedFeatureFlagSettingsClient struct { + enhancedFeatureFlagSelectors []acpv1.Selector +} + type SettingsClient interface { - GetSettings(ctx context.Context, client *azappconfig.Client) (*SettingsResponse, error) + GetSettings(ctx context.Context, client AppConfigurationClient) (*SettingsResponse, error) } -func (s *EtagSettingsClient) GetSettings(ctx context.Context, client *azappconfig.Client) (*SettingsResponse, error) { +func (s *EtagSettingsClient) GetSettings(ctx context.Context, client AppConfigurationClient) (*SettingsResponse, error) { nullString := "\x00" settingsResponse := &SettingsResponse{} for comparableFilter, pageEtags := range s.etags { @@ -90,7 +101,7 @@ func (s *EtagSettingsClient) GetSettings(ctx context.Context, client *azappconfi return settingsResponse, nil } -func (s *SentinelSettingsClient) GetSettings(ctx context.Context, client *azappconfig.Client) (*SettingsResponse, error) { +func (s *SentinelSettingsClient) GetSettings(ctx context.Context, client AppConfigurationClient) (*SettingsResponse, error) { sentinelSetting, err := client.GetSetting(ctx, s.sentinel.Key, &azappconfig.GetSettingOptions{Label: s.sentinel.Label, OnlyIfChanged: s.etag}) if err != nil { var respErr *azcore.ResponseError @@ -121,7 +132,7 @@ func (s *SentinelSettingsClient) GetSettings(ctx context.Context, client *azappc }, nil } -func (s *SelectorSettingsClient) GetSettings(ctx context.Context, client *azappconfig.Client) (*SettingsResponse, error) { +func (s *SelectorSettingsClient) GetSettings(ctx context.Context, client AppConfigurationClient) (*SettingsResponse, error) { settings := make([]azappconfig.Setting, 0) pageEtags := make(map[acpv1.ComparableSelector][]*azcore.ETag) @@ -162,7 +173,7 @@ func (s *SelectorSettingsClient) GetSettings(ctx context.Context, client *azappc }, nil } -func loadSnapshotSettings(ctx context.Context, client *azappconfig.Client, snapshotName string) ([]azappconfig.Setting, error) { +func loadSnapshotSettings(ctx context.Context, client AppConfigurationClient, snapshotName string) ([]azappconfig.Setting, error) { settings := make([]azappconfig.Setting, 0) snapshot, err := client.GetSnapshot(ctx, snapshotName, nil) if err != nil { @@ -189,3 +200,89 @@ func loadSnapshotSettings(ctx context.Context, client *azappconfig.Client, snaps return settings, nil } + +func (s *EnhancedFeatureFlagSettingsClient) GetSettings(ctx context.Context, client AppConfigurationClient) (*SettingsResponse, error) { + enhancedFeatureFlags := make([]azappconfig.FeatureFlag, 0) + pageEtags := make(map[acpv1.ComparableSelector][]*azcore.ETag) + + for _, filter := range s.enhancedFeatureFlagSelectors { + if filter.KeyFilter != nil { + selector := azappconfig.FeatureFlagSelector{ + NameFilter: filter.KeyFilter, + LabelFilter: filter.LabelFilter, + TagsFilter: filter.TagFilters, + Fields: azappconfig.AllFeatureFlagFields(), + } + pager := client.NewListFeatureFlagsPager(selector, nil) + latestEtags := make([]*azcore.ETag, 0) + + for pager.More() { + page, err := pager.NextPage(ctx) + if err != nil { + return nil, err + } else if page.FeatureFlags != nil { + enhancedFeatureFlags = append(enhancedFeatureFlags, page.FeatureFlags...) + latestEtags = append(latestEtags, page.ETag) + } + } + // update the etags for the filter + pageEtags[acpv1.MakeComparable(filter)] = latestEtags + } + } + + return &SettingsResponse{ + EnhancedFeatureFlags: enhancedFeatureFlags, + Etags: pageEtags, + }, nil +} + +func (s *EnhancedFeatureFlagEtagsClient) GetSettings(ctx context.Context, client AppConfigurationClient) (*SettingsResponse, error) { + settingsResponse := &SettingsResponse{} + for comparableFilter, storedETags := range s.etags { + filter := acpv1.FromComparable(comparableFilter) + if filter.KeyFilter != nil { + selector := azappconfig.FeatureFlagSelector{ + NameFilter: filter.KeyFilter, + LabelFilter: filter.LabelFilter, + TagsFilter: filter.TagFilters, + Fields: azappconfig.AllFeatureFlagFields(), + } + + pager := client.NewListFeatureFlagsPager(selector, nil) + latestETags := make([]*azcore.ETag, 0) + for pager.More() { + page, err := pager.NextPage(ctx) + if err != nil { + return nil, err + } + latestETags = append(latestETags, page.ETag) + } + + if !equalETagSlices(storedETags, latestETags) { + settingsResponse.Etags = make(map[acpv1.ComparableSelector][]*azcore.ETag) + return settingsResponse, nil + } + } + } + + return settingsResponse, nil +} + +// equalETagSlices reports whether two ordered slices of page ETags are equivalent. +func equalETagSlices(a, b []*azcore.ETag) bool { + if len(a) != len(b) { + return false + } + for i := range a { + if a[i] == nil || b[i] == nil { + if a[i] != b[i] { + return false + } + continue + } + if *a[i] != *b[i] { + return false + } + } + return true +}