Discovery end2end implementation + tests

Makefile

@@ -282,7 +282,7 @@ delete-outputs-dev-lab: ## Delete the outputs for the development lab cluster
 	kubectl delete -f lab/dev/resources/outputs
 
 .PHONY: apply-pipelines-dev-lab
-apply-pipelines-dev-lab: ## Apply the pipelines for the development lab cluster
+	§apply-pipelines-dev-lab: ## Apply the pipelines for the development lab cluster
 	kubectl apply -f lab/dev/resources/pipelines
 
 .PHONY: delete-pipelines-dev-lab
@@ -308,9 +308,10 @@ delete-targetsources-dev-lab: ## Delete the target sources for the development l
 ##@ Testing Lab
 
 .PHONY: run-integration-tests
-run-integration-tests: docker-build undeploy-test-cluster deploy-test-cluster install-test-cluster-dependencies load-test-image deploy install-kubectl install-gnmic install-containerlab deploy-test-topology apply-test-resources
+run-integration-tests: docker-build undeploy-test-cluster deploy-test-cluster install-test-cluster-dependencies load-test-image deploy deploy-test-http-server create-secrets-for-apiserver install-kubectl install-gnmic install-containerlab deploy-test-topology apply-test-resources send-target-to-apiserver
 	kubectl wait --for=condition=Ready cluster --all --timeout=180s
 	kubectl wait --for=condition=Ready pipeline --all --timeout=180s
+	kubectl wait --for=jsonpath='{.status.targetsCount}'=3 targetsource --all --timeout=180s
 	kubectl wait --for=jsonpath='{.status.connectionState}'=READY target --all --timeout=180s
 	kubectl get subscriptions -o yaml
 	kubectl get outputs -o yaml

api/v1alpha1/targetsource_types.go

@@ -58,18 +58,64 @@ type HTTPConfig struct {
 	// +kubebuilder:validation:Optional
 	URL string `json:"url,omitempty"`
 
+	// HTTP method used for the request.
+	//
+	// Defaults to GET if not specified.
+	//
+	// Supported values:
+	// - GET  (default, no request body)
+	// - POST (supports request body)
+	//
+	// +kubebuilder:validation:Enum=GET;POST
+	// +kubebuilder:default="GET"
+	// +kubebuilder:validation:Optional
+	Method string `json:"method,omitempty"`
+
+	// Optional HTTP headers to include in the request.
+	//
+	// These map directly to HTTP headers (key-value pairs).
+	//
+	// Example:
+	//   headers:
+	//     Content-Type: application/json
+	//     X-Custom-Header: value
+	//
+	// Precedence:
+	// - Authentication configuration overrides any conflicting headers e.g. Authorization
+	//
+	// +kubebuilder:validation:Optional
+	Headers map[string]string `json:"headers,omitempty"`
+
+	// Optional raw request body.
+	//
+	// Typically used with POST requests and contains JSON payload.
+	//
+	// Example:
+	//   body: |
+	//     {
+	//       "limit": 100,
+	//       "status": "active"
+	//     }
+	//
+	// Notes:
+	// - Ignored for GET requests
+	// - User must set appropriate Content-Type header if needed
+	//
+	// +kubebuilder:validation:Optional
+	Body string `json:"body,omitempty"`
+
 	// Optional authentication configuration for accessing the HTTP endpoint
 	// +kubebuilder:validation:Optional
 	Authentication *AuthenticationSpec `json:"authentication,omitempty"`
 
 	// Optional interval for polling the HTTP endpoint for targets
 	// TODO: document about default value
-	// +kubebuilder:default="6h"
+	// +kubebuilder:default="30m"
 	// +kubebuilder:validation:Optional
 	Interval *metav1.Duration `json:"interval,omitempty"`
 
 	// Optional timeout for HTTP requests to the endpoint
-	// +kubebuilder:default="10s"
+	// +kubebuilder:default="30s"
 	// +kubebuilder:validation:Optional
 	Timeout *metav1.Duration `json:"timeout,omitempty"`
 
@@ -132,68 +178,206 @@ type TokenAuthSpec struct {
 	TokenSecretRef *corev1.SecretKeySelector `json:"tokenSecretRef,omitempty"`
 }
 
-// PaginationSpec defines the configuration for paginating through responses from providers
+// PaginationSpec defines how pagination is handled for HTTP APIs.
+//
+// The pagination mechanism is fully server-driven. The loader will repeatedly:
+//  1. Extract the "next" reference from the response
+//  2. Use it to construct the next request
+//  3. Continue until no next reference is returned
+//
+// Supported pagination styles:
+//  1. Cursor-based:
+//     - Response returns a token (e.g. "next_page_token")
+//     - Client sends it back via a query parameter (e.g. "page_token")
+//  2. URL-based (nextLink):
+//     - Response returns a full URL
+//     - Client follows it directly without modification
+//  3. Expression-based extraction:
+//     - The next reference is extracted using a CEL expression
+//     - This allows access to nested fields or special keys
+//     (e.g. "@odata.nextLink")
+//
+// Behavior:
+//   - If the extracted value is a full URL, it will be used as-is
+//   - Otherwise, it is treated as a token and appended using RequestParam
+//   - The token is treated as opaque and must not be interpreted
+//
+// Example:
+//
+//	pagination:
+//	  nextField: "self.next_page_token"
+//	  requestParam: "page_token"
+//
+//	pagination:
+//	  nextField: "self['@odata.nextLink']"
 type PaginationSpec struct {
-	// Field name in the JSON response that contains the next page reference.
-	// The value can be either:
-	// - a full URL (used directly for the next request), or
-	// - a pagination token (appended as a query parameter using this field name as the key).
+	// CEL expression used to extract the next page reference from the response.
+	//
+	// The expression is evaluated with:
+	//   self -> full JSON response
+	//
+	// It must evaluate to either:
+	//   - string (full URL OR token), or
+	//   - null (indicates end of pagination)
+	//
+	// Examples:
+	//   "self.next"
+	//   "self.next_page_token"
+	//   "self['@odata.nextLink']"
 	//
-	// Must refer to a top-level key in the response object.
-	// Example: "next" or "nextToken"
+	// +kubebuilder:validation:Optional
 	NextField string `json:"nextField,omitempty"`
+
+	// Query parameter name used when the extracted value is a token.
+	//
+	// Required for token-based pagination.
+	// Ignored when NextField resolves to a full URL.
+	//
+	// Example:
+	//   requestParam: "page_token"
+	//
+	// +kubebuilder:validation:Optional
+	RequestParam string `json:"requestParam,omitempty"`
 }
 
-// CEL expressions to extract target fields from the response
-// and map them to the corresponding Target fields.
+// ResponseMappingSpec controls how targets are extracted from an HTTP JSON response.
+//
+// This allows you to map fields from a JSON API into targets using either:
+//   - simple direct field access (e.g. item["name"])
+//   - or CEL expressions for more advanced logic
+//
+// General behavior:
+//
+//  1. Selecting targets:
+//     - `targetsField` is a CEL expression that selects the list of targets
+//     - It runs once on the full response (`self`) and MUST return a list
+//     - If not set, the response itself must be a JSON array
+//
+//  2. Extracting fields:
+//     - Each field (name, address, port, labels, etc.) is handled independently
+//     - If a CEL expression is provided → it is evaluated
+//     - If not provided → the value is read directly from the target object
+//
+//  3. Available variables in CEL:
+//     - item -> the current target object
+//     - self -> the full HTTP response JSON
+//
+// Example:
+//
+//	Response:
+//	{
+//	  "results": [
+//	    { "name": "device1", "ip": "10.0.0.1", "env": "prod" }
+//	  ],
+//	  "meta": { "region": "eu-west" }
+//	}
+//
+//	Mapping:
+//	targetsField: "self.results"
+//
+//	name: ""            # direct → item["name"]
+//	address: "item.ip"  # CEL
+//
+//	labels:
+//	  env:    "item.env"
+//	  region: "self.meta.region"
 type ResponseMappingSpec struct {
-	// Field name in the JSON response that contains the list of items (targets).
-	// If not specified, the entire response is expected to be a list of items.
-	// All subsequent fields are specified relative to this field
-	// Example: "results" if the response is of the form {"results": [ ... list of items ... ]}
+	// CEL expression that selects the list of target objects from the response.
+	//
+	// This is evaluated once using:
+	//   self -> full JSON response
+	//
+	// Example:
+	//   targetsField: "self.results"
+	//
+	// If not set, the response itself must be a JSON array with the targets.
+	//
 	// +kubebuilder:validation:Optional
 	TargetsField string `json:"targetsField,omitempty"`
 
-	// CEL expression to extract the target name from the response
-	// If TargetsField is specified, this should be relative to TargetsField
+	// CEL expression for the target name.
+	//
+	// If not set, defaults to:
+	//   item["name"]
+	//
+	// Example:
+	//   "item.hostname"
+	//
 	// +kubebuilder:validation:Optional
-	Name string `json:"name"`
+	Name string `json:"name,omitempty"`
 
-	// CEL expression to extract the target Address from the response
-	// If TargetsField is specified, this should be relative to TargetsField
+	// CEL expression for the target address.
+	//
+	// If not set, defaults to:
+	//   item["address"]
+	//
+	// Example:
+	//   "item.ip"
+	//
 	// +kubebuilder:validation:Optional
-	Address string `json:"address"`
+	Address string `json:"address,omitempty"`
 
-	// CEL expression to extract the target port from the response
-	// If TargetsField is specified, this should be relative to TargetsField
+	// CEL expression for the target port.
+	//
+	// If not set, defaults to:
+	//   item["port"]
+	//
+	// Example:
+	//   "item.port"
+	//
 	// +kubebuilder:validation:Optional
 	Port string `json:"port,omitempty"`
 
-	// CEL expression to extract the target labels from the response
+	// CEL expression that returns a map of labels.
+	// The expression must evaluate to an object (map).
+	//
+	// Example:
+	//
+	//   labels: |
+	//     {
+	//       "env": item.environment,
+	//       "region": self.meta.region,
+	//       item.dynamicKey: "value"
+	//     }
+	//
+	// If not set, defaults to:
+	//   item["labels"]
+	//
+	// The resulting map will be converted into labels.
 	// The extracted labels will be merged with the static TargetLabels defined in the TargetSourceSpec,
 	// with values from the response taking precedence in case of conflicts.
+	//
 	// +kubebuilder:validation:Optional
-	Labels map[string]string `json:"labels,omitempty"`
+	Labels string `json:"labels,omitempty"`
 
-	// CEL expression to extract the target profile from the response
-	// If TargetsField is specified, this should be relative to TargetsField
+	// CEL expression for the target profile.
+	//
+	// If not set, defaults to:
+	//   item["targetProfile"]
+	//
+	// Example:
+	//   "item.type == 'edge' ? 'edge-profile' : 'default'"
+	//
 	// +kubebuilder:validation:Optional
 	TargetProfile string `json:"targetProfile,omitempty"`
 }
 
 // PushSpec defines the settings for event-based update mechanism (i.e. webhooks sent from the server)
 type PushSpec struct {
+	// +kubebuilder:validation:Required
 	// +kubebuilder:default=false
 	Enabled bool `json:"enabled"`
 
 	// +kubebuilder:validation:Optional
 	Auth *PushAuthSpec `json:"auth,omitempty"`
+
+	// +kubebuilder:validation:Optional
+	Signature *PushSignatureSpec `json:"signature,omitempty"`
 }
 
-// +kubebuilder:validation:ExactlyOneOf:=bearer;signature
+// +kubebuilder:validation:Optional
 type PushAuthSpec struct {
-	Bearer    *PushBearerAuthSpec    `json:"bearer,omitempty"`
-	Signature *PushSignatureAuthSpec `json:"signature,omitempty"`
+	Bearer *PushBearerAuthSpec `json:"bearer,omitempty"`
 }
 
 // +kubebuilder:validation:Required
@@ -202,15 +386,11 @@ type PushBearerAuthSpec struct {
 }
 
 // +kubebuilder:validation:Required
-type PushSignatureAuthSpec struct {
-	SecretRef *corev1.SecretKeySelector `json:"secretRef"`
-
-	// Header containing the signature
-	// +kubebuilder:validation:MinLength=1
-	Header string `json:"header"`
+type PushSignatureSpec struct {
+	SecretRef *corev1.SecretKeySelector `json:"secretRef,omitempty"`
 
 	// +kubebuilder:default="sha512"
-	// +kubebuilder:validation:Enum=sha1;sha256;sha512
+	// +kubebuilder:validation:Enum=sha256;sha512
 	Algorithm string `json:"algorithm"`
 }