Add Hydra composition for GiGL configs

deployment/configs/e2e/preprocessor.yaml

@@ -0,0 +1,11 @@
+# Shared preprocessing capacity for GiGL's end-to-end tests.
+edge_preprocessor_config:
+  num_workers: 1
+  max_num_workers: 128
+  machine_type: "n2d-highmem-32"
+  disk_size_gb: 300
+node_preprocessor_config:
+  num_workers: 1
+  max_num_workers: 128
+  machine_type: "n2d-highmem-64"
+  disk_size_gb: 300

deployment/configs/e2e/shared.yaml

@@ -0,0 +1,15 @@
+# Shared infrastructure for GiGL's end-to-end test resource configs.
+resource_labels:
+  # COMPONENT is filled by GiGL with the active pipeline component.
+  cost_resource_group_tag: dev_experiments_COMPONENT
+  cost_resource_group: gigl_platform
+common_compute_config:
+  project: "external-snap-ci-github-gigl"
+  region: "us-central1"
+  temp_assets_bucket: "gs://gigl-cicd-temp"
+  temp_regional_assets_bucket: "gs://gigl-cicd-temp"
+  perm_assets_bucket: "gs://gigl-cicd-perm"
+  temp_assets_bq_dataset_name: "gigl_temp_assets"
+  embedding_bq_dataset_name: "gigl_embeddings"
+  gcp_service_account_email: "untrusted-external-github-gigl@external-snap-ci-github-gigl.iam.gserviceaccount.com"
+  dataflow_runner: "DataflowRunner"

deployment/configs/e2e_cicd_resource_config.yaml

@@ -1,38 +1,9 @@
 # GiglResourceConfig for our e2e intergration tests.
-shared_resource_config:
-  # Resource labels are just compute labels that should be attached to all compute resources spun up by GiGL.
-  # So a practitioner can have a more fine grained understanding of resource utilization and cost of the resources in their GCP billing.
-  # Read more here: https://cloud.google.com/compute/docs/labeling-resources#what-are-labels
-  resource_labels:
-  # We have a 63 character limit for cost_resource_group_tag.
+defaults:
+  - e2e/shared@shared_resource_config
+  - e2e/preprocessor@preprocessor_config
+  - _self_
 
-  # COMPONENT is one of {pre|sgs|spl|tra|inf|pos} standing for:
-  #   {Preprocessor | Subgraph Sampler | Split Generator | Trainer | Inference
-  #   | Post  Processor} so we can get more accurate cost measurements
-  #   of each component. This will be automatically filled in code.
-    cost_resource_group_tag: dev_experiments_COMPONENT
-    cost_resource_group: gigl_platform
-  common_compute_config:
-    project: "external-snap-ci-github-gigl"
-    region: "us-central1"
-    temp_assets_bucket: "gs://gigl-cicd-temp"
-    temp_regional_assets_bucket: "gs://gigl-cicd-temp"
-    perm_assets_bucket: "gs://gigl-cicd-perm"
-    temp_assets_bq_dataset_name: "gigl_temp_assets"
-    embedding_bq_dataset_name: "gigl_embeddings"
-    gcp_service_account_email: "untrusted-external-github-gigl@external-snap-ci-github-gigl.iam.gserviceaccount.com"
-    dataflow_runner: "DataflowRunner"
-preprocessor_config:
-  edge_preprocessor_config:
-    num_workers: 1
-    max_num_workers: 128
-    machine_type: "n2d-highmem-32"
-    disk_size_gb: 300
-  node_preprocessor_config:
-    num_workers: 1
-    max_num_workers: 128
-    machine_type: "n2d-highmem-64"
-    disk_size_gb: 300
 subgraph_sampler_config:
   machine_type: "n2d-highmem-16"
   num_local_ssds: 2

deployment/configs/e2e_glt_gs_resource_config.yaml

@@ -1,31 +1,11 @@
 # Diffs from e2e_glt_resource_config.yaml
 # - Swap vertex_ai_inferencer_config for vertex_ai_graph_store_inferencer_config
 # - Swap vertex_ai_trainer_config for vertex_ai_graph_store_trainer_config
-shared_resource_config:
-  resource_labels:
-    cost_resource_group_tag: dev_experiments_COMPONENT
-    cost_resource_group: gigl_platform
-  common_compute_config:
-    project: "external-snap-ci-github-gigl"
-    region: "us-central1"
-    temp_assets_bucket: "gs://gigl-cicd-temp"
-    temp_regional_assets_bucket: "gs://gigl-cicd-temp"
-    perm_assets_bucket: "gs://gigl-cicd-perm"
-    temp_assets_bq_dataset_name: "gigl_temp_assets"
-    embedding_bq_dataset_name: "gigl_embeddings"
-    gcp_service_account_email: "untrusted-external-github-gigl@external-snap-ci-github-gigl.iam.gserviceaccount.com"
-    dataflow_runner: "DataflowRunner"
-preprocessor_config:
-  edge_preprocessor_config:
-    num_workers: 1
-    max_num_workers: 128
-    machine_type: "n2d-highmem-32"
-    disk_size_gb: 300
-  node_preprocessor_config:
-    num_workers: 1
-    max_num_workers: 128
-    machine_type: "n2d-highmem-64"
-    disk_size_gb: 300
+defaults:
+  - e2e/shared@shared_resource_config
+  - e2e/preprocessor@preprocessor_config
+  - _self_
+
 trainer_resource_config:
   vertex_ai_graph_store_trainer_config:
     graph_store_pool:

deployment/configs/e2e_glt_resource_config.yaml

@@ -1,28 +1,8 @@
-shared_resource_config:
-  resource_labels:
-    cost_resource_group_tag: dev_experiments_COMPONENT
-    cost_resource_group: gigl_platform
-  common_compute_config:
-    project: "external-snap-ci-github-gigl"
-    region: "us-central1"
-    temp_assets_bucket: "gs://gigl-cicd-temp"
-    temp_regional_assets_bucket: "gs://gigl-cicd-temp"
-    perm_assets_bucket: "gs://gigl-cicd-perm"
-    temp_assets_bq_dataset_name: "gigl_temp_assets"
-    embedding_bq_dataset_name: "gigl_embeddings"
-    gcp_service_account_email: "untrusted-external-github-gigl@external-snap-ci-github-gigl.iam.gserviceaccount.com"
-    dataflow_runner: "DataflowRunner"
-preprocessor_config:
-  edge_preprocessor_config:
-    num_workers: 1
-    max_num_workers: 128
-    machine_type: "n2d-highmem-32"
-    disk_size_gb: 300
-  node_preprocessor_config:
-    num_workers: 1
-    max_num_workers: 128
-    machine_type: "n2d-highmem-64"
-    disk_size_gb: 300
+defaults:
+  - e2e/shared@shared_resource_config
+  - e2e/preprocessor@preprocessor_config
+  - _self_
+
 trainer_resource_config:
   vertex_ai_trainer_config:
     machine_type: n1-highmem-32

docs/user_guide/config_guides/hydra_composition.md

@@ -0,0 +1,120 @@
+# Composing task and resource configs with Hydra
+
+Task and resource configs can use [Hydra Defaults Lists](https://hydra.cc/docs/advanced/defaults_list/) by adding a
+top-level `defaults` list. ConfigValidator composes every source config before publishing plain YAML snapshots.
+`ProtoUtils.read_proto_from_yaml` remains the URI-agnostic, non-composing reader; `ProtoUtils.compose_proto_from_yaml`
+is the URI-agnostic composition API.
+
+## Config root
+
+The primary file's parent directory is its Hydra config root. GiGL does not prevalidate filename extensions; Hydra 1.3
+resolves local config names with its `.yaml` convention. Remote primaries are staged with a `.yaml` extension before
+composition.
+
+```text
+configs/
+├── task.yaml
+├── resource.yaml
+├── shared/
+│   └── directed.yaml
+└── compute/
+    └── local.yaml
+```
+
+Keep primary files directly in the bundle root. GiGL does not search for a repository root or a directory named
+`configs`. Local configs retain their parent as the config root so they can select sibling fragments. GCS and HTTP
+primaries are downloaded to a temporary local file before composition; relative Defaults List entries are not downloaded
+with them. Remote primaries can opt into an existing local config root as described below.
+
+## Remote primary with local shared configs
+
+GCS and HTTPS primaries can select shared configs installed in the runtime image by adding a Hydra search path:
+
+```yaml
+hydra:
+  searchpath:
+    - pkg://my_project.configs
+
+defaults:
+  - shared@_global_: common
+  - _self_
+```
+
+Prefer `pkg://` roots so the same config works across machines and containers. The referenced package and its YAML files
+must be installed in every environment that composes the primary. Hydra removes its own `hydra` metadata from the
+composed mapping before GiGL parses the protobuf.
+
+GiGL does not download Defaults List entries relative to a GCS prefix or HTTPS location. Without an explicit search
+path, a remote primary is composed as a standalone file.
+
+## Task config example
+
+```yaml
+# task.yaml
+defaults:
+  - shared@sharedConfig: directed
+  - _self_
+```
+
+```yaml
+# shared/directed.yaml
+isGraphDirected: true
+```
+
+The package target after `@` places the selected group at the corresponding protobuf field. A fragment containing fields
+for the whole protobuf can use `@_global_`.
+
+Include `_self_` explicitly so it is clear whether values in the primary override group values or are overridden by
+them.
+
+## Resource config example
+
+```yaml
+# resource.yaml
+defaults:
+  - compute@shared_resource_config: local
+  - _self_
+```
+
+```yaml
+# compute/local.yaml
+common_compute_config:
+  project: example-project
+  region: us-central1
+  temp_regional_assets_bucket: gs://example-bucket
+```
+
+KFP submission composes a local resource source once to select the project, region, service account, and staging bucket.
+ConfigValidator composes it again inside the pipeline, and that validation-time result becomes the resource config used
+by downstream components. Resolvers such as `now`, `git_hash`, and `oc.env` are supported; environment-based resolvers
+must be available wherever the source config is composed.
+
+## Pipeline behavior
+
+ConfigValidator composes both primary configs, writes fully resolved plain-protobuf YAML snapshots, initializes its
+runtime from those exact snapshots, and validates them before publishing its outputs. Every downstream pipeline
+component receives those snapshot URIs. ConfigValidator caching is disabled so changing a selected fragment cannot reuse
+stale resolved outputs.
+
+The final composed mapping must still be a valid `GbmlConfig` or `GiglResourceConfig`. Protobuf parsing remains the
+schema and type validation boundary.
+
+## Repository example
+
+GiGL's three E2E resource configs share their infrastructure and preprocessing sections:
+
+- [`e2e_cicd_resource_config.yaml`](../../../deployment/configs/e2e_cicd_resource_config.yaml)
+- [`e2e_glt_resource_config.yaml`](../../../deployment/configs/e2e_glt_resource_config.yaml)
+- [`e2e_glt_gs_resource_config.yaml`](../../../deployment/configs/e2e_glt_gs_resource_config.yaml)
+
+Each primary selects [`e2e/shared.yaml`](../../../deployment/configs/e2e/shared.yaml) and
+[`e2e/preprocessor.yaml`](../../../deployment/configs/e2e/preprocessor.yaml), then defines only its pipeline-specific
+resources. Their names intentionally omit `resource_config` so repository validation does not mistake these partial
+fragments for complete resource configs. The unit-test resource config remains standalone because its buckets, datasets,
+and runner intentionally differ.
+
+## Boundaries
+
+- GiGL does not consume Hydra command-line overrides, multirun, launchers, or output-directory behavior.
+- GiGL does not automatically download config fragments next to a GCS or HTTPS primary.
+- Treat config bundle write access as trusted access. GiGL configs can reference importable classes and commands.

docs/user_guide/config_guides/resource_config_guide.md

@@ -4,6 +4,9 @@ GiGL Resource Config is a yaml file that is passed in at runtime and specifies t
 configurations for each component in the GiGL. The proto definition for GiGL Resource Config can be seen
 [here](https://github.com/Snapchat/GiGL/blob/main/proto/snapchat/research/gbml/gigl_resource_config.proto)
 
+Resource configs can be split into Hydra config groups. See
+[composing task and resource configs with Hydra](./hydra_composition.md).
+
 ## Prerequisites
 
 If you don't have cloud assets already setup i.e. a GCP project. See [guide](../getting_started/cloud_setup_guide.md)

docs/user_guide/config_guides/task_config_guide.md

@@ -3,6 +3,9 @@
 The task config specifies task-related configurations - guiding the behavior of components according to the needs of
 your machine learning task.
 
+Task configs can be split into Hydra config groups. See
+[composing task and resource configs with Hydra](./hydra_composition.md).
+
 Whenever we say "task config" we are talking about an instance off
 {py:class}`snapchat.research.gbml.gbml_config_pb2.GbmlConfig`
 

gigl/common/utils/hydra_config.py

@@ -0,0 +1,72 @@
+"""Hydra composition support for GiGL protobuf YAML configs."""
+
+from __future__ import annotations
+
+import os
+import threading
+from pathlib import Path
+from typing import Any, cast
+
+from hydra import compose, initialize_config_dir
+from hydra.core.global_hydra import GlobalHydra
+from omegaconf import OmegaConf
+
+from gigl.common import LocalUri
+from gigl.common.omegaconf_resolvers import now_resolver, register_resolvers
+
+_COMPOSE_LOCK = threading.RLock()
+
+
+def compose_yaml_config(uri: LocalUri) -> dict[str, Any]:
+    """Compose a YAML config with Hydra using its parent as the config root.
+
+    Args:
+        uri: Primary YAML config URI.
+
+    Returns:
+        A fully composed and resolved mapping.
+
+    Raises:
+        ValueError: If the result is not a mapping.
+        RuntimeError: If another Hydra application owns the global context.
+    """
+    primary_name = uri.get_basename()
+    config_name = primary_name.rsplit(".", 1)[0]
+
+    config_root = Path(uri.uri).absolute().parent
+
+    with _COMPOSE_LOCK:
+        if GlobalHydra.instance().is_initialized():
+            raise RuntimeError(
+                "GiGL cannot compose a config while another Hydra context is active."
+            )
+        try:
+            with initialize_config_dir(
+                config_dir=os.fspath(config_root),
+                job_name="gigl_config",
+                version_base="1.3",
+            ):
+                # Hydra installs a one-argument ``now`` resolver during
+                # initialization. GiGL's resolver is a backward-compatible
+                # superset that also supports offsets.
+                register_resolvers()
+                OmegaConf.register_new_resolver(
+                    "now",
+                    now_resolver,
+                    replace=True,
+                )
+                composed = compose(config_name=config_name, overrides=[])
+                resolved = OmegaConf.to_container(composed, resolve=True)
+        finally:
+            OmegaConf.register_new_resolver(
+                "now",
+                now_resolver,
+                replace=True,
+            )
+
+    if not isinstance(resolved, dict):
+        raise ValueError(
+            f"Hydra config {config_root / config_name} resolved to "
+            f"{type(resolved).__name__}, expected a mapping."
+        )
+    return cast(dict[str, Any], resolved)