diff --git a/docs/user_guide/getting_started/cloud_setup_guide.md b/docs/user_guide/getting_started/cloud_setup_guide.md index 0a3f420dd..5ab8e9bf6 100644 --- a/docs/user_guide/getting_started/cloud_setup_guide.md +++ b/docs/user_guide/getting_started/cloud_setup_guide.md @@ -143,6 +143,24 @@ gcloud storage buckets add-iam-policy-binding $BUCKET_NAME \ 11. Give your SA `roles/bigquery.dataOwner` on the datasets you created. See [instructions](https://cloud.google.com/bigquery/docs/control-access-to-resources-iam#bq_2). +### Console log format + +On App Engine and Kubernetes, `gigl.common.logger.Logger` routes records to Google Cloud Logging, which on GKE renders +each one as a single-line JSON envelope. To get the plain console format instead, set: + +```bash +export GIGL_DISABLE_CLOUD_LOGGING="1" +``` + +`1` and `true` (case-insensitive) disable cloud logging; anything else leaves it on. + +This is worth setting on processes whose stdout is relayed by another system before it reaches Cloud Logging. Ray is the +motivating case: it prefixes each worker line it forwards to the driver with `(RayTrainWorker pid=...)`, which makes the +line invalid JSON, so Cloud Logging stores it as a plain `textPayload` and the structured fields (`severity`, +`sourceLocation`, trace correlation) are dropped anyway. The variable is read from the environment so that processes +started with the `spawn` method — which inherit `os.environ` but not the parent's logging configuration — pick it up +without each entry point having to opt in. + ## AWS Project Setup Guide - Not yet supported diff --git a/gigl/common/logger.py b/gigl/common/logger.py index 67184b7e2..110eefb72 100644 --- a/gigl/common/logger.py +++ b/gigl/common/logger.py @@ -6,7 +6,11 @@ from google.cloud import logging as google_cloud_logging -from gigl.env.constants import GIGL_DEBUG, is_env_flag_enabled +from gigl.env.constants import ( + GIGL_DEBUG, + GIGL_DISABLE_CLOUD_LOGGING_ENV_KEY, + is_env_flag_enabled, +) _BASE_LOG_FILE_PATH = "/tmp/research/gbml/logs" @@ -14,6 +18,11 @@ class Logger(logging.LoggerAdapter): """ GiGL's custom logger class used for local and cloud logging (VertexAI, Dataflow, etc.) + + On App Engine and Kubernetes, records are routed to Google Cloud Logging, which + renders them as GCP JSON. Set ``GIGL_DISABLE_CLOUD_LOGGING`` to fall back to the + console format. + Args: logger (Optional[logging.Logger]): A custom logger to use. If not provided, the default logger will be created. name (Optional[str]): The name to be used for the logger. By default uses "root". @@ -56,8 +65,12 @@ def _setup_logger( ) -> None: handler: logging.Handler if not logger.handlers: - if os.getenv("GAE_APPLICATION") or os.environ.get( - "KUBERNETES_SERVICE_HOST" + is_cloud_environment = bool( + os.getenv("GAE_APPLICATION") + or os.environ.get("KUBERNETES_SERVICE_HOST") + ) + if is_cloud_environment and not is_env_flag_enabled( + GIGL_DISABLE_CLOUD_LOGGING_ENV_KEY ): # Google Cloud Logging client = google_cloud_logging.Client() diff --git a/gigl/env/constants.py b/gigl/env/constants.py index 425bce1b6..e471c181c 100644 --- a/gigl/env/constants.py +++ b/gigl/env/constants.py @@ -23,6 +23,7 @@ GIGL_CUDA_DOCKER_URI_ENV_KEY: Final[str] = "GIGL_CUDA_DOCKER_URI" GIGL_COMPONENT_ENV_KEY: Final[str] = "GIGL_COMPONENT" GIGL_BIGQUERY_QUOTA_PROJECT_ENV_KEY: Final[str] = "GIGL_BIGQUERY_QUOTA_PROJECT" +GIGL_DISABLE_CLOUD_LOGGING_ENV_KEY: Final[str] = "GIGL_DISABLE_CLOUD_LOGGING" GIGL_ENABLE_PERF_MONITORING: Final[str] = "GIGL_ENABLE_PERF_MONITORING" GIGL_DEBUG: Final[str] = "GIGL_DEBUG" diff --git a/tests/unit/common/test_logger.py b/tests/unit/common/logger_test.py similarity index 52% rename from tests/unit/common/test_logger.py rename to tests/unit/common/logger_test.py index 2ff4a7de9..09a156f38 100644 --- a/tests/unit/common/test_logger.py +++ b/tests/unit/common/logger_test.py @@ -1,5 +1,8 @@ import copy +import logging +import os import pickle +from unittest import mock from absl.testing import absltest @@ -33,6 +36,35 @@ def test_pickle_round_trips(self) -> None: self.assertIsInstance(clone, Logger) clone.info("pickle works") + def test_disabling_cloud_logging_uses_console_handler(self) -> None: + # KUBERNETES_SERVICE_HOST alone would send records to Google Cloud Logging as + # GCP JSON; GIGL_DISABLE_CLOUD_LOGGING must win and give the console format. + with mock.patch.dict( + os.environ, + { + "KUBERNETES_SERVICE_HOST": "10.0.0.1", + "GIGL_DISABLE_CLOUD_LOGGING": "1", + }, + ): + logger = Logger(name="test_cloud_logging_disabled") + + handlers = logger.logger.handlers + self.assertEqual(len(handlers), 1) + self.assertIsInstance(handlers[0], logging.StreamHandler) + record = logging.LogRecord( + name="test_cloud_logging_disabled", + level=logging.INFO, + pathname="export.py", + lineno=197, + msg="upload took 8.82 seconds", + args=None, + exc_info=None, + func="_flush", + ) + formatted = handlers[0].format(record) + self.assertIn("[INFO] : upload took 8.82 seconds", formatted) + self.assertIn("(export.py:_flush:197)", formatted) + if __name__ == "__main__": absltest.main()