unittest.yml

on:
  pull_request:
    branches:
      - main
      - preview
  # Trigger workflow on GitHub merge queue events
  # See https://docs.github.com/en/actions/using-workflows/events-that-trigger-workflows#merge_group
  merge_group:
    types: [checks_requested]
name: unittest

permissions:
  contents: read

concurrency:
  group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }}
  cancel-in-progress: true

env:
  PACKAGE_DIRS: packages preview-packages
  NOX_REUSE_EXISTING_VIRTUALENVS: "true"
  NOX_ENVDIR: "/tmp/shared_nox_envs"
  NOX_DEFAULT_VENV_BACKEND: "uv"
  UV_VENV_SEED: "1"
  UV_PRERELEASE: "allow"
  # Note: PARALLEL_WORKERS must remain "1" for unit tests because running pytest concurrently
  # on the same runner VM can cause socket/IPC/process deadlocks across packages.
  PARALLEL_WORKERS: "1"

jobs:
  initialize:
    runs-on: ubuntu-latest
    outputs:
      matrix: ${{ steps.set-matrix.outputs.matrix }}
      packages: ${{ steps.set-matrix.outputs.packages }}
      is_full_run: ${{ steps.check-label.outputs.is_full_run }}
    env:
      MAX_SHARDS: 16
      # Define weights for long-running unit tests to balance shard execution time
      # Each weight is roughly 1 minute of expected execution time
      # Default for unset packages is 1
      PACKAGE_WEIGHTS: |
        django-google-spanner: 2
        gapic-generator: 3
        google-ai-generativelanguage: 3
        google-api-core: 2
        google-auth: 2
        google-cloud-bigquery: 4
        google-cloud-bigquery-storage: 2
        google-cloud-bigtable: 4
        google-cloud-compute: 12
        google-cloud-compute-v1beta: 12
        google-cloud-datastore: 3
        google-cloud-dialogflow: 4
        google-cloud-dialogflow-cx: 4
        google-cloud-discoveryengine: 4
        google-cloud-firestore: 3
        google-cloud-logging: 3
        google-cloud-monitoring: 3
        google-cloud-ndb: 2
        google-cloud-pubsub: 2
        google-cloud-retail: 3
        google-cloud-spanner: 4
        google-cloud-storage: 4
        google-shopping-merchant-accounts: 3
        pandas-gbq: 2
        proto-plus: 1
        sqlalchemy-bigquery: 2
        sqlalchemy-spanner: 2
    steps:
    - name: Checkout
      uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4
      with:
        fetch-depth: 0
        filter: blob:none
        persist-credentials: false
    - name: Check for unit_test:all_packages label
      id: check-label
      run: |
        if [[ "${{ contains(github.event.pull_request.labels.*.name, 'unit_test:all_packages') }}" == "true" ]]; then
          echo "is_full_run=true" >> $GITHUB_OUTPUT
        else
          echo "is_full_run=false" >> $GITHUB_OUTPUT
        fi
    - name: Setup Python
      uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5
      with:
        python-version: "3.10"
    - name: Get package shards
      id: set-matrix
      env:
        BUILD_TYPE: presubmit
        TARGET_BRANCH: ${{ github.base_ref || github.event.merge_group.base_ref }}
        TEST_ALL_PACKAGES: ${{ steps.check-label.outputs.is_full_run }}
        MAX_SHARDS: ${{ env.MAX_SHARDS }}
        PACKAGE_WEIGHTS: ${{ env.PACKAGE_WEIGHTS }}
      run: |
        if [ -n "$TARGET_BRANCH" ]; then
          git fetch --no-tags --quiet origin "$TARGET_BRANCH" || true
        fi
        python3 ci/get_package_shards.py

  unit:
    needs: initialize
    if: needs.initialize.outputs.matrix != '[]' && needs.initialize.outputs.matrix != ''
    runs-on: ubuntu-22.04
    strategy:
      fail-fast: true
      matrix:
        python: ["3.10", "3.11", "3.12", "3.13", "3.14", "3.15"]
        package_shard: ${{ fromJson(needs.initialize.outputs.matrix) }}
    name: ${{ matrix.package_shard.is_sharded && format('unit ({0}, {1})', matrix.python, matrix.package_shard.name) || format('unit ({0})', matrix.python) }}
    steps:
    - name: Checkout
      uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3
      # Use a fetch-depth of 2 to avoid error `fatal: origin/main...HEAD: no merge base`
      # See https://github.com/googleapis/google-cloud-python/issues/12013
      # and https://github.com/actions/checkout#checkout-head.
      with:
        fetch-depth: 2
        persist-credentials: false
    - name: Setup Python
      uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6
      with:
        python-version: ${{ matrix.python }}
        cache: 'pip'
        allow-prereleases: true
    - name: Setup uv
      uses: astral-sh/setup-uv@d4b2f3b6ecc6e67c4457f6d3e41ec42d3d0fcb86 # v5
      with:
        enable-cache: true
        cache-dependency-glob: 'packages/**/testing/constraints*.txt'
    - name: Install nox and nox-uv
      run: |
        uv pip install --system nox nox-uv
    # Caches local wheels for Python 3.15+ to avoid repeatedly downloading or building heavy libraries.
    # Follow https://github.com/grpc/grpc/issues/41010 for updates.
    - name: Cache Built Python 3.15 Wheels
      if: matrix.python == '3.15'
      uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4
      with:
        path: .uv-wheel-cache
        key: ${{ runner.os }}-uv-wheels-3.15-${{ hashFiles('packages/**/testing/constraints*.txt') }}
        restore-keys: |
          ${{ runner.os }}-uv-wheels-3.15-
    # TODO: Remove this step once official Python 3.15 wheels for grpcio are published to PyPI.
    # See https://github.com/grpc/grpc/issues/41010#issuecomment-4848264910 for details.
    - name: Install grpcio wheel for Python 3.15
      if: matrix.python == '3.15'
      run: |
        mkdir -p .uv-wheel-cache
        WHEEL_FILE=".uv-wheel-cache/grpcio-1.83.0.dev0-cp315-cp315-manylinux2014_x86_64.manylinux_2_17_x86_64.whl"
        if [ ! -f "$WHEEL_FILE" ]; then
          curl -L "https://packages.grpc.io/archive/2026/06/5ccdab32010f762634e1082d19be53cf60458456-b37e49c7-5e62-4cbe-870a-c599bb907b04/python/grpcio/grpcio-1.83.0.dev0-cp315-cp315-manylinux2014_x86_64.manylinux_2_17_x86_64.whl" -o "$WHEEL_FILE"
        fi
        echo "d7706633cd5d97a8b8d3c6ed36c5940722a9b5cbb47f0d10bc433ee27af36ff0  $WHEEL_FILE" | sha256sum -c
        uv pip install --system "$WHEEL_FILE"
        echo "UV_FIND_LINKS=${{ github.workspace }}/.uv-wheel-cache" >> $GITHUB_ENV
        echo "UV_CACHE_DIR=${{ github.workspace }}/.uv-wheel-cache" >> $GITHUB_ENV
        echo "UV_PRERELEASE=allow" >> $GITHUB_ENV
    - name: Run unit tests for ${{ matrix.package_shard.description }}
      env:
        BUILD_TYPE: presubmit
        TARGET_BRANCH: ${{ github.base_ref || github.event.merge_group.base_ref }}
        TEST_TYPE: unit
        PY_VERSION: ${{ matrix.python }}
        PACKAGE_LIST: ${{ matrix.package_shard.packages }}
      run: |
        ci/run_conditional_tests.sh
    - name: Upload coverage results
      uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4
      with:
        name: coverage-artifact-${{ matrix.python }}-${{ matrix.package_shard.index }}
        path: .coverage.${{ matrix.python }}.*
        include-hidden-files: true

  core-deps:
    needs: initialize
    if: needs.initialize.outputs.matrix != '[]' && needs.initialize.outputs.matrix != ''
    runs-on: ubuntu-22.04
    strategy:
      fail-fast: true
      matrix:
        python: ["3.14"]
        package_shard: ${{ fromJson(needs.initialize.outputs.matrix) }}
    name: ${{ matrix.package_shard.is_sharded && format('core-deps handwritten ({0}, {1})', matrix.python, matrix.package_shard.name) || format('core-deps handwritten ({0})', matrix.python) }}
    steps:
    - name: Checkout
      uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3
      # Use a fetch-depth of 2 to avoid error `fatal: origin/main...HEAD: no merge base`
      # See https://github.com/googleapis/google-cloud-python/issues/12013
      # and https://github.com/actions/checkout#checkout-head.
      with:
        fetch-depth: 2
        persist-credentials: false
    - name: Setup Python
      uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6
      with:
        python-version: ${{ matrix.python }}
        cache: 'pip'
        allow-prereleases: true
    - name: Install nox
      run: |
        pip install nox
    - name: Run core_deps_from_source for ${{ matrix.package_shard.description }}
      env:
        BUILD_TYPE: presubmit
        TARGET_BRANCH: ${{ github.base_ref || github.event.merge_group.base_ref }}
        TEST_TYPE: core_deps_from_source
        PY_VERSION: ${{ matrix.python }}
        PACKAGE_LIST: ${{ matrix.package_shard.packages }}
        NOX_DEFAULT_VENV_BACKEND: "virtualenv"
        NOXFORCEPYTHON: ${{ matrix.python }}
      run: |
        ci/run_conditional_tests.sh

  all-tests:
    needs: [initialize, unit, core-deps]
    if: always()
    runs-on: ubuntu-latest
    steps:
      - name: Check test results
        run: |
          # 1. Check initialize job
          if [[ "${{ needs.initialize.result }}" != "success" ]]; then
            echo "Error: The initialize job status was: ${{ needs.initialize.result }}"
            exit 1
          fi
          # 2. Check unit test shards
          if [[ "${{ needs.unit.result }}" != "success" && "${{ needs.unit.result }}" != "skipped" ]]; then
            echo "Unit tests failed"
            exit 1
          fi
          # 3. Check core dependencies test shards
          if [[ "${{ needs.core-deps.result }}" != "success" && "${{ needs.core-deps.result }}" != "skipped" ]]; then
            echo "Core dependencies tests failed"
            exit 1
          fi
          echo "All unit and core dependencies tests passed or were skipped"

  cover:
    if: always() && !cancelled() && needs.all-tests.result == 'success'
    runs-on: ubuntu-latest
    needs:
        - all-tests
        - initialize
    steps:
    - name: Checkout
      uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4.3.1
      # Use a fetch-depth of 2 to avoid error `fatal: origin/main...HEAD: no merge base`
      # See https://github.com/googleapis/google-cloud-python/issues/12013
      # and https://github.com/actions/checkout#checkout-head.
      with:
        fetch-depth: 2
        persist-credentials: false
    - name: Setup Python
      uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5
      with:
        python-version: "3.10"
    - name: Install coverage
      if: ${{ needs.initialize.outputs.packages != '' }}
      run: |
        python -m pip install --upgrade setuptools pip wheel
        python -m pip install coverage
    - name: Download coverage results
      if: ${{ needs.initialize.outputs.packages != '' }}
      uses: actions/download-artifact@634f93cb2916e3fdff6788551b99b062d0335ce0 # v5
      with:
        path: /dev/shm/.coverage-results/
        merge-multiple: true
    - name: Report coverage results
      env:
        # TODO: default to 100% coverage after next gapic-generator release
        # https://github.com/googleapis/google-cloud-python/issues/17459
        DEFAULT_FAIL_UNDER: 99
        TEST_ALL_PACKAGES: ${{ needs.initialize.outputs.is_full_run }}
        PACKAGE_LIST: ${{ needs.initialize.outputs.packages }}
        TARGET_BRANCH: ${{ github.base_ref || github.event.merge_group.base_ref }}
        BUILD_TYPE: presubmit
      run: |
        ci/report_coverage.sh