continue_config.yml

version: 2.1

parameters:
  client:
    type: boolean
    default: false
  common:
    type: boolean
    default: false
  python:
    type: boolean
    default: false

orbs:
  windows: circleci/windows@5.0

commands:
  halt_unless_core:
    steps:
      - unless:
          condition:
            or:
              - << pipeline.parameters.common >>
              - << pipeline.parameters.python >>
              - equal: [main, << pipeline.git.branch >>]
          steps:
            - run: circleci-agent step halt
  halt_unless_client:
    steps:
      - unless:
          condition:
            or:
              - << pipeline.parameters.common >>
              - << pipeline.parameters.client >>
              - equal: [main, << pipeline.git.branch >>]
          steps:
            - run: circleci-agent step halt

jobs:
  vscode_test:
    docker:
      - image: cimg/node:20.19.1-browsers
    resource_class: small
    steps:
      - checkout
      - run:
          name: Install Dependencies
          command: |
            pnpm install
      - run:
          name: Run VSCode extension CI
          command: |
            cd vscode/extension
            pnpm run ci
  doc_tests:
    docker:
      - image: cimg/python:3.10
    resource_class: small
    steps:
      - halt_unless_core
      - checkout
      - run:
          name: Install dependencies
          command: make install-dev install-doc
      - run:
          name: Run doc tests
          command: make doc-test

  style_and_cicd_tests:
    parameters:
      python_version:
        type: string
    docker:
      - image: cimg/python:<< parameters.python_version >>
    resource_class: large
    environment:
      PYTEST_XDIST_AUTO_NUM_WORKERS: 8
    steps:
      - halt_unless_core
      - checkout
      - run:
          name: Install OpenJDK
          command: sudo apt-get update && sudo apt-get install default-jdk
      - run:
          name: Install ODBC
          command: sudo apt-get install unixodbc-dev
      - run:
          name: Install SQLMesh dev dependencies
          command: make install-dev
      - run:
          name: Fix Git URL override
          command: git config --global --unset url."ssh://git@github.com".insteadOf
      - run:
          name: Run linters and code style checks
          command: make py-style
      - unless:
          condition:
            equal: ["3.9", << parameters.python_version >>]
          steps:
            - run:
                name: Exercise the benchmarks
                command: make benchmark-ci
      - run:
          name: Run cicd tests
          command: make cicd-test
      - store_test_results:
          path: test-results

  cicd_tests_windows:
    executor:
      name: windows/default
      size: large
    steps:
      - halt_unless_core
      - run:
          name: Enable symlinks in git config
          command: git config --global core.symlinks true
      - checkout
      - run:
          name: Install System Dependencies
          command: |
            choco install make which -y
            refreshenv
      - run:
          name: Install SQLMesh dev dependencies
          command: |
            python -m venv venv
            . ./venv/Scripts/activate
            python.exe -m pip install --upgrade pip
            make install-dev
      - run:
          name: Run fast unit tests
          command: |
            . ./venv/Scripts/activate
            which python
            python --version
            make fast-test
      - store_test_results:
          path: test-results

  migration_test:
    docker:
      - image: cimg/python:3.10
    resource_class: small
    environment:
      SQLMESH__DISABLE_ANONYMIZED_ANALYTICS: "1"
    steps:
      - halt_unless_core
      - checkout
      - run:
          name: Run the migration test - sushi
          command: ./.circleci/test_migration.sh sushi "--gateway duckdb_persistent"
      - run:
          name: Run the migration test - sushi_dbt
          command: ./.circleci/test_migration.sh sushi_dbt "--config migration_test_config"

  ui_style:
    docker:
      - image: cimg/node:20.19.0
    resource_class: small
    steps:
      - checkout
      - restore_cache:
          name: Restore pnpm Package Cache
          keys:
            - pnpm-packages-{{ checksum "pnpm-lock.yaml" }}
      - run:
          name: Install Dependencies
          command: |
            pnpm install
      - save_cache:
          name: Save pnpm Package Cache
          key: pnpm-packages-{{ checksum "pnpm-lock.yaml" }}
          paths:
            - .pnpm-store
      - run:
          name: Run linters and code style checks
          command: pnpm run lint

  ui_test:
    docker:
      - image: mcr.microsoft.com/playwright:v1.54.1-jammy
    resource_class: medium
    steps:
      - halt_unless_client
      - checkout
      - restore_cache:
          name: Restore pnpm Package Cache
          keys:
            - pnpm-packages-{{ checksum "pnpm-lock.yaml" }}
      - run:
          name: Install pnpm package manager
          command: |
            npm install --global corepack@latest
            corepack enable
            corepack prepare pnpm@latest-10 --activate
            pnpm config set store-dir .pnpm-store
      - run:
          name: Install Dependencies
          command: |
            pnpm install
      - save_cache:
          name: Save pnpm Package Cache
          key: pnpm-packages-{{ checksum "pnpm-lock.yaml" }}
          paths:
            - .pnpm-store
      - run:
          name: Run tests
          command: npm --prefix web/client run test

  engine_tests_docker:
    parameters:
      engine:
        type: string
    machine:
      image: ubuntu-2404:2024.05.1
      docker_layer_caching: true
    resource_class: large
    environment:
      SQLMESH__DISABLE_ANONYMIZED_ANALYTICS: "1"
    steps:
      - halt_unless_core
      - checkout
      - run:
          name: Install OS-level dependencies
          command: ./.circleci/install-prerequisites.sh "<< parameters.engine >>"
      - run:
          name: Run tests
          command: make << parameters.engine >>-test
          no_output_timeout: 20m
      - store_test_results:
          path: test-results

  engine_tests_cloud:
    parameters:
      engine:
        type: string
    docker:
      - image: cimg/python:3.12
    resource_class: medium
    environment:
      PYTEST_XDIST_AUTO_NUM_WORKERS: 4
      SQLMESH__DISABLE_ANONYMIZED_ANALYTICS: "1"
    steps:
      - halt_unless_core
      - checkout
      - run:
          name: Install OS-level dependencies
          command: ./.circleci/install-prerequisites.sh "<< parameters.engine >>"
      - run:
          name: Generate database name
          command: |
            UUID=`cat /proc/sys/kernel/random/uuid`
            TEST_DB_NAME="circleci_${UUID:0:8}"
            echo "export TEST_DB_NAME='$TEST_DB_NAME'" >> "$BASH_ENV"
            echo "export SNOWFLAKE_DATABASE='$TEST_DB_NAME'" >> "$BASH_ENV"
            echo "export DATABRICKS_CATALOG='$TEST_DB_NAME'" >> "$BASH_ENV"
            echo "export REDSHIFT_DATABASE='$TEST_DB_NAME'" >> "$BASH_ENV"
            echo "export GCP_POSTGRES_DATABASE='$TEST_DB_NAME'" >> "$BASH_ENV"
            echo "export FABRIC_DATABASE='$TEST_DB_NAME'" >> "$BASH_ENV"

            # Make snowflake private key available            
            echo $SNOWFLAKE_PRIVATE_KEY_RAW | base64 -d > /tmp/snowflake-keyfile.p8
            echo "export SNOWFLAKE_PRIVATE_KEY_FILE='/tmp/snowflake-keyfile.p8'" >> "$BASH_ENV"
      - run:
          name: Create test database
          command: ./.circleci/manage-test-db.sh << parameters.engine >> "$TEST_DB_NAME" up
      - run:
          name: Run tests
          command: |
            make << parameters.engine >>-test
          no_output_timeout: 20m
      - run:
          name: Tear down test database
          command: ./.circleci/manage-test-db.sh << parameters.engine >> "$TEST_DB_NAME" down
          when: always
      - store_test_results:
          path: test-results

workflows:
  main_pr:
    jobs:
      - doc_tests
      - style_and_cicd_tests:
          matrix:
            parameters:
              python_version:
                - "3.9"
                - "3.10"
                - "3.11"
                - "3.12"
                - "3.13"
      - cicd_tests_windows
      - engine_tests_docker:
          name: engine_<< matrix.engine >>
          matrix:
            parameters:
              engine:
                - duckdb
                - postgres
                - mysql
                - mssql
                - trino
                - spark
                - clickhouse
                - risingwave
                - starrocks
      - engine_tests_cloud:
          name: cloud_engine_<< matrix.engine >>
          context:
            - sqlmesh_cloud_database_integration
          requires:
            - engine_tests_docker
          matrix:
            parameters:
              engine:
                - snowflake
                - databricks
                - redshift
                - bigquery
                - clickhouse-cloud
                - athena
                - fabric
                - gcp-postgres
          filters:
            branches:
              only:
                - main
      - ui_style
      - ui_test
      - vscode_test
      - migration_test