efficientAI/.github/workflows/backend-tests-postgres.yml at main · EfficientAI-tech/efficientAI · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
name: Backend Tests (Postgres)

on:
  pull_request:
    branches:
      - main
      - master
  push:
    branches:
      - main
      - master

jobs:
  postgres-test:
    runs-on: ubuntu-latest
    timeout-minutes: 30

    services:
      postgres:
        image: postgres:16
        env:
          POSTGRES_USER: postgres
          POSTGRES_PASSWORD: postgres
          POSTGRES_DB: efficientai_test
        ports:
          - 5432:5432
        options: >-
          --health-cmd "pg_isready -U postgres -d efficientai_test"
          --health-interval 10s
          --health-timeout 5s
          --health-retries 10

    env:
      DATABASE_URL: postgresql://postgres:postgres@localhost:5432/efficientai_test
      TEST_DATABASE_URL: postgresql://postgres:postgres@localhost:5432/efficientai_test
      POSTGRES_USER: postgres
      POSTGRES_PASSWORD: postgres
      POSTGRES_DB: efficientai_test
      POSTGRES_HOST: localhost
      POSTGRES_PORT: 5432
      REDIS_URL: redis://localhost:6379/0
      CELERY_BROKER_URL: redis://localhost:6379/0
      CELERY_RESULT_BACKEND: redis://localhost:6379/0

    steps:
      - name: Checkout repository
        uses: actions/checkout@v4

      - name: Set up Python and install dependencies
        uses: ./.github/actions/setup-python-deps

      - name: Run backend test suite on Postgres
        run: make test-parallel PYTEST_ARGS="--durations=25"

  db-sharding:
    runs-on: ubuntu-latest
    timeout-minutes: 20

    services:
      postgres:
        image: postgres:16
        env:
          POSTGRES_USER: postgres
          POSTGRES_PASSWORD: postgres
          POSTGRES_DB: efficientai_test
        ports:
          - 5432:5432
        options: >-
          --health-cmd "pg_isready -U postgres -d efficientai_test"
          --health-interval 10s
          --health-timeout 5s
          --health-retries 10

    env:
      PGHOST: localhost
      PGPORT: 5432
      PGUSER: postgres
      PGPASSWORD: postgres
      DATABASE_URL: postgresql://postgres:postgres@localhost:5432/efficientai_catalog
      TEST_DATABASE_URL: postgresql://postgres:postgres@localhost:5432/efficientai_test
      CATALOG_DATABASE_URL: postgresql://postgres:postgres@localhost:5432/efficientai_catalog
      SHARD_DATABASE_URL_01: postgresql://postgres:postgres@localhost:5432/efficientai_data_01
      SHARD_DATABASE_URL_02: postgresql://postgres:postgres@localhost:5432/efficientai_data_02
      SHARDING_INTEGRATION_TEST: "1"
      REDIS_URL: redis://localhost:6379/0

    steps:
      - name: Checkout repository
        uses: actions/checkout@v4

      - name: Set up Python and install dependencies
        uses: ./.github/actions/setup-python-deps

      - name: Create catalog and row-shard databases
        run: |
          for db in efficientai_catalog efficientai_data_01 efficientai_data_02; do
            psql -d postgres -v ON_ERROR_STOP=1 -c "DROP DATABASE IF EXISTS ${db};"
            psql -d postgres -v ON_ERROR_STOP=1 -c "CREATE DATABASE ${db};"
          done

      - name: Run db_sharding unit tests
        run: make test-sharding PYTEST_ARGS="--durations=10"

      - name: Run sharding Postgres integration tests
        run: make test-sharding-integration PYTEST_ARGS="--durations=10"