Skip to content

Commit 0dcf726

Browse files
chore: parameterize project ID in dataset and add JSON file to environment substitution script
1 parent 8810634 commit 0dcf726

2 files changed

Lines changed: 6 additions & 5 deletions

File tree

evals/dataset.json

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -2,28 +2,28 @@
22
"scenarios": [
33
{
44
"id": "bq-search-and-insight",
5-
"starting_prompt": "Find tables related to sales in project ext-test-bigquery-analytics.",
5+
"starting_prompt": "Find tables related to sales in project ${GOOGLE_CLOUD_PROJECT}.",
66
"conversation_plan": "First, ask the agent to find tables related to sales. Once it lists the tables (which should include 'sales_data' in 'evalbench_ci'), ask it to identify the top product by sales in that table.",
77
"expected_trajectory": [
88
"search_catalog",
99
"ask_data_insights"
1010
],
1111
"env": {
12-
"GOOGLE_CLOUD_PROJECT": "ext-test-bigquery-analytics"
12+
"GOOGLE_CLOUD_PROJECT": "${GOOGLE_CLOUD_PROJECT}"
1313
},
1414
"kind": "tools",
1515
"max_turns": 4
1616
},
1717
{
1818
"id": "bq-insight-and-forecast",
19-
"starting_prompt": "What are the top products by sales in the table 'sales_data' in dataset 'evalbench_ci' and project 'ext-test-bigquery-analytics'?",
19+
"starting_prompt": "What are the top products by sales in the table 'sales_data' in dataset 'evalbench_ci' and project '${GOOGLE_CLOUD_PROJECT}'?",
2020
"conversation_plan": "First, ask the agent to find the top products by sales in the sales_data table. After it identifies the top products, ask it to forecast the sales for the top product for the next 5 steps.",
2121
"expected_trajectory": [
2222
"ask_data_insights",
2323
"forecast"
2424
],
2525
"env": {
26-
"GOOGLE_CLOUD_PROJECT": "ext-test-bigquery-analytics"
26+
"GOOGLE_CLOUD_PROJECT": "${GOOGLE_CLOUD_PROJECT}"
2727
},
2828
"kind": "tools",
2929
"max_turns": 4

evals/substitute_env.py

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -6,7 +6,8 @@ def main():
66
workspace = os.environ.get('EVAL_WORKSPACE', '/workspace')
77
yaml_paths = [
88
os.path.join(workspace, 'evals/model_config.yaml'),
9-
os.path.join(workspace, 'evals/run_config.yaml')
9+
os.path.join(workspace, 'evals/run_config.yaml'),
10+
os.path.join(workspace, 'evals/dataset.json')
1011
]
1112

1213
for yaml_path in yaml_paths:

0 commit comments

Comments
 (0)