Skip to content

Commit 1dafd1f

Browse files
jeffreyavenjeffreyaven
committed
databricks example
1 parent fa6cc01 commit 1dafd1f

15 files changed

Lines changed: 494 additions & 217 deletions

File tree

‎README.md‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -68,6 +68,8 @@ examples/databricks/serverless dev \
6868
-e AWS_ACCOUNT_ID=${AWS_ACCOUNT_ID} \
6969
-e DATABRICKS_ACCOUNT_ID=${DATABRICKS_ACCOUNT_ID} \
7070
-e DATABRICKS_AWS_ACCOUNT_ID=${DATABRICKS_AWS_ACCOUNT_ID} \
71+
--show-queries \
72+
--log-level debug \
7173
--dry-run
7274

7375
pgrep -f "stackql srv"

examples/databricks/serverless/resources/aws/iam/policy_statements/ec2_permissions.json renamed to examples/databricks/serverless/resources/aws/iam/policy_statements/cross_account_role/ec2_permissions.json

File renamed without changes.

examples/databricks/serverless/resources/aws/iam/policy_statements/iam_service_linked_role.json renamed to examples/databricks/serverless/resources/aws/iam/policy_statements/cross_account_role/iam_service_linked_role.json

File renamed without changes.

‎examples/databricks/serverless/resources/aws/iam/policy_statements/metastore_access_role/kms_decrypt.json‎

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,11 @@
1+
{
2+
"Effect": "Allow",
3+
"Action": [
4+
"kms:Decrypt",
5+
"kms:Encrypt",
6+
"kms:GenerateDataKey*"
7+
],
8+
"Resource": [
9+
"arn:aws:kms:<KMS-KEY>"
10+
]
11+
}

‎examples/databricks/serverless/resources/aws/iam/policy_statements/metastore_access_role/managed_file_events_list.json‎

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,13 @@
1+
{
2+
"Sid": "ManagedFileEventsListStatement",
3+
"Effect": "Allow",
4+
"Action": [
5+
"sqs:ListQueues",
6+
"sqs:ListQueueTags",
7+
"sns:ListTopics"
8+
],
9+
"Resource": [
10+
"arn:aws:sqs:*:*:csms-*",
11+
"arn:aws:sns:*:*:csms-*"
12+
]
13+
}

‎examples/databricks/serverless/resources/aws/iam/policy_statements/metastore_access_role/managed_file_events_setup.json‎

Lines changed: 30 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,30 @@
1+
{
2+
"Sid": "ManagedFileEventsSetupStatement",
3+
"Effect": "Allow",
4+
"Action": [
5+
"s3:GetBucketNotification",
6+
"s3:PutBucketNotification",
7+
"sns:ListSubscriptionsByTopic",
8+
"sns:GetTopicAttributes",
9+
"sns:SetTopicAttributes",
10+
"sns:CreateTopic",
11+
"sns:TagResource",
12+
"sns:Publish",
13+
"sns:Subscribe",
14+
"sqs:CreateQueue",
15+
"sqs:DeleteMessage",
16+
"sqs:ReceiveMessage",
17+
"sqs:SendMessage",
18+
"sqs:GetQueueUrl",
19+
"sqs:GetQueueAttributes",
20+
"sqs:SetQueueAttributes",
21+
"sqs:TagQueue",
22+
"sqs:ChangeMessageVisibility",
23+
"sqs:PurgeQueue"
24+
],
25+
"Resource": [
26+
"{{ aws_s3_metastore_bucket.arn }}",
27+
"arn:aws:sqs:*:*:csms-*",
28+
"arn:aws:sns:*:*:csms-*"
29+
]
30+
}

‎examples/databricks/serverless/resources/aws/iam/policy_statements/metastore_access_role/managed_file_events_teardown.json‎

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,13 @@
1+
{
2+
"Sid": "ManagedFileEventsTeardownStatement",
3+
"Effect": "Allow",
4+
"Action": [
5+
"sns:Unsubscribe",
6+
"sns:DeleteTopic",
7+
"sqs:DeleteQueue"
8+
],
9+
"Resource": [
10+
"arn:aws:sqs:*:*:csms-*",
11+
"arn:aws:sns:*:*:csms-*"
12+
]
13+
}

‎examples/databricks/serverless/resources/aws/iam/policy_statements/metastore_access_role/s3_access.json‎

Lines changed: 17 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,17 @@
1+
{
2+
"Effect": "Allow",
3+
"Action": [
4+
"s3:GetObject",
5+
"s3:PutObject",
6+
"s3:DeleteObject",
7+
"s3:ListBucket",
8+
"s3:GetBucketLocation",
9+
"s3:ListBucketMultipartUploads",
10+
"s3:ListMultipartUploadParts",
11+
"s3:AbortMultipartUpload"
12+
],
13+
"Resource": [
14+
"{{ aws_s3_metastore_bucket.arn }}/*",
15+
"{{ aws_s3_metastore_bucket.arn }}"
16+
]
17+
}

‎examples/databricks/serverless/resources/aws/iam/policy_statements/metastore_access_role/sts_assume_role.json‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,9 @@
1+
{
2+
"Effect": "Allow",
3+
"Action": [
4+
"sts:AssumeRole"
5+
],
6+
"Resource": [
7+
"arn:aws:iam::{{ databricks_aws_account_id }}:role/{{ stack_name }}-{{ stack_env }}-metastore-role"
8+
]
9+
}

‎examples/databricks/serverless/resources/aws/s3/bucket_policies.iql‎

Lines changed: 44 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,44 @@
1+
/*+ exists */
2+
SELECT count(*) as count
3+
FROM awscc.s3.bucket_policies
4+
WHERE region = '{{ region }}' AND
5+
Identifier = '{{ bucket_name }}'
6+
;
7+
8+
/*+ create */
9+
INSERT INTO awscc.s3.bucket_policies (
10+
Bucket,
11+
PolicyDocument,
12+
region
13+
)
14+
SELECT
15+
'{{ bucket_name }}',
16+
'{{ policy_document }}',
17+
'{{ region }}';
18+
19+
/*+ update */
20+
UPDATE awscc.s3.bucket_policies
21+
SET PatchDocument = string('{{ {
22+
"PolicyDocument": policy_document
23+
} | generate_patch_document }}')
24+
WHERE
25+
region = '{{ region }}' AND
26+
Identifier = '{{ bucket_name }}';
27+
28+
/*+ statecheck, retries=5, retry_delay=10 */
29+
SELECT
30+
count(*) as count
31+
FROM (
32+
SELECT
33+
AWS_POLICY_EQUAL(policy_document, '{{ policy_document }}') as test_policy_document
34+
FROM awscc.s3.bucket_policies
35+
WHERE region = '{{ region }}'
36+
AND Identifier = '{{ bucket_name }}'
37+
)t
38+
WHERE test_policy_document = 1;
39+
40+
/*+ delete */
41+
DELETE FROM awscc.s3.bucket_policies
42+
WHERE
43+
Identifier = '{{ bucket_name }}' AND
44+
region = '{{ region }}';

0 commit comments

Comments
 (0)