Skip to content

Commit 034d8ba

Browse files
alza-bitzalza-bitz
committed
Working notebook, needs tidying up
Also includes devcontainer config
1 parent 706efbc commit 034d8ba

7 files changed

Lines changed: 185 additions & 3 deletions

File tree

‎.devcontainer/Dockerfile‎

Lines changed: 32 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,32 @@
1+
ARG BASE_IMAGE=temurin-21-tools-deps-jammy
2+
FROM clojure:${BASE_IMAGE}
3+
4+
ENV USERNAME=vscode
5+
ARG USER_UID=1000
6+
ARG USER_GID=$USER_UID
7+
8+
# Create the user
9+
RUN groupadd --gid $USER_GID $USERNAME \
10+
&& useradd --uid $USER_UID --gid $USER_GID -m $USERNAME -s /bin/bash \
11+
#
12+
# [Optional] Add sudo support. Omit if you don't need to install software after connecting.
13+
&& apt-get update \
14+
&& apt-get install -y sudo \
15+
&& echo $USERNAME ALL=\(root\) NOPASSWD:ALL > /etc/sudoers.d/$USERNAME \
16+
&& chmod 0440 /etc/sudoers.d/$USERNAME
17+
18+
19+
# [Optional] Set the default user. Omit if you want to keep the default as root.
20+
USER $USERNAME
21+
SHELL ["/bin/bash", "-ec"]
22+
ENTRYPOINT ["bash"]
23+
24+
25+
# Prepare clojure tools
26+
RUN clojure -Ttools list && \
27+
clojure -Ttools install io.github.seancorfield/clj-new '{:git/tag "v1.2.404" :git/sha "d4a6508"}' :as clj-new && \
28+
clojure -Ttools install-latest :lib io.github.seancorfield/deps-new :as new && \
29+
clojure -Ttools list
30+
31+
RUN sudo apt-get update && \
32+
sudo apt-get install -y lsb-release

‎.devcontainer/devcontainer.json‎

Lines changed: 27 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,27 @@
1+
// For format details, see https://aka.ms/devcontainer.json. For config options, see the
2+
// README at: https://github.com/scicloj/devcontainer-templates/tree/main/src/basecloj
3+
{
4+
"name": "Base clojure dev env",
5+
"build": {
6+
"dockerfile": "Dockerfile",
7+
"args": {
8+
"BASE_IMAGE": "temurin-21-tools-deps-jammy",
9+
"USERNAME": "${localEnv:USER:vscode}"
10+
}
11+
},
12+
"remoteUser": "${localEnv:USER}",
13+
"containerUser": "${localEnv:USER}",
14+
"features": {
15+
"ghcr.io/devcontainers/features/git:1": {},
16+
"ghcr.io/devcontainers-extra/features/apt-get-packages:1": {
17+
"packages": "bash-completion,libxrender1,libxtst6,libxi6" // xlibs required to load tech.v3.libs.buffered-image
18+
}
19+
},
20+
"customizations": {
21+
"vscode": {
22+
"extensions": [
23+
"betterthantomorrow.calva"
24+
]
25+
}
26+
}
27+
}

‎.gitignore‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -35,3 +35,4 @@ _site/
3535
.calva/repl.calva-repl
3636
.calva/mcp-server/port
3737
.portal/vs-code.edn
38+
**/password.edn

‎deps.edn‎

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -26,10 +26,12 @@
2626
{;; Build the site with `clojure -M:clay -A:markdown`
2727
;; Run Clay in watch mode with `clojure -M:clay`
2828
:clay {:main-opts ["-m" "scicloj.clay.v2.main"]
29-
:jvm-opts ["-Dclojure.main.report=stderr"]}
29+
:jvm-opts ["-Dclojure.main.report=stderr"
30+
"--add-opens=java.base/java.nio=ALL-UNNAMED"]}
3031
;; When debugging libraries
3132
:local-deps {:override-deps {org.scicloj/clay {:local/root "../clay"}
3233
org.scicloj/kindly {:local/root "../kindly"}
3334
org.scicloj/kindly-advice {:local/root "../kindly-advice"}
3435
org.scicloj/kindly-render {:local/root "../kindly-render"}}}
35-
:neil {:project {:name io.github.timothypratley/clojurecivitas}}}}
36+
:neil {:project {:name io.github.timothypratley/clojurecivitas}}
37+
:dev {:jvm-opts ["--add-opens=java.base/java.nio=ALL-UNNAMED"]}}}

‎site/db.edn‎

Lines changed: 8 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -87,7 +87,14 @@
8787
:image "https://avatars.githubusercontent.com/u/4629681?v=4"
8888
:email ""
8989
:affiliation [:scicloj]
90-
:links [{:icon "github" :href "https://github.com/joinr"}]}]
90+
:links [{:icon "github" :href "https://github.com/joinr"}]}
91+
{:id :alza-bitz
92+
:name "Alex Coyle"
93+
:url "https://github.com/alza-bitz"
94+
:image "https://avatars.githubusercontent.com/u/1161048?v=4"
95+
:email ""
96+
:affiliation [:scicloj]
97+
:links [{:icon "github" :href "https://github.com/alza-bitz"}]}]
9198

9299
:affiliation
93100
[{:id :clojure.core

‎src/data_engineering/clojure_support_for_popular_data_tools.clj‎

Lines changed: 100 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,100 @@
1+
^{:kindly/hide-code true ; don't render this code to the HTML document
2+
:clay {:title "Clojure Support for Popular Data Tools: A Data Engineer's Perspective, and a New Clojure API for Snowflake"
3+
:quarto {:author :alza-bitz
4+
:draft true ; remove to publish
5+
:type :post
6+
:date "2025-09-04"
7+
:category :clojure
8+
:tags [:metadata :civitas]}}}
9+
(ns data-engineering.clojure-support-for-popular-data-tools)
10+
11+
;; In this article I look at the extent of Clojure support for some popular on-cluster data processing tools that Clojure users might need for their data engineering or data science tasks. Then for [Snowflake](https://snowflake.com) in particular **I go further and present a new Clojure API.**
12+
13+
;; Why is the level of Clojure support important? As an example, consider that [Scicloj](https://scicloj.org) is mostly focused on in-memory processing. As such, if you need to work with a large dataset it will be necessary to compute on-cluster and extract a smaller result before continuing your data science task locally.
14+
15+
;; However, without sufficient Clojure support for on-cluster processing, anyone needing that facility for their data science or data engineering task would be forced to reach outside the Clojure ecosystem. That adds complexity in terms of interop, compatibility and overall stack requirements.
16+
17+
;; With that in mind, let's examine the level of Clojure support for some popular on-cluster data processing tools. For each tool I selected its official Clojure library if one exists, or if not the most popular and well-known community-supported alternative with at least 100 stars and 10 contributors on GitHub. I then used the following criteria against the library to classify it as "supported" or "support unknown":
18+
19+
;; 1. CI/CD build passing
20+
;; 1. Most recent commit less than 12 months ago
21+
;; 1. Most recent release less than 12 months ago
22+
;; 1. Maintainers responded to any issue or question less than 12 months ago
23+
;; 1. Maintainers either accepted or rejected any PR less than 12 months ago
24+
25+
;; If I couldn't find any such library at all, I classified it as having "no support".
26+
27+
;; | Tool Category | Supported | Support Unknown | No Support |
28+
;; |---------------|---------------------|--------------|---------------|
29+
;; | **On-cluster batch processing** | | 1. [Spark](https://spark.apache.org) (see [Spark Interop with Geni](#spark_interop_with_geni) below) | |
30+
;; | **On-cluster stream processing** | | 2. [Kafka Streams](https://kafka.apache.org/documentation/streams) (see [Kafka Interop with Jackdaw](#kafka_interop_with_jackdaw) below) | 3. [Spark Structured Streaming](https://spark.apache.org/streaming),\<br\>4. [Flink](https://flink.apache.org) |
31+
;; | **On-cluster batch and stream processing** | | | 5. [Databricks](https://databricks.com) (see [Spark Interop with Geni](#spark_interop_with_geni) below),\<br\>6. [Snowflake](https://snowflake.com) (see [Snowflake Interop](#snowflake_interop_with_a_new_clojure_api!) below) |
32+
33+
;; Please note, I don't wish to make any critical judgments based on either the summary analysis above or the more detailed analysis below. The goal is to understand the situation with respect to Clojure support and highlight any gaps, although I suppose I am also inadvertently highlighting the difficulties of maintaining open source software!
34+
35+
;; ### Spark Interop with Geni
36+
37+
;; [Geni](https://github.com/zero-one-group/geni) is the go-to library for Spark interop. Some months back, I was motivated to evaluate the coverage of Spark features. In particular, I wanted to understand what would be involved to support [Spark Connect](https://spark.apache.org/spark-connect/) as it would reduce the complexity of computing on-cluster directly from the Clojure REPL.
38+
39+
;; However, I found a number of issues that would need to be addressed in order to support Spark Connect and Databricks:
40+
41+
;; 1. Problems with the [default session](https://github.com/zero-one-group/geni/issues/345).
42+
;; 1. Problems with [support for Databricks](https://github.com/zero-one-group/geni/issues/356), although I suspect this is related to point 1.
43+
44+
;; Also, in general by my criteria the support classification is "support unknown":
45+
;; 1. CI/CD build [failing.](https://github.com/zero-one-group/geni/actions)
46+
;; 1. Version [0.0.42 api docs broken](https://cljdoc.org/d/zero.one/geni/0.0.42/doc/readme%20%20https://cljdoc.org/builds/73977), also affects version 0.0.41
47+
;; 1. No commits since November 2023.
48+
;; 1. No releases since November 2023.
49+
;; 1. No PRs accepted or rejected since November 2023.
50+
;; 1. No response when attempting to contact the author or maintainers.
51+
52+
;; ### Kafka Interop with Jackdaw
53+
54+
;; [Jackdaw](https://github.com/FundingCircle/jackdaw) is the go-to library for Kafka interop. However, by my criteria the support classification is also "support unknown":
55+
56+
;; 1. No commits since August 2024.
57+
;; 1. No releases since December 2023.
58+
;; 1. No PRs accepted or rejected since August 2024. As a further example, [here's a PR](https://github.com/FundingCircle/jackdaw/pull/374) raised in May 2024 but not yet commented on either way by the maintainers.
59+
60+
;; ### Snowflake Interop with a New Clojure API!
61+
62+
;; Although the [Snowpark](https://docs.snowflake.com/en/developer-guide/snowpark/java) library has Java and Scala bindings, it doesn't provide anything for Clojure. As such, it's currently not possible to interact with Snowflake using the Clojure way.
63+
64+
;; To address this gap, I decided to try my hand at creating a [Clojure API for Snowflake](https://github.com/alza-bitz/snowflake-clj) as part of a broader effort to improve the overall situation regarding Clojure support for popular data tools.
65+
66+
;; The aim is to validate this approach as a foundation for enabling a wide range of data science or data engineering use cases from the Clojure REPL, in situations where Snowflake is the data warehouse of choice.
67+
68+
;; The [README](https://github.com/alza-bitz/snowpark-clj/blob/main/README.md) provides usage examples for all the current features, but I've copied the essential ones here to illustrate the API:
69+
70+
;; #### Feature 1. Load data from local and save to a Snowflake table
71+
72+
(require '[clojure.repl.deps :refer [add-lib]])
73+
(add-lib 'io.github.alza-bitz/snowpark-clj {:git/url "https://github.com/alza-bitz/snowpark-clj.git"
74+
:git/sha "7856d9ca2080b188f9feec115ca709d3f54877b0"})
75+
76+
(require '[snowpark-clj.core :as sp])
77+
78+
;; Sample data
79+
(def employee-data
80+
[{:id 1 :name "Alice" :age 25 :department "Engineering" :salary 75000}
81+
{:id 2 :name "Bob" :age 30 :department "Marketing" :salary 65000}
82+
{:id 3 :name "Charlie" :age 35 :department "Engineering" :salary 80000}])
83+
84+
;; Create session and save data
85+
(with-open [session (sp/create-session "src/data_engineering/snowflake.edn")]
86+
(-> (sp/create-dataframe session employee-data)
87+
(sp/save-as-table "employees" :overwrite)))
88+
89+
;; #### Feature 2. Compute over Snowflake table(s) on-cluster to produce a smaller result for local processing
90+
91+
(with-open [session (sp/create-session "src/data_engineering/snowflake.edn")]
92+
(let [table-df (sp/table session "employees")]
93+
(-> table-df
94+
(sp/filter (sp/gt (sp/col table-df :salary) (sp/lit 70000)))
95+
(sp/select [:name :salary])
96+
(sp/collect))))
97+
98+
;; As an early-stage proof-of-concept, it only covers the essential parts of the underlying API without being too concerned with performance or completeness. Other more advanced features are noted and planned, pending further elaboration.
99+
100+
;; **I hope you find it useful and I welcome any feedback or contributions!**

‎src/data_engineering/snowflake.edn‎

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,13 @@
1+
{:url "https://vkvupnf-yyb56283.snowflakecomputing.com"
2+
:user "ALZA"
3+
:password #mask #include "password.edn"
4+
:role "SNOWFLAKE_LEARNING_ROLE"
5+
:warehouse "SNOWFLAKE_LEARNING_WH"
6+
:db "SNOWFLAKE_LEARNING_DB"
7+
:schema "SNOWPARK_CLJ_TEST_SCHEMA"
8+
;; SSL Configuration - for dev container environment
9+
;; https://community.snowflake.com/s/article/How-to-turn-off-OCSP-checking-in-Snowflake-client-drivers
10+
;; Driver version 3.22.0 or higher:
11+
;; :disableOCSPChecks true
12+
;; Driver version 3.5.0 or higher:
13+
:insecureMode true}

0 commit comments

Comments
 (0)