Updated metadata and did some cleaning

tombarys · tombarys · commit d08aa7ff51be · 2025-10-14T13:24:22.000+02:00
diff --git a/site/db.edn b/site/db.edn
@@ -116,6 +116,12 @@
    :email       "edwardaw@connect.hku.hk"
    :affiliation [:cim]
    :links       [{:icon "github" :href "https://github.com/W-Edward"}]}
+  {:id    :tombarys
+   :name  "Tomáš Baránek"
+   :image "https://avatars.githubusercontent.com/u/5729352?v=4"
+   :url   "https://barys.me/#english-section"
+   :email "tom@barys.me"
+   :links [{:icon "github" :href "https://github.com/tombarys"}]}
   {:id          :mattb
    :name        "Matthias Buehlmaier"
    :affiliation [:cim]}
@@ -145,8 +151,7 @@
    :url "https://techascent.com"}
   {:id   :cim
    :name "cim"
-   :url  "https://cim.hkubs.hku.hk/"
-  }]
+   :url  "https://cim.hkubs.hku.hk/"}]
 
  :topic
  [{:id        :core
diff --git a/src/data_analysis/book_sales_analysis/about_apriori.clj b/src/data_analysis/book_sales_analysis/about_apriori.clj
@@ -1,6 +1,8 @@
 ^{:kindly/hide-code true     ; don't render this code to the HTML document
   :clay             {:title  "From Correlations to Recommendations"
                      :quarto {:author   [:tombarys]
+                              :image "src/data_analysis/book_sales_analysis/graphviz.png"
+                              :description "A Publisher's Journey into Data-Driven Book Sales – exploring how association rule mining can transform business insights using the SciCloj stack."
                               :type     :draft
                               :date     "2025-10-13"
                               :category :data-analysis
@@ -24,7 +26,6 @@
 
 ;; When you run an indie publishing house with over 160 titles and sell thousands of books each month, one question keeps coming back: **Which books do our customers buy together?** This seemingly simple question led me down a fascinating path from basic correlation analysis to building a more robust recommendation system using association rule mining—all with Clojure and the SciCloj ecosystem. 
 
-;; > *Disclaimer: please consider that my experience in Clojure is intermediate and entry-level at SciCloj and data-science. I can still assure you my enthusiasm is high.*
 
 ;; ## The Starting Point: Understanding Our Data
 
@@ -37,7 +38,7 @@
       (tc/random 5)))
 
 ^:kindly/hide-code
-(kind/table 
+(kind/table
  orders-sample {:element/max-height 400})
 
 ;; *(for clarity, many columns were omitted here; rows were generated with `(tc/random 5)` from anonymized dataset)*
@@ -50,18 +51,23 @@
 
 ;; ## The Transformation: Making Data Analysis-Ready
 
-;; The transformation from raw orders to an analysis-ready format was crucial. Using Tablecloth, the transformation pipeline was surprisingly readable:
-
-;; ❗ FIXME this is too simplified, I have to change this ❗
+;; The transformation from raw orders to an analysis-ready format was crucial. Using Tablecloth, the transformation pipeline was easy (and can be even more simplified).
 
-^:kindly/hide-code 
+^:kindly/hide-code
 (kind/code
- ";; From customer orders with book lists...
-(-> orders
-    (tc/group-by :zakaznik)           ;; Group by customer
-    (tc/aggregate                     ;; Aggregate their purchases
-      {:books #(distinct-books %)})
-    ;; ...to binary matrix where each column is a book")
+";; From customer orders with book lists...
+(map
+ (fn [customer-row]
+   (let [customer-name (:zakaznik customer-row)
+         books-bought-set (set (parse-books-from-list (:all-products customer-row)))
+         one-hot-map (reduce (fn [acc book]
+                               (assoc acc book (if (contains? books-bought-set book) 1 0)))
+                             {}
+                             all-titles)]
+     (merge {:zakaznik customer-name}
+            one-hot-map)))
+ (tc/rows customer+orders :as-maps))
+;; ...to binary matrix where each column is a book")
 
 ;; After transformation, each customer became a row, and each book a column with 1 or 0:
 
@@ -73,7 +79,7 @@
       (tc/reorder-columns [:zakaznik])
       (tc/random 5)
       (tc/head 4)
-      (tc/select-columns #"^:zakaznik|^:book-00.*")))
+      (tc/select-columns #"^:zakaznik|^:book-00.*"))) ;; just quick reduction of table width
 
 ^:kindly/hide-code
 (kind/table onehot-sample)
@@ -176,23 +182,28 @@ scatter-plot
 
 ;; One of the most elegant aspects of the Apriori implementation is how it generates larger itemsets from smaller ones without creating duplicates. Consider generating 3-item sets from 2-item sets:
 
+;; ```
+;; We want: [:book-a :book-b :book-c] ✓  
+;; Not:     [:book-a :book-c :book-b] ✗ (same set, different order)  
+;;          [:book-b :book-a :book-c] ✗ (same set, different order)  
+;; ```
+
 ^:kindly/hide-code
 (kind/code
- ";; We want: [:book-a :book-b :book-c] ✓
-;; Not:     [:book-a :book-c :book-b] ✗ (same set, different order)
-;;          [:book-b :book-a :book-c] ✗ (same set, different order)
-
-(defn join-itemsets [frequent-itemsets k]
+ "(defn join-itemsets
+  [frequent-itemsets k]
   (let [k-1 (dec k)
+        ;; Only process itemsets of the correct size
         valid-sets (filter #(= (count %) k-1) frequent-itemsets)
+        ;; Group by prefix (first k-2 elements) for efficiency
         by-prefix (group-by #(vec (take (- k 2) %)) valid-sets)]
     (mapcat
      (fn [[prefix items]]
        (for [set1 items
              set2 items
              :let [last1 (last set1)
                    last2 (last set2)]
-             ;; This line ensures canonical ordering
+             ;; Only join if last2 > last1 (enforces canonical order)
              :when (and (not= last1 last2)
                         (pos? (compare last2 last1)))]
          (concat prefix [last1 last2])))
@@ -206,21 +217,21 @@ scatter-plot
 
 ;; **Support** measures how frequently an itemset appears:
 
-^:kindly/hide-code
-(kind/tex
- "\\text{Support}(\\{A, B\\}) = \\dfrac{\\text{orders with A and B}}{\\text{total orders}}")
+;; $$  
+;; \text{Support}(\{A, B\}) = \dfrac{\text{orders with A and B}}{\text{total orders}}  
+;; $$ 
 
 ;; **Confidence** measures how often B appears when A is purchased:
 
-^:kindly/hide-code
-(kind/tex
- "\\text{Confidence}(A \\rightarrow B) = \\dfrac{\\text{Support}(\\{A, B\\})}{\\text{Support}(\\{A\\})}")
+;; $$ 
+;; \text{Confidence}(A \rightarrow B) = \dfrac{\text{Support}(\{A, B\})}{\text{Support}(\{A\})}
+;; $$ 
 
 ;; **Lift** measures whether this happens more than random chance:
 
-^:kindly/hide-code
-(kind/tex
- "\\text{Lift}(A \\rightarrow B) = \\dfrac{\\text{Confidence}(A \\rightarrow B)}{\\text{Support}(\\{B\\})}")
+;; $$ 
+;; "\text{Lift}(A \rightarrow B) = \dfrac{\text{Confidence}(A \rightarrow B)}{\text{Support}(\{B\})}
+;; $$ 
 
 ;; A lift greater than 1 indicates positive association—the items are purchased together more often than if they were independent. A lift of 2.3 means the combination is 2.3 times more likely than chance.
 
diff --git a/src/data_analysis/book_sales_analysis/core_helpers_v2.clj b/src/data_analysis/book_sales_analysis/core_helpers_v2.clj
@@ -7,16 +7,12 @@
             [clojure.string :as str]
             [java-time.api :as jt]
             [fastmath.stats :as stats]
-            [scicloj.kindly.v4.kind :as kind]))
+            [scicloj.kindly.v4.kind :as kind]
+            [data-analysis.book-sales-analysis.data-sources-v2 :as data]))
 
 ;; ## Data Transformation Functions
 ;; Common data processing functions used across multiple analysis files
 
-;; ### Date Time
-
-(def end-time
-  (jt/local-date 2025 10 1))
-
 ;; ### Scicloj Helpers
 
 (defn merge-csvs [file-list options]
@@ -118,37 +114,18 @@
                    (map #(str/replace % #"\+" ""))
                    (map #(str/trim %))
                    (map sanitize-str)
-                   (map #(str/replace % #"\-\-.+$" "")) ;; zdvojené názvy
-                   (map #(str/replace % #"\-+$" "")) ;; pomlčky na konci
+                   (map #(str/replace % #"\-\-.+$" "")) 
+                   (map #(str/replace % #"\-+$" "")) 
                    (map #(str/replace % #"^3" "k3"))
-                   (map #(str/replace % #"^5" "k5"));; eliminace čísel 3 na začátku dvou knih
+                   (map #(str/replace % #"^5" "k5"))
                    (remove (fn [item] (some (fn [substr] (str/includes? (name item) substr))
                                             ["balicek" "poukaz" "zapisnik" "limitovana-edice" "taska" "aktualizovane-vydani" "cd" "puvodni-vydani/neni-skladem"
                                              "merch"])))
                    distinct
                    (mapv keyword))
       nil))
 
-;; ### Melvil Data Enriching and Convenience Functions
-
-(defn months-between "Calculate how many months a product has been on market"
-  [start-date end-date]
-  (let [days (if (and start-date end-date)
-               (jt/time-between start-date end-date :days)
-               0)]
-    (long (Math/round (/ days 30.4375)))))
-
-(defn months-on-market
-  "Months `book` is on a market. Zero if not at all."
-  [books-ds book end-date]
-  (let [date (try
-               (-> books-ds
-                   (tc/select-columns [:titul :datum-zahajeni-prodeje])
-                   (tc/select-rows #(str/starts-with? (name (:titul %)) (name book)))
-                   (tc/get-entry :datum-zahajeni-prodeje 0))
-               (catch Exception e nil))
-        month (if (nil? date) 0 (months-between date end-date))]
-    month))
+;; ### Metadata Enriching and Convenience Functions
 
 (defn czech-author? [book-title]
   (let [czech-books #{:k30-hodin
@@ -182,71 +159,10 @@
       (rand-int 2)
       (if (contains? czech-books (keyword book-title)) 1 0))))
 
-(def category-enrichments
-  {:k30-hodin "podnikani,firemni-kultura"
-   :k5-principu-rodicovstvi "vzdelavani-a-vychova,kariera,psychologie,mezilidska-komunikace"
-   :autismus-bez-masky "psychologie,spolecnost,mezilidska-komunikace"
-   :genialne-potraviny :zdravi
-   :genialni-potraviny :zdravi
-   :jak-zabranit-dalsi-pandemii "budoucnost,spolecnost,ekologie"
-   :krvavy-uterek :historie
-   :male-experimenty "kariera,produktivita,psychologie"
-   :myty-a-nadeje-digitalniho-sveta "budoucnost,spolecnost,kariera"
-   :nestekej-na-sveho-psa "vzdelavani-a-vychova,mezilidska-komunikace"
-   :nove-zbrane-vlivu "podnikani,psychologie,kariera,mezilidska-komunikace"
-   :pamet "zdravi,psychologie"
-   :pomala-produktivita "produktivita,kariera"
-   :poridte-si-druhy-mozek "produktivita,kariera"
-   :prezit "zdravi,psychologie,mezilidska-komunikace"
-   :stastnejsi "psychologie,zdravi,mezilidska-komunikace"
-   :ultrazpracovani-lide :zdravi
-   :vitamin-l "psychologie,mezilidska-komunikace"
-   :zazracna-imunita :zdravi
-   :heureka! "podnikani,firemni-kultura"
-   :zrozeni-evropanu "historie,spolecnost"})
-
-(defn enrich-metadata-csv
-  "Takes a CSV `file` with book titles, subtitles, categories and technical parameters
-   and enriches it with supplemental categories and info about author nationalities."
-  [file]
-  (let [summary-raw-ds (-> (tc/dataset
-                            file
-                            {:key-fn #(keyword (sanitize-column-name-str %))
-                             :separator \;
-                             :parser-fn {:datum-zahajeni-prodeje :string}
-                             :encoding :utf-8})
-                           (tc/update-columns :datum-zahajeni-prodeje
-                                              #(map (fn [date-str]
-                                                      (when (not-empty date-str)
-                                                        (parse-csv-date date-str)))
-                                                    %))
-                           #_(tc/drop-missing [:edice :datum-zahajeni-prodeje]))
-        sanitized-colnames-ds (-> summary-raw-ds
-                                  (tc/rename-columns :all (fn [col] (if col (keyword (sanitize-column-name-str (name col))) col))))
-
-        sanitized-rows-ds (-> sanitized-colnames-ds
-                              (tc/update-columns  [:titul :podtitul :vazba :barevnost :edice :cenova-kategorie :tloustka]
-                                                  (fn [column-data] (map sanitize-column-name-str column-data)))
-                              (tc/update-columns  [:kategorie-na-e-shopu :kategorie-tema]
-                                                  (fn [column-data] (map sanitize-category-str column-data)))
-                              (tc/update-columns [:titul] #(map (comp keyword parse-book-name) %)))
-
-        enriched--categories-ds (tc/update-columns sanitized-rows-ds :kategorie-na-e-shopu
-                                                   (fn [categories]
-                                                     (map (fn [title category]
-                                                            (if-let [enriched-category (get category-enrichments title)]
-                                                              (name enriched-category)
-                                                              category))
-                                                          (sanitized-rows-ds :titul)
-                                                          categories)))
-        enriched-ds (tc/add-column enriched--categories-ds
-                                   :cesky-autor (fn [ds] (map czech-author? (ds :titul))))]
-    (-> enriched-ds
-        (tc/rename-columns {:column-0 :titul}))))
-
 ;; ### One-Hot Encoding Functions
 
-(defn onehot-encode-by-customers
+   
+(defn onehot-encode-by-customers ;; FIXME needs refactor and simplification :)
   "One-hot encode dataset aggregated by customer. 
    Each customer gets one row with 0/1 values for each book they bought.
    Used for market basket analysis, customer segmentation, etc."
@@ -296,21 +212,6 @@
       0.0
       (double (/ transactions-with-itemset total-transactions)))))
 
-(defn build-popularity-index
-  "Creates a popularity index (map in format `{:book :popularity}`) for all items in one-hot encoded dataset"
-  [dataset]
-  (let [items (-> dataset (tc/drop-columns :zakaznik) tc/column-names)]
-    (reduce (fn [acc item]
-              (assoc acc item (calculate-support dataset [item])))
-            {}
-            items)))
-
-^:kindly/hide-code
-#_(-> (build-popularity-index (onehot-encode-by-customers data-sources/orders))
-      (tc/dataset)
-      (tc/pivot->longer)
-      (tc/order-by :$value))
-
 
 ^:kindly/hide-code
 (defn calculate-adaptive-coefficient
diff --git a/src/data_analysis/book_sales_analysis/data_sources_v2.clj b/src/data_analysis/book_sales_analysis/data_sources_v2.clj
@@ -29,9 +29,6 @@
 
 ;; ### Book Metadata
 
-#_(def enriched-book-metadata-ds
-  (helpers/enrich-metadata-csv "data/summary-all-time-all-books-300725.csv"))
-
 ;; ## Quick Access
 ;; Most commonly used datasets with short aliases
 
diff --git a/src/data_analysis/book_sales_analysis/graphviz.png b/src/data_analysis/book_sales_analysis/graphviz.png
diff --git a/src/data_analysis/book_sales_analysis/market_basket_analysis_v2.clj b/src/data_analysis/book_sales_analysis/market_basket_analysis_v2.clj
@@ -12,8 +12,6 @@
             [clojure.math.combinatorics :as combo]
             [clojure.string :as str]
             [clojure.set]
-            [scicloj.tableplot.v1.plotly :as plotly]
-            [scicloj.kindly.v4.kind :as kind]
             [data-analysis.book-sales-analysis.data-sources-v2 :as data]
             [data-analysis.book-sales-analysis.core-helpers-v2 :as helpers]))
 
@@ -403,7 +401,7 @@
 (comment
   ;; Generate once - takes ~1 minute
   (def quick-analysis
-    (generate-market-basket-analysis data/orders-slides 
+    (generate-market-basket-analysis data/orders-share 
                                      :subset-size 6000 
                                      :min-confidence 0.03))
 
@@ -492,7 +490,7 @@
 
 ;; ### Prediction Demo
 
-;; 🎯 **LIVE DEMO**: Show how the website will make recommendations
+;; 🎯 **LIVE DEMO**: Show how the website should make recommendations
 
 ^:kindly/hide-code
 (comment
