diff --git a/.github/workflows/prod.yml b/.github/workflows/prod.yml index b4f0cd8..65741ab 100644 --- a/.github/workflows/prod.yml +++ b/.github/workflows/prod.yml @@ -73,6 +73,13 @@ jobs: retention-days: 14 archive: false + - name: Upload analytics page artifact + uses: actions/upload-artifact@v7 + with: + path: dashboard.html + retention-days: 14 + archive: false + - name: Upload API Log artifact if: '!cancelled()' #This ensures this step runs even if the previous steps failed only if manually cancelled it doesnt run uses: actions/upload-artifact@v7 diff --git a/database.py b/database.py new file mode 100644 index 0000000..1d5e49c --- /dev/null +++ b/database.py @@ -0,0 +1,29 @@ +import json +import traceback +import logging + +from pathlib import Path + +logger = logging.getLogger(__name__) + + +def read_database(database_path): + path = Path(database_path) + if not path.exists() or path.stat().st_size == 0: + return {} + + try: + with path.open() as database_file: + return json.load(database_file) + except (json.JSONDecodeError, ValueError) as exc: + logger.error("%s:%s", type(exc).__name__, exc) + traceback.print_exc() + return {} + + +def write_database(database_path, data): + path = Path(database_path) + temporary_path = path.with_name(f"{path.name}.tmp") + with temporary_path.open("w") as database_file: + json.dump(data, database_file, indent=4) + temporary_path.replace(path) diff --git a/main.py b/main.py index 1b85ddc..9961d03 100644 --- a/main.py +++ b/main.py @@ -13,6 +13,8 @@ import requests from adoption_sources import SourceManual, SourceRescueGroups +from database import read_database, write_database +from metrics_dashboard import dashboard from metric_collectors.bluesky import CollectorBluesky from metric_collectors.instagram import CollectorInstagram from metric_collectors.mastodon import CollectorMastodon @@ -103,6 +105,7 @@ def run(sources, posters, collectors=None, database_path="database.json"): record_publish_results(pet, published_results, database_path=database_path) collect_metrics(collectors or [], database_path=database_path) + dashboard(database_path=database_path) return results @@ -131,7 +134,7 @@ def publish_posts(pet, posters): def pick_pet(pets, database_path="database.json"): - data = _read_database(database_path) + data = read_database(database_path) posted_pet_ids = { posted_pet["pet_id"] for posted_pet in data.get("posted_pets", []) } @@ -149,7 +152,7 @@ def pick_pet(pets, database_path="database.json"): def record_publish_results(pet, results, database_path="database.json"): - data = _read_database(database_path) + data = read_database(database_path) posted_pets = data.setdefault("posted_pets", []) posts = data.setdefault("posts", []) posted_at = datetime.now(timezone.utc).isoformat() @@ -182,12 +185,12 @@ def record_publish_results(pet, results, database_path="database.json"): for item in posts if datetime.fromisoformat(item["posted_at"]) >= cutoff ] - _write_database(database_path, data) + write_database(database_path, data) def collect_metrics(collectors, database_path="database.json", window_days=14): try: - data = _read_database(database_path) + data = read_database(database_path) posts = data.get("posts", []) if not posts: return @@ -228,33 +231,11 @@ def collect_metrics(collectors, database_path="database.json", window_days=14): ) if updated: - _write_database(database_path, data) + write_database(database_path, data) except Exception as exc: logger.error("Metric collection failed: %s", exc) -def _read_database(database_path): - path = Path(database_path) - if not path.exists() or path.stat().st_size == 0: - return {} - - try: - with path.open() as database_file: - return json.load(database_file) - except (json.JSONDecodeError, ValueError) as exc: - logger.error("%s:%s", type(exc).__name__, exc) - traceback.print_exc() - return {} - - -def _write_database(database_path, data): - path = Path(database_path) - temporary_path = path.with_name(f"{path.name}.tmp") - with temporary_path.open("w") as database_file: - json.dump(data, database_file, indent=4) - temporary_path.replace(path) - - # Slack incoming-webhook messages have a ~40k-char limit; cap the traceback # well below that so the post stays readable and is never rejected. MAX_TRACEBACK_CHARS = 2500 diff --git a/metrics_dashboard.py b/metrics_dashboard.py new file mode 100644 index 0000000..74d4745 --- /dev/null +++ b/metrics_dashboard.py @@ -0,0 +1,173 @@ +import pandas as pd +import plotly.express as px +from database import read_database + + +def empty_dashboard_html(message): + return f""" + +
+{message}
+ + + """ + + +def dashboard(html_file_path="dashboard.html", database_path="database.json"): + # 1. Fetch data + data = read_database(database_path) + posts = data.get("posts", []) + posted_pets = data.get("posted_pets", []) + + if not posts or not posted_pets: + with open(html_file_path, "w", encoding="utf-8") as f: + f.write(empty_dashboard_html("No analytics data available yet.")) + return + + df_pets = pd.DataFrame(posted_pets) + df_posts = pd.json_normalize( + posts, + record_path=["metrics"], # Unpacks the nested metrics array + meta=["pet_id", "platform", "post_id", "post_url"], + ) + + # Merge pets metadata with detailed post metrics + df_metrics = pd.merge(df_pets, df_posts, on="pet_id", how="left") + + # Posts with no recorded metrics yet leave the metric columns absent + missing = [col for col in ("likes", "reposts", "comments") if col not in df_metrics.columns] + df_metrics = df_metrics.reindex( + columns=[*df_metrics.columns, *missing] + ) + + if not df_metrics["likes"].notna().any(): + with open(html_file_path, "w", encoding="utf-8") as f: + f.write(empty_dashboard_html("No metric data available yet.")) + return + + # 2. Get unique platforms including 'All Platforms' + platforms_html = ["All Platforms"] + sorted( + [ + str(p) + for p in df_metrics["platform"].dropna().unique() + if p != "All Platforms" + ] + ) + + # 3. Build options HTML for the dropdown + dropdown_options = "".join( + [f'' for p in platforms_html] + ) + + # 4. Generate HTML blocks for each platform and metric + sections_html = "" + for p in platforms_html: + if p == "All Platforms": + filtered_platform_temp = df_metrics + else: + filtered_platform_temp = df_metrics[df_metrics["platform"] == p] + + platform_charts_html = "" + + for col in ["likes", "reposts", "comments"]: + temp = ( + filtered_platform_temp[["post_id", "pet_id", "name", col]] + .groupby(["post_id", "pet_id", "name"]) + .max() + .reset_index() + ) + + agg_temp = ( + temp[["pet_id", "name", col]] + .groupby(["pet_id", "name"])[col] + .agg(["max", "sum"]) + .reset_index() + ) + + temp1 = agg_temp.sort_values("max", ascending=False).head(10) + temp2 = agg_temp.sort_values("sum", ascending=False).head(10) + + metric_label = col.capitalize() + + fig_max = px.bar( + data_frame=temp1, + x="name", + y="max", + title=f"Top 10 Pets by Max {metric_label} [{p}]", + labels={"name": "Pet", "max": f"Max {metric_label}"}, + ) + fig_sum = px.bar( + data_frame=temp2, + x="name", + y="sum", + title=f"Top 10 Pets by Total {metric_label} [{p}]", + labels={"name": "Pet", "sum": f"Total {metric_label}"}, + ) + + platform_charts_html += f""" +