diff --git a/docs/guide/api/api.md b/docs/guide/api/api.md index 77b20f97..96b25349 100644 --- a/docs/guide/api/api.md +++ b/docs/guide/api/api.md @@ -906,7 +906,10 @@ Superadmins must specify the organization ID since they can create projects for }, "demand_overview": { "community_demands": 320, - "individual_demands": 215 + "individual_demands": 215, + "unclassified_demands": 40, + "total_classified_demands": 535, + "total_demands": 575 }, "commons_connect_operational": { "active_tehsils": 25, @@ -957,6 +960,9 @@ Superadmins must specify the organization ID since they can create projects for ``` - **Notes**: - `demand_overview`: counts Community Demands and Individual Demands across all NRM maintenance (Section E) and NRM works (Section F) records for the filtered plans + - `total_demands`: every demand record encountered, regardless of whether its ownership value could be classified + - `total_classified_demands`: `community_demands + individual_demands` + - `unclassified_demands`: `total_demands - total_classified_demands` — records whose raw ownership value didn't match a known community/individual keyword - `landscape_stewards.total_stewards`: only counts facilitators who are **App User** group members and do **not** belong to the CFPT organization - `landscape_stewards.gender_breakdown`: male/female/other counts from the User table for the active stewards; users without a gender set are excluded from all buckets - `by_organization` in `landscape_stewards` is omitted when `?organization` filter is applied diff --git a/plans/reports.py b/plans/reports.py index 87d85e78..6a8595aa 100644 --- a/plans/reports.py +++ b/plans/reports.py @@ -2,7 +2,7 @@ import io from calendar import monthrange -from django.db.models import Q +from django.db.models import Count, Q from django.utils import timezone @@ -154,3 +154,320 @@ def generate_summary_csv(report_date=None): w.writerow(row) return buf.getvalue().encode("utf-8") + + +def _tally_demand_types(demand_totals, raw_values): + from dpr.mapping import classify_demand_type + + for raw in raw_values: + demand_totals["total"] += 1 + classified = classify_demand_type(raw) + if classified == "Community Demand": + demand_totals["community"] += 1 + demand_totals["total_classified"] += 1 + elif classified == "Individual Demand": + demand_totals["individual"] += 1 + demand_totals["total_classified"] += 1 + else: + demand_totals["unclassified"] += 1 + + +def generate_dpr_demand_overview_csv(report_date=None): + """ + DPR review/approval activity and demand-type overview for the report + month only (not cumulative). Each metric is scoped by its own source + timestamp field: + - plan flags -> PlanApp.updated_at + - DPR report status -> DPR_Report.last_updated_at + - demand records -> each ODK/maintenance model's submission_time + + Note: PlanApp.updated_at / DPR_Report.last_updated_at are bumped on ANY + save, not just the specific transition being counted, so a record last + touched this month for an unrelated reason will still show up here. + """ + from dpr.models import ( + Agri_maintenance, + DPR_Report, + GW_maintenance, + ODK_agri, + ODK_agrohorticulture, + ODK_groundwater, + ODK_livelihood, + SWB_maintenance, + SWB_RS_maintenance, + ) + from dpr.services import CFPT_ORG_ID + + first_of_month, end_of_month = _get_month_boundaries(report_date) + month_window = (first_of_month, end_of_month) + + plans = _base_queryset(end_of_month) + plan_id_strs = [str(pid) for pid in plans.values_list("id", flat=True)] + + plan_meta = plans.filter(updated_at__range=month_window).aggregate( + completed=Count("id", filter=Q(is_completed=True)), + dpr_reviewed=Count("id", filter=Q(is_dpr_reviewed=True)), + dpr_approved=Count("id", filter=Q(is_dpr_approved=True)), + ) + + dpr_counts = ( + DPR_Report.objects.filter(plan_id__in=plans, last_updated_at__range=month_window) + .exclude(plan_id__organization_id=CFPT_ORG_ID) + .aggregate( + submitted=Count( + "dpr_report_id", filter=Q(status__in=["SUBMITTED", "APPROVED"]) + ), + approved=Count("dpr_report_id", filter=Q(status="APPROVED")), + ) + ) + + demand_totals = { + "community": 0, + "individual": 0, + "unclassified": 0, + "total": 0, + "total_classified": 0, + } + + def in_month(qs): + return qs.filter(submission_time__range=month_window) + + _tally_demand_types( + demand_totals, + ( + (d or {}).get("demand_type") + for d in in_month( + GW_maintenance.objects.filter(plan_id__in=plan_id_strs).exclude(is_deleted=True) + ).values_list("data_gw_maintenance", flat=True) + ), + ) + _tally_demand_types( + demand_totals, + ( + (d or {}).get("demand_type") + for d in in_month( + Agri_maintenance.objects.filter(plan_id__in=plan_id_strs).exclude(is_deleted=True) + ).values_list("data_agri_maintenance", flat=True) + ), + ) + _tally_demand_types( + demand_totals, + ( + (d or {}).get("demand_type") + for d in in_month( + SWB_maintenance.objects.filter(plan_id__in=plan_id_strs).exclude(is_deleted=True) + ).values_list("data_swb_maintenance", flat=True) + ), + ) + _tally_demand_types( + demand_totals, + ( + (d or {}).get("demand_type") + for d in in_month( + SWB_RS_maintenance.objects.filter(plan_id__in=plan_id_strs).exclude(is_deleted=True) + ).values_list("data_swb_rs_maintenance", flat=True) + ), + ) + _tally_demand_types( + demand_totals, + ( + (d or {}).get("demand_type") + for d in in_month( + ODK_groundwater.objects.filter(plan_id__in=plan_id_strs) + .exclude(is_deleted=True) + .exclude(status_re="rejected") + ).values_list("data_groundwater", flat=True) + ), + ) + _tally_demand_types( + demand_totals, + ( + (d or {}).get("demand_type_irrigation") + for d in in_month( + ODK_agri.objects.filter(plan_id__in=plan_id_strs) + .exclude(is_deleted=True) + .exclude(status_re="rejected") + ).values_list("data_agri", flat=True) + ), + ) + + for _, dl in ( + in_month( + ODK_livelihood.objects.filter(plan_id__in=plan_id_strs) + .exclude(is_deleted=True) + .exclude(status_re="rejected") + ).values_list("submission_time", "data_livelihood") + ): + dl = dl or {} + livestock = dl.get("Livestock") or {} + fisheries = dl.get("fisheries") or {} + plantations = dl.get("plantations") or {} + kitchen_garden = dl.get("kitchen_gardens") or {} + raws = [] + if ( + str(livestock.get("is_demand_livestock", "")).lower() == "yes" + or str(dl.get("select_one_demand_promoting_livestock", "")).lower() == "yes" + ): + raws.append(livestock.get("livestock_demand")) + if ( + str(fisheries.get("is_demand_fisheris", "")).lower() == "yes" + or str(dl.get("select_one_demand_promoting_fisheries", "")).lower() == "yes" + ): + raws.append(fisheries.get("demand_type_fisheries")) + if ( + str(dl.get("select_one_demand_plantation", "")).lower() == "yes" + or str(plantations.get("select_plantation_demands", "")).lower() == "yes" + ): + raws.append(plantations.get("demand_type_plantations")) + if ( + str(dl.get("indi_assets", "")).lower() == "yes" + or str(kitchen_garden.get("assets_kg", "")).lower() == "yes" + ): + raws.append(kitchen_garden.get("demand_type_kitchen_garden")) + _tally_demand_types(demand_totals, raws) + + # ODK_agrohorticulture has no submission_time field (removed in dpr + # migration 0004), so its demand records can't be scoped to this month + # and are intentionally excluded here — see the note row in the CSV. + + buf = io.StringIO() + w = csv.writer(buf) + month_label = end_of_month.strftime("%B %Y") + + w.writerow(["Report Month", month_label]) + w.writerow( + [ + "Report Window", + f"{first_of_month.strftime('%Y-%m-%d')} to {end_of_month.strftime('%Y-%m-%d')}", + ] + ) + w.writerow([]) + w.writerow(["DPR & Plan Status", "Count", "Source", "Timestamp Field"]) + w.writerow(["Plans Completed", plan_meta["completed"], "PlanApp.is_completed", "PlanApp.updated_at"]) + w.writerow(["Plans DPR Reviewed", plan_meta["dpr_reviewed"], "PlanApp.is_dpr_reviewed", "PlanApp.updated_at"]) + w.writerow(["Plans DPR Approved", plan_meta["dpr_approved"], "PlanApp.is_dpr_approved", "PlanApp.updated_at"]) + w.writerow( + [ + "DPR Submitted (Submitted + Approved)", + dpr_counts["submitted"] or 0, + "DPR_Report.status", + "DPR_Report.last_updated_at", + ] + ) + w.writerow(["DPR Approved", dpr_counts["approved"] or 0, "DPR_Report.status", "DPR_Report.last_updated_at"]) + w.writerow([]) + w.writerow(["Demand Overview", "Count"]) + w.writerow(["Community Demands", demand_totals["community"]]) + w.writerow(["Individual Demands", demand_totals["individual"]]) + w.writerow(["Unclassified Demands", demand_totals["unclassified"]]) + w.writerow(["Total Classified Demands", demand_totals["total_classified"]]) + w.writerow(["Total Demands", demand_totals["total"]]) + w.writerow([]) + w.writerow( + [ + "Note", + "Agrohorticulture demand records have no submission timestamp " + "and are excluded from this month-scoped report.", + ] + ) + + return buf.getvalue().encode("utf-8") + + +def generate_resource_demand_status_csv(report_date=None): + """ + Per-type status counts for every resource and demand model (mirrors + dpr.services RESOURCE_TYPE_MAP / DEMAND_TYPE_MAP, imported directly so + this never drifts from the real status-tracking API), scoped to records + submitted during the report month via each model's submission_time. + """ + from dpr.services import DEMAND_TYPE_MAP, RESOURCE_TYPE_MAP + + first_of_month, end_of_month = _get_month_boundaries(report_date) + month_window = (first_of_month, end_of_month) + + plans = _base_queryset(end_of_month) + plan_id_strs = [str(pid) for pid in plans.values_list("id", flat=True)] + + def has_submission_time(model): + return "submission_time" in {f.name for f in model._meta.get_fields()} + + def status_rows(type_map): + rows = [] + for type_name, (model, pk_field, demand_field) in type_map.items(): + timestamped = has_submission_time(model) + qs = model.objects.exclude(is_deleted=True).filter(plan_id__in=plan_id_strs) + if timestamped: + qs = qs.filter(submission_time__range=month_window) + else: + qs = qs.none() + counts = dict(qs.values_list(demand_field).annotate(count=Count(pk_field))) + rows.append( + { + "type": type_name, + "pending": counts.get("PENDING", 0), + "submitted": counts.get("SUBMITTED", 0) + counts.get("APPROVED", 0), + "approved": counts.get("APPROVED", 0), + "reverted": counts.get("REVERTED", 0), + "rejected": counts.get("REJECTED", 0), + "total": sum(counts.values()), + "timestamped": timestamped, + } + ) + return rows + + buf = io.StringIO() + w = csv.writer(buf) + month_label = end_of_month.strftime("%B %Y") + + w.writerow(["Report Month", month_label]) + w.writerow( + [ + "Report Window", + f"{first_of_month.strftime('%Y-%m-%d')} to {end_of_month.strftime('%Y-%m-%d')}", + ] + ) + w.writerow([]) + w.writerow( + [ + "Section", + "Type", + "Pending", + "Submitted (incl. Approved)", + "Approved", + "Reverted", + "Rejected", + "Total", + "Timestamp Available", + ] + ) + for row in status_rows(RESOURCE_TYPE_MAP): + w.writerow( + [ + "Resource", + row["type"], + row["pending"], + row["submitted"], + row["approved"], + row["reverted"], + row["rejected"], + row["total"], + row["timestamped"], + ] + ) + for row in status_rows(DEMAND_TYPE_MAP): + w.writerow( + [ + "Demand", + row["type"], + row["pending"], + row["submitted"], + row["approved"], + row["reverted"], + row["rejected"], + row["total"], + row["timestamped"], + ] + ) + + return buf.getvalue().encode("utf-8") diff --git a/plans/tasks.py b/plans/tasks.py index 475625b6..528ba371 100644 --- a/plans/tasks.py +++ b/plans/tasks.py @@ -7,7 +7,12 @@ from utilities.logger import setup_logger from utilities.mailutils import send_email -from .reports import generate_plan_details_csv, generate_summary_csv +from .reports import ( + generate_dpr_demand_overview_csv, + generate_plan_details_csv, + generate_resource_demand_status_csv, + generate_summary_csv, +) logger = setup_logger(__name__) @@ -30,6 +35,8 @@ def send_monthly_plan_report(self, force=False): details_csv = generate_plan_details_csv(now) summary_csv = generate_summary_csv(now) + dpr_demand_overview_csv = generate_dpr_demand_overview_csv(now) + resource_demand_status_csv = generate_resource_demand_status_csv(now) suffix = now.strftime("%b_%Y").lower() send_email( @@ -38,6 +45,8 @@ def send_monthly_plan_report(self, force=False): f"Please find attached the monthly plan report for {month_label}.\n\n" f"1. Plan Details - Complete list of all plans\n" f"2. Summary - Overview with organization-level breakdown\n" + f"3. DPR & Demand Overview - DPR review/approval and demand-type counts for the month\n" + f"4. Resource & Demand Status - per-type status counts for the month\n" ), to_emails=recipients, attachments=[ @@ -51,6 +60,16 @@ def send_monthly_plan_report(self, force=False): "content": summary_csv, "mimetype": "text/csv", }, + { + "filename": f"dpr_demand_overview_{suffix}.csv", + "content": dpr_demand_overview_csv, + "mimetype": "text/csv", + }, + { + "filename": f"resource_demand_status_{suffix}.csv", + "content": resource_demand_status_csv, + "mimetype": "text/csv", + }, ], ) diff --git a/plans/views.py b/plans/views.py index 5e0a611d..2055db40 100755 --- a/plans/views.py +++ b/plans/views.py @@ -211,6 +211,8 @@ def _livelihood_demand_types(): return { "community_demands": community, "individual_demands": individual, + "unclassified_demands": total - community - individual, + "total_classified_demands": community + individual, "total_demands": total, }