ci: wire issue-932 repro and harden sitemap url-root assertion (#25)

.github/workflows/e2e-issue-981.yml

@@ -87,7 +87,7 @@ jobs:
           fi
           echo "KEY=$KEY" >> "$GITHUB_ENV"
 
-      - name: Issue-15 setup - install the SEO test interview and configure dispatch + social.og (no image)
+      - name: Issue-15 setup - install the SEO test interview and configure dispatch + social.og (no image) + url root
         run: |
           D=$(docker exec da bash -c "ls -d /usr/share/docassemble/local*/lib/python3*/site-packages/docassemble/demo/data/questions")
           docker cp .github/workflows/e2e/test_seo.yml da:"$D/test_seo.yml"
@@ -99,14 +99,15 @@ jobs:
           c=yaml.safe_load(open(p)) or {}
           c.setdefault('dispatch', {})['seo']='docassemble.demo:data/questions/test_seo.yml'
           og=c.setdefault('social', {}).setdefault('og', {}); og.pop('image', None); og.setdefault('locale','en_US')
+          c['url root']='https://law.example.org'
           yaml.safe_dump(c, open(p,'w'), default_flow_style=False)
           PYEOF
           docker exec da bash -c "supervisorctl restart uwsgi"
           sleep 30
 
       - name: Issue-15 control - SEO head tags and sitemap must be absent on the unpatched release
         run: |
-          if ! bash .github/workflows/e2e/seo_check.sh http://localhost:8080 expect-fail; then
+          if ! bash .github/workflows/e2e/seo_check.sh http://localhost:8080 expect-fail https://law.example.org; then
             echo "::notice::ALREADY FIXED upstream: issue 15 — SEO head tags present in pinned image"
           fi
 
@@ -131,6 +132,9 @@ jobs:
             echo "::notice::ALREADY FIXED upstream: issue 18 — a11y fixes already present in pinned image"
           fi
 
+      - name: Issue-932 control - unpatched release must crash when geopy is broken
+        run: bash .github/workflows/e2e/issue_932_repro.sh http://localhost:8080 da expect-fail
+
       - name: Control - unpatched release must reject the plain dict
         if: env.ALREADY_FIXED != '1'
         run: bash .github/workflows/e2e/issue_981_repro.sh http://localhost:8080 "$KEY" expect-fail
@@ -164,12 +168,13 @@ jobs:
             echo "::notice::ALREADY FIXED upstream: issue 968 — hasattr error already reports real attribute"
           fi
 
-      - name: Install this branch's util.py and email_crypto.py, restart
+      - name: Install this branch's util.py, email_crypto.py, and geocode.py, restart
         run: |
           U=$(docker exec da bash -c "ls /usr/share/docassemble/local*/lib/python3*/site-packages/docassemble/base/util.py")
           docker cp docassemble_base/docassemble/base/util.py da:"$U"
           B=$(docker exec da bash -c "ls -d /usr/share/docassemble/local*/lib/python3*/site-packages/docassemble/base")
           docker cp docassemble_base/docassemble/base/email_crypto.py da:"$B/email_crypto.py"
+          docker cp docassemble_base/docassemble/base/geocode.py da:"$B/geocode.py"
           docker exec da bash -c "supervisorctl restart uwsgi"
           sleep 30
 
@@ -179,6 +184,9 @@ jobs:
           PY=$(docker exec da bash -c "ls /usr/share/docassemble/local*/bin/python3 | head -1")
           docker exec da "$PY" /tmp/email_crypto_check.py
 
+      - name: Issue-932 - patched server must import fine when geopy is broken
+        run: bash .github/workflows/e2e/issue_932_repro.sh http://localhost:8080 da expect-pass
+
       - name: Pickle hardening - legit stored objects load, RCE payloads raise (#32)
         run: |
           F=$(docker exec da bash -c "ls /usr/share/docassemble/local*/lib/python3*/site-packages/docassemble/webapp/utils/fixpickle.py")
@@ -419,7 +427,7 @@ jobs:
           docker cp docassemble_webapp/docassemble/webapp/main/views.py da:"$WB/main/views.py"
           docker exec da bash -c "supervisorctl restart uwsgi"
           sleep 30
-          bash .github/workflows/e2e/seo_check.sh http://localhost:8080 expect-pass
+          bash .github/workflows/e2e/seo_check.sh http://localhost:8080 expect-pass https://law.example.org
 
       - name: Issue-19 - install FontAwesome CSS build; JS bundle gone, page weight must drop
         run: |

.github/workflows/e2e/seo_check.sh

@@ -1,12 +1,13 @@
 #!/usr/bin/env bash
 # Fail-first SEO check for fork issue #15.
 #
-# Usage: seo_check.sh <base_url> expect-fail|expect-pass
+# Usage: seo_check.sh <base_url> expect-fail|expect-pass [url_root]
 #
 # Requires the test interview test_seo.yml installed in docassemble.demo and
 # the server config to carry:
 #   dispatch: {seo: docassemble.demo:data/questions/test_seo.yml}
 #   social: {og: {locale: en_US}}      # og configured WITHOUT an image
+#   url root: <url_root>               # must differ from base_url for meaningful check
 #
 # Assertions against the live interview page and sitemap:
 #   1. <meta name="description"> carries the interview's metadata description,
@@ -15,19 +16,21 @@
 #   2. <link rel="canonical"> points at the clean interview entry URL.
 #   3. og:title is emitted even though no og:image is configured (ungated).
 #   4. /sitemap.xml lists the dispatch entry.
+#   5. Every <loc> in sitemap.xml starts with url_root (when url_root is given).
 #
 # expect-fail: every assertion must FAIL on the unpatched release. expect-pass:
 # every assertion must hold.
 set -euo pipefail
-base="$1"; mode="$2"
+base="$1"; mode="$2"; url_root="${3:-}"
 yaml="docassemble.demo:data/questions/test_seo.yml"
 page=$(curl -sL "$base/interview?i=$yaml")
 
 desc=$(echo "$page" | grep -c 'name="description" content="A test interview whose description must appear' || true)
 desc_total=$(echo "$page" | grep -c 'name="description"' || true)
 canon=$(echo "$page" | grep -c 'rel="canonical" href="' || true)
 og=$(echo "$page" | grep -c 'name="og:title"' || true)
-sitemap=$(curl -s "$base/sitemap.xml" | grep -cE '<loc>.*/start/seo/?</loc>' || true)
+sitemap_body=$(curl -s "$base/sitemap.xml")
+sitemap=$(echo "$sitemap_body" | grep -cE '<loc>.*/start/seo/?</loc>' || true)
 
 echo "description=$desc (total description tags=$desc_total) canonical=$canon og:title=$og sitemap=$sitemap"
 
@@ -43,6 +46,16 @@ case "$mode" in
       echo "FAIL: SEO fix not fully present"
       exit 1
     fi
+    # When url_root is provided, verify every <loc> entry starts with it
+    if [ -n "$url_root" ]; then
+      loc_count=$(echo "$sitemap_body" | grep -cE '<loc>' || true)
+      loc_with_root=$(echo "$sitemap_body" | grep -cE "<loc>${url_root}/" || true)
+      if [ "$loc_count" != "$loc_with_root" ] || [ "$loc_count" = "0" ]; then
+        echo "FAIL: sitemap <loc> entries do not all start with url root '$url_root' ($loc_with_root of $loc_count)"
+        exit 1
+      fi
+      echo "pass: all $loc_count sitemap <loc> entries start with $url_root"
+    fi
     echo "pass: description, canonical, ungated og:title, and sitemap entry all present" ;;
   *) echo "mode must be expect-fail or expect-pass"; exit 2 ;;
 esac

docassemble_webapp/docassemble/webapp/main/views.py

@@ -413,7 +413,16 @@ def favicon_browserconfig():
 def sitemap():
     """List the public, listed dispatch interviews so search engines can
     find every court form without crawling for them. Unlisted interviews
-    and anything not in `dispatch` are omitted."""
+    and anything not in `dispatch` are omitted.
+
+    Returns 404 when `url root` is not configured in the server config.
+    Sitemaps require absolute URLs per the sitemaps.org protocol; emitting
+    relative paths would produce an invalid sitemap that search engines
+    would reject. Returning 404 (rather than 500) signals that the sitemap
+    is intentionally unavailable, not broken."""
+    url_root = daconfig.get('url root', '').rstrip('/')
+    if not url_root:
+        return ('Sitemap unavailable: url root not configured', 404)
     urls = []
     for key, yaml_filename in sorted(daconfig.get('dispatch', {}).items()):
         try:
@@ -424,7 +433,7 @@ def sitemap():
             continue
         # Built from url root, not the request scheme/host, so a plain-http hop
         # behind a TLS-terminating proxy cannot emit http:// entries.
-        urls.append(daconfig.get('url root', '').rstrip('/') + url_for('interview.redirect_to_interview', dispatch=key))
+        urls.append(url_root + url_for('interview.redirect_to_interview', dispatch=key))
     body = '<?xml version="1.0" encoding="UTF-8"?>\n<urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">\n'
     for url in urls:
         body += '  <url><loc>' + url.replace('&', '&amp;') + '</loc></url>\n'