Audit the Slavic readings and commemorations against oca.org

.gitignore

@@ -9,3 +9,7 @@ ve
 .firebase
 static
 default-cache
+
+# Raw HTML cache for the oca.org harvests; the parsed JSON beside it is
+# what matters, and this is re-fetchable (slowly -- see tools/oca/README.md).
+data/oca_raw/_cache/

calendarium/liturgics/day.py

@@ -10,6 +10,10 @@
 
 from .. import datetools, models
 from ..datetools import Calendar, Tradition, Weekday, FastLevels, FastLevelDesc, FastExceptions, FeastLevels, FloatIndex
+
+# FloatIndex values start at 1001, well above any Pascha-distance, so a reading
+# carrying one is a floating commemoration rather than part of the daily cycle.
+FLOAT_PDIST = 1000
 from commemorations.models import DayCommemoration
 
 from .year import SlavicYear, GreekYear
@@ -673,6 +677,21 @@ async def _add_ordo_alternatives(self, fetch_content=False):
 
     get_readings = async_to_sync(aget_readings)
 
+    @staticmethod
+    def _first_epistle_and_gospel(readings):
+        """The first Epistle and the first Gospel at or after it, or None."""
+        sources = [r.source for r in readings]
+        if 'Epistle' not in sources or 'Gospel' not in sources:
+            return None
+        epistle = readings[start := sources.index('Epistle')]
+        try:
+            gospel = readings[sources.index('Gospel', start)]
+        except ValueError:
+            # Some epistles are out of order in the data; be forgiving rather
+            # than drop the gospel entirely.
+            gospel = readings[sources.index('Gospel')]
+        return [epistle, gospel]
+
     async def aget_abbreviated_readings(self, fetch_content=False):
         """Return an abbreviated list of lectionary readings."""
 
@@ -724,21 +743,45 @@ async def aget_abbreviated_readings(self, fetch_content=False):
         readings = [reading async for reading in queryset.order_by('ordering')]
         readings = _prefer_tradition(readings, self.tradition)
 
-        # Include only the first Epistle and Gospel if we have them.
-        sources = [r.source for r in readings]
-        if 'Epistle' in sources and 'Gospel' in sources:
-            epistle = readings[start := sources.index('Epistle')]
-            try:
-                # Attempt to fetch the gospel that comes after the selected epistle
-                gospel = readings[sources.index('Gospel', start)]
-            except ValueError:
-                # If this goes wrong, we'll take any epistle. This shouldn't
-                # happen, but there are some epistles out of order in the
-                # database. This should be fixed in the data, but for now we'll
-                # be forgiving.
-                gospel = readings[sources.index('Gospel')]
-
-            readings = [epistle, gospel]
+        # Reduce to a single Epistle and Gospel -- this is what the Alexa skill
+        # speaks, so it has to be the pair that characterises the day.
+        #
+        # Which pair that is depends on rank, and the thresholds below are
+        # measured rather than assumed. Against antiochian.org and goarch.org,
+        # which each publish exactly one pair a day, on days that have a
+        # proper: at feast level 6 and above the proper always wins, at level 3
+        # to 5 it wins on 88% of weekdays, and at levels 0-2 the ordinary daily
+        # reading wins more often than not.
+        #
+        # Sunday is the exception that decides where this logic lives. On a
+        # Sunday the resurrectional reading takes precedence over almost any
+        # saint -- 7 of 8 observed -- and that cannot be expressed in the
+        # static `ordering` column, because whether a fixed date falls on a
+        # Sunday changes from year to year. See docs/oca-audit.md.
+        # Floating commemorations rank first, ahead of even the Sunday rule.
+        # They are not a saint landing on a day, they *are* the day: the
+        # memorial Saturdays, the Sundays of the Forefathers and of the
+        # Fathers, the Saturdays and Sundays before and after a great feast.
+        # oca.org prints their readings ahead of the ordinary ones -- Demetrius
+        # Saturday leads with the Departed pair, the Saturday before Nativity
+        # with its own -- and the Sunday statistic that governs fixed-date
+        # propers below was measured on saints displacing a Sunday, which is a
+        # different question.
+        floats = [r for r in readings if r.pdist and r.pdist >= FLOAT_PDIST]
+        propers = [r for r in readings if r.month and r not in floats]
+        ordinary = [r for r in readings if r not in floats and not r.month]
+
+        prefer_propers = self.feast_level >= 3 and self.weekday != Weekday.Sunday
+        groups = [floats] + ([propers, ordinary] if prefer_propers else [ordinary, propers])
+        for group in groups:
+            pair = self._first_epistle_and_gospel(group)
+            if pair:
+                readings = pair
+                break
+        else:
+            pair = self._first_epistle_and_gospel(readings)
+            if pair:
+                readings = pair
 
         if fetch_content:
             for reading in readings:

calendarium/models.py

@@ -106,14 +106,57 @@ async def aget_passage(self, language='en', translation=None):
             return self.passage
 
     def get_passage(self, language='en', translation=None):
+        # Two pericopes -- Composite 17 and 18, the Entrance of the Theotokos
+        # Vespers lessons -- carry a deliberate zero-width space (U+200B) in
+        # front of "Composite" so this match FAILS and they fall through to the
+        # scripture lookup below. It is invisible on the page and it is load
+        # bearing; do not "clean" it out of the fixture.
+        #
+        # It is a stopgap, not a design. The Composite table has no rows 17 or
+        # 18, and d4d5d54 (Nov 2023) moved a hardcoded `not in ('17', '18')`
+        # out of this function and into the data. A real composite is the
+        # better representation and should replace this if the text is ever
+        # obtained.
+        #
+        # Not for want of a parser. bible.models.lookup_reference handles
+        # out-of-order segments: it renders "Lev 26:3-12, 14-17, 19-20, 22, 33,
+        # 23-25" with verse 33 ahead of 23-25 as written, and Composite 20's
+        # "Isaiah 55:1; 12:3-4; 55:2-13" jumping chapters backwards and
+        # forwards. Order follows the OR terms in the generated query rather
+        # than an explicit order_by (there is no Meta.ordering on Verse), and
+        # production already depends on it for the Holy Thursday composite
+        # gospel, a five-segment cross-book reference.
+        #
+        # The obstacle is that for most composites NO verse list exists to
+        # write down. Archimandrite Ephrem's Prophetologion, which is where
+        # this text comes from, gives specifications like "Proverbs 10:31-32 &
+        # Selection", "40:1 and selection", "3 & 4 and selection" -- he did not
+        # enumerate them, so the stored text IS the specification. Only
+        # Composite 24 is fully spelled out there, and its sdisplay carries
+        # that spec.
+        #
+        # Consequently EIGHT sdisplay values here are chapter-only and wrong if
+        # they are ever resolved: "Isa 40, 41, 45, 48, 54" fetches 123 verses
+        # where the reading is a short selection, and "3 Kgs 18, 19" fetches
+        # 67. They are inert today because those pericopes route through the
+        # Composite table. Do not put a zero-width space on one of them.
+        #
+        # Why 17 and 18 are missing: they are the Slavic propers for that feast
+        # (Exodus 40; 3 Kingdoms 7-8). The composites we do have come from
+        # Archimandrite Ephrem Lash's Prophetologion, and his book assigns the
+        # Greek Marian set to the Entrance instead -- Genesis 28, Ezekiel 43,
+        # Proverbs 9 -- so these two readings simply are not in it. Only the
+        # Ezekiel overlaps, which is why it alone is a plain reference here.
         match = re.match(r'Composite (\d+)', self.display)
         if match:
             return Composite.objects.filter(
                     composite_num=match.group(1)
             ).annotate(
                     # Make the composite look like a Verse instance. Composite
-                    # readings only have one hardcoded (KJV-sourced) content
-                    # column, regardless of the requested translation.
+                    # readings only have one hardcoded content column,
+                    # regardless of the requested translation. That text is
+                    # Archimandrite Ephrem Lash's, not KJV as this comment
+                    # previously said -- see ~/src/anastasis.
                     book=models.Value(''),
                     chapter=models.Value(1),
                     verse=models.Value(1),

calendarium/tests/data/january.json

@@ -28,8 +28,8 @@
         ],
         "service_notes": null,
         "abbreviated_reading_indices": [
-            5,
-            8
+            6,
+            9
         ],
         "readings": [
             {
@@ -64,14 +64,6 @@
                 "short_display": "John 10.9-16",
                 "passage": null
             },
-            {
-                "source": "Epistle",
-                "book": "Apostol",
-                "description": "Saturday before Theophany",
-                "display": "1 Timothy 3.14-4.5",
-                "short_display": "1 Tim 3.14-4.5",
-                "passage": null
-            },
             {
                 "source": "Epistle",
                 "book": "Apostol",
@@ -89,11 +81,11 @@
                 "passage": null
             },
             {
-                "source": "Gospel",
-                "book": "Matthew",
+                "source": "Epistle",
+                "book": "Apostol",
                 "description": "Saturday before Theophany",
-                "display": "Matthew 3.1-11",
-                "short_display": "Matt 3.1-11",
+                "display": "1 Timothy 3.14-4.5",
+                "short_display": "1 Tim 3.14-4.5",
                 "passage": null
             },
             {
@@ -111,6 +103,14 @@
                 "display": "Luke 6.17-23",
                 "short_display": "Luke 6.17-23",
                 "passage": null
+            },
+            {
+                "source": "Gospel",
+                "book": "Matthew",
+                "description": "Saturday before Theophany",
+                "display": "Matthew 3.1-11",
+                "short_display": "Matt 3.1-11",
+                "passage": null
             }
         ]
     },
@@ -139,6 +139,7 @@
         "fast_abstentions": [],
         "saints": [
             "Repose of St Seraphim of Sarov",
+            "Sylvester of the Kiev Caves",
             "St Sylvester, Pope of Rome (335)",
             "Venerable Juliana of Lazarevskoye (1604)",
             "Holy New Martyr George the Georgian (1770)"
@@ -290,6 +291,9 @@
         "fast_abstentions": [],
         "saints": [
             "Synaxis of Seventy Apostles",
+            "Our Righteous Father Theoctistus, Abbot of Cucomo in Sicily",
+            "Nikephoros the Leper",
+            "Eustathios Archbishop of Serbia",
             "The Ethiopian Eunuch of Queen Candace",
             "Venerable Apollinaria (5th c.)"
         ],
@@ -347,6 +351,7 @@
             "oil"
         ],
         "saints": [
+            "Apollinaria of Egypt",
             "Holy Martyrs Theopemptus and Theonas (ca. 290)",
             "Our Venerable Mother Syncletike (4th c.)"
         ],
@@ -803,6 +808,7 @@
         "fast_abstentions": [],
         "saints": [
             "Ven. George the Chozebite",
+            "Gregory of the Kiev Caves",
             "Our Venerable Mother Domnica (Domnina) (ca. 474)",
             "Saint Atticus, Patriarch of Constantinople (425)",
             "Saint Severinus (482)"
@@ -918,7 +924,7 @@
         "titles": [
             "Monday of the 30th week after Pentecost"
         ],
-        "summary_title": "St Gregory of Nyssa; St Theophan the Recluse",
+        "summary_title": "St Gregory of Nyssa; Dometian, Bishop of Melitene; St Theophan the Recluse",
         "feast_level": 4,
         "feast_level_description": "Red cross (polyeleos typikon symbol)",
         "feasts": null,
@@ -929,6 +935,7 @@
         "fast_abstentions": [],
         "saints": [
             "St Gregory of Nyssa",
+            "Dometian, Bishop of Melitene",
             "St Theophan the Recluse"
         ],
         "service_notes": null,
@@ -982,8 +989,8 @@
         ],
         "service_notes": null,
         "abbreviated_reading_indices": [
-            4,
-            6
+            5,
+            7
         ],
         "readings": [
             {
@@ -1348,8 +1355,8 @@
         ],
         "service_notes": null,
         "abbreviated_reading_indices": [
-            4,
-            6
+            5,
+            7
         ],
         "readings": [
             {
@@ -1549,8 +1556,8 @@
         ],
         "service_notes": null,
         "abbreviated_reading_indices": [
-            4,
-            6
+            5,
+            7
         ],
         "readings": [
             {
@@ -1854,8 +1861,8 @@
         "saints": null,
         "service_notes": null,
         "abbreviated_reading_indices": [
-            4,
-            6
+            5,
+            7
         ],
         "readings": [
             {
@@ -2006,8 +2013,8 @@
         "saints": null,
         "service_notes": null,
         "abbreviated_reading_indices": [
-            4,
-            6
+            5,
+            7
         ],
         "readings": [
             {
@@ -2087,7 +2094,7 @@
         "titles": [
             "Friday of the 32nd week after Pentecost"
         ],
-        "summary_title": "Ven. Ephrem the Syrian; Our Holy Father Isaac the Syrian, bishop of Nineveh (7th c.)",
+        "summary_title": "Ven. Ephrem the Syrian; Theodosius of Totma; Our Holy Father Isaac the Syrian, bishop of Nineveh (7th c.)",
         "feast_level": 2,
         "feast_level_description": "Black squigg (6-stich typikon symbol)",
         "feasts": null,
@@ -2105,6 +2112,7 @@
         ],
         "saints": [
             "Ven. Ephrem the Syrian",
+            "Theodosius of Totma",
             "Our Holy Father Isaac the Syrian, bishop of Nineveh (7th c.)"
         ],
         "service_notes": null,