Speed up the test suite

alexa/tests/test_intents.py

@@ -16,8 +16,6 @@
 
 
 class IntentTestCase(TestCase):
-    fixtures = ['calendarium.json', 'commemorations.json']
-
     def test_launch_intent(self):
         with open(BASE_DIR / 'data/launch_intent_envelope.json') as f:
             envelope = f.read()

alexa/tests/test_speech.py

@@ -9,8 +9,6 @@
 
 
 class SpeechTestCase(TestCase):
-    fixtures = ['calendarium.json', 'commemorations.json']
-
     def test_human_join(self):
         items = 'Anthony', 'Athanasius', 'Cyril'
         expected = 'Anthony, Athanasius and Cyril'

bible/migrations/0002_load_scriptures.py

@@ -7,14 +7,15 @@
 def load_default_scriptures(apps, schema_editor):
     Verse = apps.get_model('bible', 'Verse')
 
-    for verse in parse_usfx('data/eng-kjv_usfx.xml'):
-        Verse.objects.create(language='en', **verse)
-
-    for verse in parse_usfx('data/ron-rccv.usfx.xml'):
-        Verse.objects.create(language='ro', **verse)
-
-    for verse in parse_usfx('data/srp1865_usfx.xml'):
-        Verse.objects.create(language='sr', **verse)
+    # bulk_create rather than one INSERT per verse: this runs in every image
+    # build, and the row-at-a-time version took most of the build's migrate.
+    for filename, language in (
+        ('data/eng-kjv_usfx.xml', 'en'),
+        ('data/ron-rccv.usfx.xml', 'ro'),
+        ('data/srp1865_usfx.xml', 'sr'),
+    ):
+        Verse.objects.bulk_create(
+            Verse(language=language, **verse) for verse in parse_usfx(filename))
 
 def unload_default_scriptures(apps, schema_editor):
     Verse = apps.get_model('bible', 'Verse')

bible/migrations/0004_load_lxx2012_web.py

@@ -17,17 +17,12 @@
 def load_lxx2012_web(apps, schema_editor):
     Verse = apps.get_model('bible', 'Verse')
 
-    for verse in parse_usfx('data/eng-lxx2012_usfx.xml'):
-        if verse['chapter'] is None or verse['verse'] is None:
-            continue
-        Verse.objects.create(language='en', translation=TRANSLATION, **verse)
-
-    for verse in parse_usfx('data/eng-web_usfx.xml'):
-        if verse['book'] not in NT_BOOKS:
-            continue
-        if verse['chapter'] is None or verse['verse'] is None:
-            continue
-        Verse.objects.create(language='en', translation=TRANSLATION, **verse)
+    verses = list(parse_usfx('data/eng-lxx2012_usfx.xml'))
+    verses += [verse for verse in parse_usfx('data/eng-web_usfx.xml') if verse['book'] in NT_BOOKS]
+
+    Verse.objects.bulk_create(
+        Verse(language='en', translation=TRANSLATION, **verse) for verse in verses
+        if verse['chapter'] is not None and verse['verse'] is not None)
 
 
 def unload_lxx2012_web(apps, schema_editor):

bible/migrations/0005_load_douay_rheims.py

@@ -7,10 +7,10 @@
 
 def load_douay_rheims(apps, schema_editor):
     Verse = apps.get_model('bible', 'Verse')
-    for verse in parse_usfx('data/eng-dra_usfx.xml'):
-        if verse['chapter'] is None or verse['verse'] is None:
-            continue
-        Verse.objects.create(language='en', translation=TRANSLATION, **verse)
+    Verse.objects.bulk_create(
+        Verse(language='en', translation=TRANSLATION, **verse)
+        for verse in parse_usfx('data/eng-dra_usfx.xml')
+        if verse['chapter'] is not None and verse['verse'] is not None)
 
 
 def unload_douay_rheims(apps, schema_editor):

bible/parse.py

@@ -1,6 +1,6 @@
 import re
 
-from xml.dom import pulldom
+from xml.parsers import expat
 
 space_re = re.compile(r'\s+', flags=re.DOTALL)
 
@@ -38,40 +38,47 @@ def make_verse():
 
         return result
 
-    for event, node in pulldom.parse(filename):
-        match [event, node.nodeName]:
+    # Driven by expat directly. pulldom, which this used before, builds a DOM
+    # node and attribute objects for every element and text run, and that was
+    # most of the time the image build spends loading the scriptures.
+    verses = []
+
+    def start_element(name, attributes):
+        nonlocal book, chapter, verse, is_valid_content, was_valid_content, paragraph_start
+
+        match name:
             # Book element
-            case [pulldom.START_ELEMENT, 'book']:
+            case 'book':
                 if is_valid_content:
-                    yield make_verse()
+                    verses.append(make_verse())
 
-                book = node.getAttribute('id')
+                book = attributes.get('id', '')
 
             # Chapter element
-            case [pulldom.START_ELEMENT, 'c']:
+            case 'c':
                 if is_valid_content:
-                    yield make_verse()
+                    verses.append(make_verse())
 
-                chapter = node.getAttribute('id')
+                chapter = attributes.get('id', '')
 
             # Verse elements
-            case [pulldom.START_ELEMENT, 'v']:
+            case 'v':
                 if is_valid_content:
-                    yield make_verse()
+                    verses.append(make_verse())
 
-                verse = node.getAttribute('id')
+                verse = attributes.get('id', '')
                 if verse and '-' in verse:
                     # A verse bridge (e.g. "1-2"), where the source merges
                     # two verses into one printed unit -- store it under the
                     # first number rather than failing to parse an int; the
                     # combined text is already all in this one entry.
                     verse = verse.split('-')[0]
                 is_valid_content = True
-            case [pulldom.START_ELEMENT, 've']:
-                yield make_verse()
+            case 've':
+                verses.append(make_verse())
 
             # paragraph element
-            case [pulldom.START_ELEMENT, 'p']:
+            case 'p':
                 paragraph_start = True
 
             # Footnote and cross-reference elements -- <x> (cross-reference,
@@ -80,16 +87,34 @@ def make_verse():
             # (footnote), so it needs the same is_valid_content suppression;
             # without it, the reference text gets appended straight into the
             # verse content.
-            case [pulldom.START_ELEMENT, 'f' | 'x']:
+            case 'f' | 'x':
                 was_valid_content = is_valid_content
                 is_valid_content = False
-            case [pulldom.END_ELEMENT, 'f' | 'x']:
-                # Restore whatever was in effect before the aside rather than
-                # assuming True -- these can appear in content (like Psalm
-                # title <d> blocks) that isn't part of a verse.
-                is_valid_content = was_valid_content
-
-            # Character content
-            case [pulldom.CHARACTERS, _]:
-                if is_valid_content:
-                    strings.append(node.wholeText)
+
+    def end_element(name):
+        nonlocal is_valid_content
+
+        if name in ('f', 'x'):
+            # Restore whatever was in effect before the aside rather than
+            # assuming True -- these can appear in content (like Psalm
+            # title <d> blocks) that isn't part of a verse.
+            is_valid_content = was_valid_content
+
+    # Character content
+    def character_data(data):
+        if is_valid_content:
+            strings.append(data)
+
+    parser = expat.ParserCreate()
+    parser.StartElementHandler = start_element
+    parser.EndElementHandler = end_element
+    parser.CharacterDataHandler = character_data
+
+    with open(filename, 'rb') as f:
+        while chunk := f.read(1 << 16):
+            parser.Parse(chunk, False)
+            yield from verses
+            verses.clear()
+
+    parser.Parse(b'', True)
+    yield from verses

calendarium/tests/test_antiochian_readings.py

@@ -120,8 +120,6 @@ def test_preserves_discontinuous_ranges(self):
 
 
 class TestExactGreekReadingBoundaries(TestCase):
-    fixtures = ['calendarium.json', 'commemorations.json']
-
     # Every field reported in the 2026 Antiochian/GOA comparison.
     CASES = (
         (2, 9, 'Epistle', '1 John 2.18-29; 3.1-8'),
@@ -201,8 +199,6 @@ class TestAntiochianReadingCharacterization(TestCase):
         docker compose run --rm local python tools/greek/antiochian_characterize.py --write
     """
 
-    fixtures = ['calendarium.json', 'commemorations.json']
-
     def test_antiochian_comparison_is_unchanged(self):
         expected = GOLDEN.read_text().splitlines()
         entries = [line.split('\t')[:2] for line in expected]

calendarium/tests/test_api.py

@@ -18,8 +18,6 @@
 
 
 class DayAPITestCase(TestCase):
-    fixtures = ['calendarium.json', 'commemorations.json']
-
     def test_get_day(self):
         with open(BASE_DIR / 'data/last_bday.json') as f:
             expected = json.loads(f.read())
@@ -179,10 +177,10 @@ def test_calendar_month_julian_range_error_high(self):
     def test_errors(self):
         """We shouldn't have any errors in the API."""
 
-        # This is just a brute force test to make sure we don't have any
-        # errors in the API.
-        start_dt = timezone.datetime(2023, 1, 1)
-        end_dt = timezone.datetime(2029, 12, 31)
+        # A brute force smoke test over one year. The fasting snapshot test
+        # covers ten years of Day itself.
+        start_dt = timezone.datetime(2027, 1, 1)
+        end_dt = timezone.datetime(2027, 12, 31)
         for dt in rrule(DAILY, dtstart=start_dt, until=end_dt):
             with self.subTest(dt=dt):
                 url = reverse('api:get_calendar_day', kwargs={
@@ -216,8 +214,6 @@ def test_oembed_calendar(self):
 class MonthCacheTestCase(TestCase):
     """api._get_calendar_month caches each month's response."""
 
-    fixtures = ['calendarium.json', 'commemorations.json']
-
     def setUp(self):
         cache.clear()
 
@@ -251,8 +247,6 @@ def test_months_are_cached_separately(self):
 class DayCacheTestCase(TestCase):
     """api._get_calendar_day caches each day's response near today."""
 
-    fixtures = ['calendarium.json', 'commemorations.json']
-
     def setUp(self):
         api.day_cache.clear()
 
@@ -318,8 +312,6 @@ def test_invalid_date_is_still_a_404(self):
 class TrailingSlashTestCase(TestCase):
     """orthocal.middleware.append_api_slash and the APPEND_SLASH redirect"""
 
-    fixtures = ['calendarium.json', 'commemorations.json']
-
     def test_slashless_api_url_is_served_directly(self):
         url = reverse('api:get_calendar_day', kwargs={'cal': 'gregorian', 'year': 2022, 'month': 1, 'day': 7})
         slashless = self.client.get(url.rstrip('/'))
@@ -343,8 +335,6 @@ def test_slashless_page_url_redirects_cacheably(self):
 class RecordWorkerTestCase(TestCase):
     """orthocal.middleware.record_worker"""
 
-    fixtures = ['calendarium.json', 'commemorations.json']
-
     def test_transaction_is_tagged_with_the_worker(self):
         url = reverse('api:get_calendar_day', kwargs={'cal': 'gregorian', 'year': 2022, 'month': 1, 'day': 7})
 

calendarium/tests/test_commemoration_links.py

@@ -1,7 +1,6 @@
 from datetime import date
 from types import SimpleNamespace
 
-from django.core.management import call_command
 from django.test import TestCase
 from django.urls import reverse
 
@@ -109,17 +108,10 @@ def test_passes_empty_stories_through(self):
 
 
 class StoryLinksOnPagesTestCase(TestCase):
-    fixtures = ['calendarium.json', 'commemorations.json']
-
     # The Jan 2 story of St Seraphim of Sarov mentions the saint he was named
     # for: "Hieromartyr Seraphim, Bishop of Phanarion (Dec. 4)".
     SERAPHIM = 5209
 
-    def setUp(self):
-        # Slugs aren't in the fixture; the Dockerfile backfills them after
-        # loading it, and so does this, mirroring commemorations.tests.
-        call_command('backfill_saint_slugs')
-
     def test_readings_page_links_story_dates(self):
         response = self.client.get(readings_url(date(2026, 1, 2)))
         self.assertContains(response, f'<a class="commemoration-date" href="{readings_url(date(2025, 12, 4))}">Dec. 4</a>')

calendarium/tests/test_crawler_window.py

@@ -15,8 +15,6 @@
 class CrawlerWindowTestCase(TestCase):
     """orthocal.middleware.finite_calendar_for_crawlers"""
 
-    fixtures = ['calendarium.json', 'commemorations.json']
-
     def setUp(self):
         self.this_year = timezone.localdate().year
 

calendarium/tests/test_fasting_characterization.py

@@ -31,8 +31,6 @@ class TestFastingCharacterization(TestCase):
     See docs/fasting-refactor-scope.md.
     """
 
-    fixtures = ['calendarium.json']
-
     def render(self):
         lines = []
         for tradition, name in ((Tradition.Slavic, 'slavic'), (Tradition.Greek, 'greek')):

calendarium/tests/test_feasts.py

@@ -9,8 +9,6 @@
 
 
 class YearOfFeastsTestCase(TestCase):
-    fixtures = ['calendarium.json', 'commemorations.json']
-
     def test_gregorian(self):
         feasts, seasons = liturgics.year_of_feasts(2027, calendar=Calendar.Gregorian, tradition=Tradition.Slavic)
 
@@ -69,8 +67,6 @@ def test_last_supported_year(self):
 
 
 class FeastsViewTestCase(TestCase):
-    fixtures = ['calendarium.json', 'commemorations.json']
-
     def test_default(self):
         response = self.client.get(reverse('feasts-default'))
 
@@ -108,8 +104,6 @@ def test_remembers_calendar(self):
 
 
 class FeastsAPITestCase(TestCase):
-    fixtures = ['calendarium.json', 'commemorations.json']
-
     def test_feasts(self):
         response = self.client.get('/api/slavic/julian/feasts/2027/')
         self.assertEqual(response.status_code, 200)

calendarium/tests/test_feeds.py

@@ -10,8 +10,6 @@
 
 
 class FeedTest(TestCase):
-    fixtures = ['calendarium.json']
-
     def setUp(self):
         # The feed view is wrapped in cache_page (see calendarium/api_urls.py),
         # backed by a real FileBasedCache outside local dev (local_settings.py's

calendarium/tests/test_ical.py

@@ -18,8 +18,6 @@
 
 
 class CalendarTest(TestCase):
-    fixtures = ['calendarium.json', 'commemorations.json']
-
     def test_ical(self):
         """ical endpoint should return 200."""
         url = reverse('ical', kwargs={'cal': Calendar.Gregorian})