From f1c913de2b235108f33f1f9d18cb50d0b550f6b6 Mon Sep 17 00:00:00 2001 From: Brian Glass Date: Fri, 6 Nov 2026 14:18:48 -0500 Subject: [PATCH 1/4] Load the test data once per run, and shorten the API sweep Thirty-eight test classes each reloaded both fixtures (0.8s apiece), and the saint search tests backfilled slugs and rebuilt the search index before every test. A test runner now does all of that once, after the test database is created, as the Dockerfile does for production. The day API sweep in test_errors covers one year instead of seven; the fasting snapshot already covers ten years of Day. The tests now take 13s instead of 87-102s. Building the test database, which loads the scriptures, still takes about 38s. Co-Authored-By: Claude Opus 5.5 --- alexa/tests/test_intents.py | 2 -- alexa/tests/test_speech.py | 2 -- calendarium/tests/test_antiochian_readings.py | 4 --- calendarium/tests/test_api.py | 18 +++--------- calendarium/tests/test_commemoration_links.py | 8 ------ calendarium/tests/test_crawler_window.py | 2 -- .../tests/test_fasting_characterization.py | 2 -- calendarium/tests/test_feasts.py | 6 ---- calendarium/tests/test_feeds.py | 2 -- calendarium/tests/test_ical.py | 2 -- calendarium/tests/test_liturgics.py | 28 ------------------- calendarium/tests/test_views.py | 6 ---- commemorations/tests.py | 24 ---------------- mcp_svc/tests/test_tools.py | 22 --------------- orthocal/settings.py | 1 + orthocal/test_runner.py | 25 +++++++++++++++++ 16 files changed, 30 insertions(+), 124 deletions(-) create mode 100644 orthocal/test_runner.py diff --git a/alexa/tests/test_intents.py b/alexa/tests/test_intents.py index f2988987..508d7af9 100644 --- a/alexa/tests/test_intents.py +++ b/alexa/tests/test_intents.py @@ -16,8 +16,6 @@ class IntentTestCase(TestCase): - fixtures = ['calendarium.json', 'commemorations.json'] - def test_launch_intent(self): with open(BASE_DIR / 'data/launch_intent_envelope.json') as f: envelope = f.read() diff --git a/alexa/tests/test_speech.py b/alexa/tests/test_speech.py index f76174cb..2f5767cf 100644 --- a/alexa/tests/test_speech.py +++ b/alexa/tests/test_speech.py @@ -9,8 +9,6 @@ class SpeechTestCase(TestCase): - fixtures = ['calendarium.json', 'commemorations.json'] - def test_human_join(self): items = 'Anthony', 'Athanasius', 'Cyril' expected = 'Anthony, Athanasius and Cyril' diff --git a/calendarium/tests/test_antiochian_readings.py b/calendarium/tests/test_antiochian_readings.py index 162ef7d3..b34a10c3 100644 --- a/calendarium/tests/test_antiochian_readings.py +++ b/calendarium/tests/test_antiochian_readings.py @@ -120,8 +120,6 @@ def test_preserves_discontinuous_ranges(self): class TestExactGreekReadingBoundaries(TestCase): - fixtures = ['calendarium.json', 'commemorations.json'] - # Every field reported in the 2026 Antiochian/GOA comparison. CASES = ( (2, 9, 'Epistle', '1 John 2.18-29; 3.1-8'), @@ -201,8 +199,6 @@ class TestAntiochianReadingCharacterization(TestCase): docker compose run --rm local python tools/greek/antiochian_characterize.py --write """ - fixtures = ['calendarium.json', 'commemorations.json'] - def test_antiochian_comparison_is_unchanged(self): expected = GOLDEN.read_text().splitlines() entries = [line.split('\t')[:2] for line in expected] diff --git a/calendarium/tests/test_api.py b/calendarium/tests/test_api.py index 3a54810b..042f6c94 100644 --- a/calendarium/tests/test_api.py +++ b/calendarium/tests/test_api.py @@ -18,8 +18,6 @@ class DayAPITestCase(TestCase): - fixtures = ['calendarium.json', 'commemorations.json'] - def test_get_day(self): with open(BASE_DIR / 'data/last_bday.json') as f: expected = json.loads(f.read()) @@ -179,10 +177,10 @@ def test_calendar_month_julian_range_error_high(self): def test_errors(self): """We shouldn't have any errors in the API.""" - # This is just a brute force test to make sure we don't have any - # errors in the API. - start_dt = timezone.datetime(2023, 1, 1) - end_dt = timezone.datetime(2029, 12, 31) + # A brute force smoke test over one year. The fasting snapshot test + # covers ten years of Day itself. + start_dt = timezone.datetime(2027, 1, 1) + end_dt = timezone.datetime(2027, 12, 31) for dt in rrule(DAILY, dtstart=start_dt, until=end_dt): with self.subTest(dt=dt): url = reverse('api:get_calendar_day', kwargs={ @@ -216,8 +214,6 @@ def test_oembed_calendar(self): class MonthCacheTestCase(TestCase): """api._get_calendar_month caches each month's response.""" - fixtures = ['calendarium.json', 'commemorations.json'] - def setUp(self): cache.clear() @@ -251,8 +247,6 @@ def test_months_are_cached_separately(self): class DayCacheTestCase(TestCase): """api._get_calendar_day caches each day's response near today.""" - fixtures = ['calendarium.json', 'commemorations.json'] - def setUp(self): api.day_cache.clear() @@ -318,8 +312,6 @@ def test_invalid_date_is_still_a_404(self): class TrailingSlashTestCase(TestCase): """orthocal.middleware.append_api_slash and the APPEND_SLASH redirect""" - fixtures = ['calendarium.json', 'commemorations.json'] - def test_slashless_api_url_is_served_directly(self): url = reverse('api:get_calendar_day', kwargs={'cal': 'gregorian', 'year': 2022, 'month': 1, 'day': 7}) slashless = self.client.get(url.rstrip('/')) @@ -343,8 +335,6 @@ def test_slashless_page_url_redirects_cacheably(self): class RecordWorkerTestCase(TestCase): """orthocal.middleware.record_worker""" - fixtures = ['calendarium.json', 'commemorations.json'] - def test_transaction_is_tagged_with_the_worker(self): url = reverse('api:get_calendar_day', kwargs={'cal': 'gregorian', 'year': 2022, 'month': 1, 'day': 7}) diff --git a/calendarium/tests/test_commemoration_links.py b/calendarium/tests/test_commemoration_links.py index 00e94cc0..f87ea0b6 100644 --- a/calendarium/tests/test_commemoration_links.py +++ b/calendarium/tests/test_commemoration_links.py @@ -1,7 +1,6 @@ from datetime import date from types import SimpleNamespace -from django.core.management import call_command from django.test import TestCase from django.urls import reverse @@ -109,17 +108,10 @@ def test_passes_empty_stories_through(self): class StoryLinksOnPagesTestCase(TestCase): - fixtures = ['calendarium.json', 'commemorations.json'] - # The Jan 2 story of St Seraphim of Sarov mentions the saint he was named # for: "Hieromartyr Seraphim, Bishop of Phanarion (Dec. 4)". SERAPHIM = 5209 - def setUp(self): - # Slugs aren't in the fixture; the Dockerfile backfills them after - # loading it, and so does this, mirroring commemorations.tests. - call_command('backfill_saint_slugs') - def test_readings_page_links_story_dates(self): response = self.client.get(readings_url(date(2026, 1, 2))) self.assertContains(response, f'Dec. 4') diff --git a/calendarium/tests/test_crawler_window.py b/calendarium/tests/test_crawler_window.py index 399881ca..8a8a3d46 100644 --- a/calendarium/tests/test_crawler_window.py +++ b/calendarium/tests/test_crawler_window.py @@ -15,8 +15,6 @@ class CrawlerWindowTestCase(TestCase): """orthocal.middleware.finite_calendar_for_crawlers""" - fixtures = ['calendarium.json', 'commemorations.json'] - def setUp(self): self.this_year = timezone.localdate().year diff --git a/calendarium/tests/test_fasting_characterization.py b/calendarium/tests/test_fasting_characterization.py index 70eb89fa..c9b8cef0 100644 --- a/calendarium/tests/test_fasting_characterization.py +++ b/calendarium/tests/test_fasting_characterization.py @@ -31,8 +31,6 @@ class TestFastingCharacterization(TestCase): See docs/fasting-refactor-scope.md. """ - fixtures = ['calendarium.json'] - def render(self): lines = [] for tradition, name in ((Tradition.Slavic, 'slavic'), (Tradition.Greek, 'greek')): diff --git a/calendarium/tests/test_feasts.py b/calendarium/tests/test_feasts.py index b35c08a5..0848cf7f 100644 --- a/calendarium/tests/test_feasts.py +++ b/calendarium/tests/test_feasts.py @@ -9,8 +9,6 @@ class YearOfFeastsTestCase(TestCase): - fixtures = ['calendarium.json', 'commemorations.json'] - def test_gregorian(self): feasts, seasons = liturgics.year_of_feasts(2027, calendar=Calendar.Gregorian, tradition=Tradition.Slavic) @@ -69,8 +67,6 @@ def test_last_supported_year(self): class FeastsViewTestCase(TestCase): - fixtures = ['calendarium.json', 'commemorations.json'] - def test_default(self): response = self.client.get(reverse('feasts-default')) @@ -108,8 +104,6 @@ def test_remembers_calendar(self): class FeastsAPITestCase(TestCase): - fixtures = ['calendarium.json', 'commemorations.json'] - def test_feasts(self): response = self.client.get('/api/slavic/julian/feasts/2027/') self.assertEqual(response.status_code, 200) diff --git a/calendarium/tests/test_feeds.py b/calendarium/tests/test_feeds.py index d7468e9e..63971c32 100644 --- a/calendarium/tests/test_feeds.py +++ b/calendarium/tests/test_feeds.py @@ -10,8 +10,6 @@ class FeedTest(TestCase): - fixtures = ['calendarium.json'] - def setUp(self): # The feed view is wrapped in cache_page (see calendarium/api_urls.py), # backed by a real FileBasedCache outside local dev (local_settings.py's diff --git a/calendarium/tests/test_ical.py b/calendarium/tests/test_ical.py index 3f2c53d4..3123c823 100644 --- a/calendarium/tests/test_ical.py +++ b/calendarium/tests/test_ical.py @@ -18,8 +18,6 @@ class CalendarTest(TestCase): - fixtures = ['calendarium.json', 'commemorations.json'] - def test_ical(self): """ical endpoint should return 200.""" url = reverse('ical', kwargs={'cal': Calendar.Gregorian}) diff --git a/calendarium/tests/test_liturgics.py b/calendarium/tests/test_liturgics.py index 68e9ccbd..b1c22dbb 100644 --- a/calendarium/tests/test_liturgics.py +++ b/calendarium/tests/test_liturgics.py @@ -12,8 +12,6 @@ class TestYear(TestCase): - fixtures = ['calendarium.json', 'commemorations.json'] - def test_pascha(self): year = liturgics.SlavicYear(2018) pascha = datetools.gregorian_to_jdn(date(2018, 4, 8)) @@ -182,8 +180,6 @@ class TestTraditionOverlay(TestCase): Byzantine base -- these cover the overlay mechanism itself (Year class selection, Reading fallback/override).""" - fixtures = ['calendarium.json', 'commemorations.json'] - def test_slavic_and_greek_year_are_distinct_subclasses(self): # Note: SlavicYear/GreekYear are decorated with @lru_cache, which # (as with the pre-existing Year class) turns the name into a @@ -399,8 +395,6 @@ class TestGreekFasting(TestCase): the Nativity Fast, that difference is a Day-row data split, not a code difference in _apply_fasting_adjustments.""" - fixtures = ['calendarium.json', 'commemorations.json'] - def test_day_dispatches_to_tradition_specific_subclass(self): slavic = liturgics.Day(2026, 11, 20, tradition=Tradition.Slavic) greek = liturgics.Day(2026, 11, 20, tradition=Tradition.Greek) @@ -888,8 +882,6 @@ class TestGreekSundayTitles(TestCase): Pentecost; see GreekDay._apply_title_conventions. Expected names are antiochian.org's for the same dates.""" - fixtures = ['calendarium.json', 'commemorations.json'] - def test_sunday_titles(self): for (year, month, day), expected in ( ((2026, 6, 7), 'Sunday of All Saints'), @@ -929,8 +921,6 @@ class TestGreekWeekdayTitles(TestCase): being read; see GreekDay._apply_title_conventions. Expected numbers are antiochian.org's for the same dates.""" - fixtures = ['calendarium.json', 'commemorations.json'] - def test_weekday_titles(self): for (year, month, day), expected in ( ((2026, 9, 21), 'Monday of the 1st week of Luke'), @@ -956,8 +946,6 @@ class TestHasStory(TestCase): """_has_story decides whether a commemoration earns a clickable title and an entry in the story panel (see day.saint_links and day.stories).""" - fixtures = ['calendarium.json', 'commemorations.json'] - def test_none_and_empty_are_not_a_story(self): self.assertFalse(_has_story(SimpleNamespace(story=None))) self.assertFalse(_has_story(SimpleNamespace(story=''))) @@ -977,8 +965,6 @@ def test_every_story_has_visible_text(self): class TestDay(TestCase): - fixtures = ['calendarium.json', 'commemorations.json'] - def test_minimal_saints_keeps_three_of_the_full_list(self): """Oct 1 Greek has 6 commemorations; minimal_saints keeps 3 of them.""" @@ -1603,8 +1589,6 @@ class TestGreekPreTriodionWeekdayCycle(TestCase): asserted here. """ - fixtures = ['calendarium.json', 'commemorations.json'] - # Luke-section week -> weekday (Mon=0) -> Gospel, in this project's own # `Pericope.sdisplay` notation. Week 14 Friday follows goarch.org's and # antiochian.org's `Mark 10:24-32` in the Greek tradition. @@ -1685,8 +1669,6 @@ class TestGreekAnnualOrdoGospels(TestCase): overlay -- the only one in this codebase. """ - fixtures = ['calendarium.json', 'commemorations.json'] - # (year, month, day, expected Gospel), transcribed from goarch.org. # Deliberately spread across years whose ordo picks very different # Matthean Sundays, so a formula fitted to one shape cannot pass. @@ -1767,8 +1749,6 @@ class TestGreekMenaionReadings(TestCase): assumed, because two of these override a common Gospel. """ - fixtures = ['calendarium.json', 'commemorations.json'] - # (month, day, source, reading Greek must now have, what Slavic must have) # # The Slavic column is explicit per case rather than "must not contain the @@ -1834,8 +1814,6 @@ class TestGreekSundayPrecedence(TestCase): goarch.org published for that date. """ - fixtures = ['calendarium.json', 'commemorations.json'] - def test_abbreviated_pair(self): data = [ # The saint's Epistle, the Sunday's Gospel @@ -1880,8 +1858,6 @@ class TestOrdoJurisdictionLabels(TestCase): this project already gives a saint's reading standing beside the cycle's. """ - fixtures = ['calendarium.json', 'commemorations.json'] - def gospels(self, year, month, day, tradition=Tradition.Greek): pday = liturgics.Day(year, month, day, tradition=tradition) return [(r.pericope.sdisplay, r.desc) @@ -1959,8 +1935,6 @@ class TestAug28JobOfPochaev(TestCase): that is how most of the Greek Menaion data in this project was found. """ - fixtures = ['calendarium.json', 'commemorations.json'] - JOB_SET = ['Wis 3.1-9', 'Wis 5.15-6.3', 'Wis 4.7-15', 'Matt 11.27-30', 'Gal 5.22-6.2', 'Luke 6.17-23'] @@ -1993,8 +1967,6 @@ class TestDayOverrides(TestCase): These guard the second shape, which is the one that can rot quietly.""" # commemorations.json too: one override exists only to anchor them. - fixtures = ['calendarium.json', 'commemorations.json'] - @staticmethod def _overrides(): """(base, override) for every slot with a common row and a tradition one.""" diff --git a/calendarium/tests/test_views.py b/calendarium/tests/test_views.py index 1643b2cc..94aa2184 100644 --- a/calendarium/tests/test_views.py +++ b/calendarium/tests/test_views.py @@ -10,8 +10,6 @@ class TestReadingsView(TestCase): - fixtures = ['calendarium.json', 'commemorations.json'] - def test_gregorian_default(self): now = timezone.localtime() url = reverse('index') @@ -124,8 +122,6 @@ def test_greek_extra_sundays_overflow_does_not_500(self): class TestCalendarView(TestCase): - fixtures = ['calendarium.json', 'commemorations.json'] - def test_gregorian_default(self): now = timezone.localtime() this_month = date(now.year, now.month, 1) @@ -179,8 +175,6 @@ class TestTraditionRouting(TestCase): behaving exactly as they always have. """ - fixtures = ['calendarium.json', 'commemorations.json'] - def test_every_tradition_segment_resolves(self): for segment in ('slavic', 'oca', 'greek', 'goa', 'antiochian'): with self.subTest(segment=segment): diff --git a/commemorations/tests.py b/commemorations/tests.py index 16b4c4f5..088aaeb9 100644 --- a/commemorations/tests.py +++ b/commemorations/tests.py @@ -21,18 +21,6 @@ def test_known_variant_pairs_match(self): class SaintSearchTransliterationTestCase(TestCase): - fixtures = ['calendarium.json', 'commemorations.json'] - - def setUp(self): - # Fixture loading bypasses Saint.save() (same reason `slug` needs - # its own backfill command, see the model), so normalized_name and - # slug are blank on freshly test-loaded fixture data unless - # backfilled here -- mirrors what the Dockerfile does for a real - # deployment. - call_command('backfill_saint_slugs') - call_command('backfill_saint_normalized_names') - call_command('build_saint_search_index') - def test_greek_spelling_finds_latin_spelled_saint(self): response = self.client.get(reverse('saint-search'), {'q': 'Athanasios'}) @@ -74,13 +62,6 @@ def test_single_result_redirects_to_detail_page(self): class LifeSearchTestCase(TestCase): - fixtures = ['calendarium.json', 'commemorations.json'] - - def setUp(self): - call_command('backfill_saint_slugs') - call_command('backfill_saint_normalized_names') - call_command('build_saint_search_index') - def _search(self, query, **headers): return self.client.get(reverse('saint-search'), {'q': query}, headers=headers) @@ -185,11 +166,6 @@ def test_index_holds_text_without_markup(self): class CanonicalLinkTestCase(TestCase): - fixtures = ['calendarium.json', 'commemorations.json'] - - def setUp(self): - call_command('backfill_saint_slugs') - def test_content_pages_name_their_canonical_url(self): from commemorations.models import Saint diff --git a/mcp_svc/tests/test_tools.py b/mcp_svc/tests/test_tools.py index 450f0f57..8a6ed04b 100644 --- a/mcp_svc/tests/test_tools.py +++ b/mcp_svc/tests/test_tools.py @@ -1,4 +1,3 @@ -from django.core.management import call_command from django.test import TestCase from mcp.server.mcpserver.exceptions import ToolError @@ -7,18 +6,7 @@ from ..tools import get_day, get_saint, search_saints -def _backfill_saints(): - # Fixture loading bypasses Saint.save(), so slug and normalized_name are - # blank until backfilled, and the search index is empty until built -- - # mirrors the Dockerfile (see commemorations/tests.py). - call_command('backfill_saint_slugs') - call_command('backfill_saint_normalized_names') - call_command('build_saint_search_index') - - class GetDayTestCase(TestCase): - fixtures = ['calendarium.json', 'commemorations.json'] - async def test_normal_date(self): result = await get_day(2026, 7, 31) @@ -56,11 +44,6 @@ async def test_default_translation_is_lxx2012_web(self): class SearchSaintsTestCase(TestCase): - fixtures = ['calendarium.json', 'commemorations.json'] - - def setUp(self): - _backfill_saints() - async def test_finds_matching_saint(self): results = await search_saints('Seraphim of Sarov') @@ -158,11 +141,6 @@ async def test_moveable_commemoration_has_no_month_or_day(self): class GetSaintTestCase(TestCase): - fixtures = ['calendarium.json', 'commemorations.json'] - - def setUp(self): - _backfill_saints() - async def _slug(self, query): return (await search_saints(query))[0]['saint_slugs'][0] diff --git a/orthocal/settings.py b/orthocal/settings.py index 8f0a369f..fec5f998 100644 --- a/orthocal/settings.py +++ b/orthocal/settings.py @@ -20,6 +20,7 @@ from django.utils.translation import gettext_lazy as _ TESTING = sys.argv[1:2] == ['test'] +TEST_RUNNER = 'orthocal.test_runner.FixtureRunner' # Build paths inside the project like this: BASE_DIR / 'subdir'. BASE_DIR = Path(__file__).resolve().parent.parent diff --git a/orthocal/test_runner.py b/orthocal/test_runner.py new file mode 100644 index 00000000..fa543f30 --- /dev/null +++ b/orthocal/test_runner.py @@ -0,0 +1,25 @@ +from io import StringIO + +from django.core.management import call_command +from django.test.runner import DiscoverRunner + + +class FixtureRunner(DiscoverRunner): + """Load the calendar and commemoration data once for the whole run. + + Nearly every test class needs it, and reloading it for each one through + `TestCase.fixtures` cost half a minute a run. Each test still rolls back + its own changes. As in the Dockerfile, loading the fixtures bypasses + Saint.save(), so slugs and normalized names are backfilled and the search + index built afterwards.""" + + def setup_databases(self, **kwargs): + config = super().setup_databases(**kwargs) + + quiet = {'stdout': StringIO()} + call_command('loaddata', 'calendarium', 'commemorations', verbosity=0) + call_command('backfill_saint_slugs', **quiet) + call_command('backfill_saint_normalized_names', **quiet) + call_command('build_saint_search_index', **quiet) + + return config From 8ae845364590b66171977f00dc123065b7547600 Mon Sep 17 00:00:00 2001 From: Brian Glass Date: Fri, 6 Nov 2026 14:25:20 -0500 Subject: [PATCH 2/4] Start the tests from the database the Docker image built The image's db.sqlite3 already has the migrations applied, including the scripture loads that took about 38s of every run, and the fixtures, slugs and search index loaded. The test runner now copies it into place as the test database and keeps it, so a Docker test run takes 14s instead of 57s. Without a current db.sqlite3 it builds from scratch and loads the data once, as before. Co-Authored-By: Claude Opus 5.5 --- orthocal/test_runner.py | 50 ++++++++++++++++++++++++++++++++--------- 1 file changed, 39 insertions(+), 11 deletions(-) diff --git a/orthocal/test_runner.py b/orthocal/test_runner.py index fa543f30..5a2e8881 100644 --- a/orthocal/test_runner.py +++ b/orthocal/test_runner.py @@ -1,25 +1,53 @@ +import shutil + from io import StringIO +from pathlib import Path +from django.conf import settings from django.core.management import call_command from django.test.runner import DiscoverRunner +FIXTURES = [settings.BASE_DIR / 'fixtures' / f'{name}.json' for name in ('calendarium', 'commemorations')] + class FixtureRunner(DiscoverRunner): - """Load the calendar and commemoration data once for the whole run. + """Start the tests from the database the Docker image already built. - Nearly every test class needs it, and reloading it for each one through - `TestCase.fixtures` cost half a minute a run. Each test still rolls back - its own changes. As in the Dockerfile, loading the fixtures bypasses - Saint.save(), so slugs and normalized names are backfilled and the search - index built afterwards.""" + The image's db.sqlite3 has every migration applied -- including the three + that load the scriptures, about 40s -- and the calendar and commemoration + data loaded, with the slugs, normalized names and search index the + Dockerfile builds. The test database starts as a copy of it. Each test + still rolls back its own changes, and no test uses TransactionTestCase, + which would empty the tables. + + Without a current db.sqlite3 (one older than the fixtures, say), the test + database is built from scratch and the data loaded into it once, which is + still far cheaper than each test class loading its own fixtures.""" def setup_databases(self, **kwargs): + copied = self._copy_built_database() + if copied: + self.keepdb = True + config = super().setup_databases(**kwargs) - quiet = {'stdout': StringIO()} - call_command('loaddata', 'calendarium', 'commemorations', verbosity=0) - call_command('backfill_saint_slugs', **quiet) - call_command('backfill_saint_normalized_names', **quiet) - call_command('build_saint_search_index', **quiet) + if not copied: + quiet = {'stdout': StringIO()} + call_command('loaddata', *(path.stem for path in FIXTURES), verbosity=0) + call_command('backfill_saint_slugs', **quiet) + call_command('backfill_saint_normalized_names', **quiet) + call_command('build_saint_search_index', **quiet) return config + + def _copy_built_database(self): + database = settings.DATABASES['default'] + built, test = Path(database['NAME']), Path(database['TEST']['NAME']) + + if not built.exists() or test.exists(): + return False + if any(built.stat().st_mtime < path.stat().st_mtime for path in FIXTURES): + return False + + shutil.copyfile(built, test) + return True From fd3fac2d3f57c5863f64e7028b0594f7787d894e Mon Sep 17 00:00:00 2001 From: Brian Glass Date: Fri, 6 Nov 2026 14:34:19 -0500 Subject: [PATCH 3/4] Load the scriptures with expat and bulk_create parse_usfx went through pulldom, which builds a DOM node and attribute objects for every element and text run, and the migrations inserted the 171,116 verses one row at a time. The same parsing logic now runs from expat callbacks, and the verses go in with bulk_create. Loading the scriptures takes 6.4s instead of 31.2s, which every image build and every fresh test database pays; the verse table comes out byte for byte the same. Co-Authored-By: Claude Opus 5.5 --- bible/migrations/0002_load_scriptures.py | 17 ++--- bible/migrations/0004_load_lxx2012_web.py | 17 ++--- bible/migrations/0005_load_douay_rheims.py | 8 +-- bible/parse.py | 77 ++++++++++++++-------- 4 files changed, 70 insertions(+), 49 deletions(-) diff --git a/bible/migrations/0002_load_scriptures.py b/bible/migrations/0002_load_scriptures.py index 457a0d9f..c490dfad 100644 --- a/bible/migrations/0002_load_scriptures.py +++ b/bible/migrations/0002_load_scriptures.py @@ -7,14 +7,15 @@ def load_default_scriptures(apps, schema_editor): Verse = apps.get_model('bible', 'Verse') - for verse in parse_usfx('data/eng-kjv_usfx.xml'): - Verse.objects.create(language='en', **verse) - - for verse in parse_usfx('data/ron-rccv.usfx.xml'): - Verse.objects.create(language='ro', **verse) - - for verse in parse_usfx('data/srp1865_usfx.xml'): - Verse.objects.create(language='sr', **verse) + # bulk_create rather than one INSERT per verse: this runs in every image + # build, and the row-at-a-time version took most of the build's migrate. + for filename, language in ( + ('data/eng-kjv_usfx.xml', 'en'), + ('data/ron-rccv.usfx.xml', 'ro'), + ('data/srp1865_usfx.xml', 'sr'), + ): + Verse.objects.bulk_create( + Verse(language=language, **verse) for verse in parse_usfx(filename)) def unload_default_scriptures(apps, schema_editor): Verse = apps.get_model('bible', 'Verse') diff --git a/bible/migrations/0004_load_lxx2012_web.py b/bible/migrations/0004_load_lxx2012_web.py index 8b039975..46ec67c0 100644 --- a/bible/migrations/0004_load_lxx2012_web.py +++ b/bible/migrations/0004_load_lxx2012_web.py @@ -17,17 +17,12 @@ def load_lxx2012_web(apps, schema_editor): Verse = apps.get_model('bible', 'Verse') - for verse in parse_usfx('data/eng-lxx2012_usfx.xml'): - if verse['chapter'] is None or verse['verse'] is None: - continue - Verse.objects.create(language='en', translation=TRANSLATION, **verse) - - for verse in parse_usfx('data/eng-web_usfx.xml'): - if verse['book'] not in NT_BOOKS: - continue - if verse['chapter'] is None or verse['verse'] is None: - continue - Verse.objects.create(language='en', translation=TRANSLATION, **verse) + verses = list(parse_usfx('data/eng-lxx2012_usfx.xml')) + verses += [verse for verse in parse_usfx('data/eng-web_usfx.xml') if verse['book'] in NT_BOOKS] + + Verse.objects.bulk_create( + Verse(language='en', translation=TRANSLATION, **verse) for verse in verses + if verse['chapter'] is not None and verse['verse'] is not None) def unload_lxx2012_web(apps, schema_editor): diff --git a/bible/migrations/0005_load_douay_rheims.py b/bible/migrations/0005_load_douay_rheims.py index 2cd47c5b..f7d274cd 100644 --- a/bible/migrations/0005_load_douay_rheims.py +++ b/bible/migrations/0005_load_douay_rheims.py @@ -7,10 +7,10 @@ def load_douay_rheims(apps, schema_editor): Verse = apps.get_model('bible', 'Verse') - for verse in parse_usfx('data/eng-dra_usfx.xml'): - if verse['chapter'] is None or verse['verse'] is None: - continue - Verse.objects.create(language='en', translation=TRANSLATION, **verse) + Verse.objects.bulk_create( + Verse(language='en', translation=TRANSLATION, **verse) + for verse in parse_usfx('data/eng-dra_usfx.xml') + if verse['chapter'] is not None and verse['verse'] is not None) def unload_douay_rheims(apps, schema_editor): diff --git a/bible/parse.py b/bible/parse.py index 59ac31dd..92c6f987 100644 --- a/bible/parse.py +++ b/bible/parse.py @@ -1,6 +1,6 @@ import re -from xml.dom import pulldom +from xml.parsers import expat space_re = re.compile(r'\s+', flags=re.DOTALL) @@ -38,28 +38,35 @@ def make_verse(): return result - for event, node in pulldom.parse(filename): - match [event, node.nodeName]: + # Driven by expat directly. pulldom, which this used before, builds a DOM + # node and attribute objects for every element and text run, and that was + # most of the time the image build spends loading the scriptures. + verses = [] + + def start_element(name, attributes): + nonlocal book, chapter, verse, is_valid_content, was_valid_content, paragraph_start + + match name: # Book element - case [pulldom.START_ELEMENT, 'book']: + case 'book': if is_valid_content: - yield make_verse() + verses.append(make_verse()) - book = node.getAttribute('id') + book = attributes.get('id', '') # Chapter element - case [pulldom.START_ELEMENT, 'c']: + case 'c': if is_valid_content: - yield make_verse() + verses.append(make_verse()) - chapter = node.getAttribute('id') + chapter = attributes.get('id', '') # Verse elements - case [pulldom.START_ELEMENT, 'v']: + case 'v': if is_valid_content: - yield make_verse() + verses.append(make_verse()) - verse = node.getAttribute('id') + verse = attributes.get('id', '') if verse and '-' in verse: # A verse bridge (e.g. "1-2"), where the source merges # two verses into one printed unit -- store it under the @@ -67,11 +74,11 @@ def make_verse(): # combined text is already all in this one entry. verse = verse.split('-')[0] is_valid_content = True - case [pulldom.START_ELEMENT, 've']: - yield make_verse() + case 've': + verses.append(make_verse()) # paragraph element - case [pulldom.START_ELEMENT, 'p']: + case 'p': paragraph_start = True # Footnote and cross-reference elements -- (cross-reference, @@ -80,16 +87,34 @@ def make_verse(): # (footnote), so it needs the same is_valid_content suppression; # without it, the reference text gets appended straight into the # verse content. - case [pulldom.START_ELEMENT, 'f' | 'x']: + case 'f' | 'x': was_valid_content = is_valid_content is_valid_content = False - case [pulldom.END_ELEMENT, 'f' | 'x']: - # Restore whatever was in effect before the aside rather than - # assuming True -- these can appear in content (like Psalm - # title blocks) that isn't part of a verse. - is_valid_content = was_valid_content - - # Character content - case [pulldom.CHARACTERS, _]: - if is_valid_content: - strings.append(node.wholeText) + + def end_element(name): + nonlocal is_valid_content + + if name in ('f', 'x'): + # Restore whatever was in effect before the aside rather than + # assuming True -- these can appear in content (like Psalm + # title blocks) that isn't part of a verse. + is_valid_content = was_valid_content + + # Character content + def character_data(data): + if is_valid_content: + strings.append(data) + + parser = expat.ParserCreate() + parser.StartElementHandler = start_element + parser.EndElementHandler = end_element + parser.CharacterDataHandler = character_data + + with open(filename, 'rb') as f: + while chunk := f.read(1 << 16): + parser.Parse(chunk, False) + yield from verses + verses.clear() + + parser.Parse(b'', True) + yield from verses From c9fc5dc7588869cb7ca1eff05756295ddd9cd383 Mon Sep 17 00:00:00 2001 From: Brian Glass Date: Sat, 7 Nov 2026 09:15:07 -0500 Subject: [PATCH 4/4] Remove the unused ingest_usfx command It passed parse_usfx a language argument it never accepted, so it failed as soon as it ran. The scripture migrations load the Bibles instead. Co-Authored-By: Claude Opus 5.5 --- bible/management/__init__.py | 0 bible/management/commands/__init__.py | 0 bible/management/commands/ingest_usfx.py | 15 --------------- 3 files changed, 15 deletions(-) delete mode 100644 bible/management/__init__.py delete mode 100644 bible/management/commands/__init__.py delete mode 100644 bible/management/commands/ingest_usfx.py diff --git a/bible/management/__init__.py b/bible/management/__init__.py deleted file mode 100644 index e69de29b..00000000 diff --git a/bible/management/commands/__init__.py b/bible/management/commands/__init__.py deleted file mode 100644 index e69de29b..00000000 diff --git a/bible/management/commands/ingest_usfx.py b/bible/management/commands/ingest_usfx.py deleted file mode 100644 index 559a0949..00000000 --- a/bible/management/commands/ingest_usfx.py +++ /dev/null @@ -1,15 +0,0 @@ -from django.core.management.base import BaseCommand, CommandError - -from bible import models, parse - - -class Command(BaseCommand): - help = 'Ingest a USFX Bible.' - - def add_arguments(self, parser): - parser.add_argument('path') - parser.add_argument('-l', '--language') - - def handle(self, *args, **options): - for verse in parse.parse_usfx(options['path'], language=options['language']): - models.Verse.objects.create(**verse)