OpenConceptLab/ocl_online#476 | Covering list indexes so deep pages walk their OFFSET in the index

core/concepts/migrations/0090_concepts_repo_list_ids_idx.py

@@ -0,0 +1,55 @@
+"""
+Adds concepts_repo_list_ids_idx: concepts_repo_list_idx's key and predicate with INCLUDE (id), so a deep page of a source's HEAD
+concepts list can walk its OFFSET in the index alone, without reading every skipped row from the table
+(OpenConceptLab/ocl_online#476). concepts_repo_list_idx stays until EXPLAIN on staging and production shows the new index in use;
+a later migration drops it.
+
+On a large table, build the index by hand before deploying: every API task runs `migrate` before it serves, and a
+build that outlasts the health check's grace leaves an INVALID index behind. This migration then only checks it. It
+builds the index itself only on a small table (a fresh, test or dev database).
+"""
+from django.db import migrations
+
+TABLE = 'concepts'
+NEW_INDEX = 'concepts_repo_list_ids_idx'
+CREATE_NEW_INDEX = (
+    f'create index concurrently if not exists {NEW_INDEX} on {TABLE} (parent_id, updated_at desc) include (id) '
+    'where not retired and is_active and id = versioned_object_id;'
+)
+MAX_ROWS_TO_BUILD_HERE = 3_000_000
+
+
+def add_index(apps, schema_editor):  # pylint: disable=unused-argument
+    with schema_editor.connection.cursor() as cursor:
+        cursor.execute('select to_regclass(%s) is not null', [NEW_INDEX])
+        if not cursor.fetchone()[0]:
+            cursor.execute('select reltuples from pg_class where oid = %s::regclass', [TABLE])
+            rows = cursor.fetchone()[0]
+            if rows > MAX_ROWS_TO_BUILD_HERE:
+                raise RuntimeError(
+                    f'{TABLE} has about {rows:,.0f} rows: build {NEW_INDEX} by hand before deploying, '
+                    f'then migrate again. {CREATE_NEW_INDEX}')
+            cursor.execute(CREATE_NEW_INDEX)
+
+        cursor.execute('select indisvalid from pg_index where indexrelid = %s::regclass', [NEW_INDEX])
+        if not cursor.fetchone()[0]:
+            raise RuntimeError(
+                f'{NEW_INDEX} is INVALID (an interrupted build). Run `drop index concurrently {NEW_INDEX};`, '
+                f'build it again, then migrate again. {CREATE_NEW_INDEX}')
+
+
+def drop_index(apps, schema_editor):  # pylint: disable=unused-argument
+    with schema_editor.connection.cursor() as cursor:
+        cursor.execute(f'drop index concurrently if exists {NEW_INDEX};')
+
+
+class Migration(migrations.Migration):
+    atomic = False  # CREATE/DROP INDEX CONCURRENTLY can't run in a transaction
+
+    dependencies = [
+        ('concepts', '0089_merge_20261005_0816'),
+    ]
+
+    operations = [
+        migrations.RunPython(add_index, drop_index),
+    ]

core/integration_tests/tests_list_indexes.py

@@ -0,0 +1,63 @@
+import importlib
+from unittest.mock import MagicMock
+
+from django.db import connection
+from django.test import SimpleTestCase, TestCase
+
+MIGRATIONS = (
+    'core.concepts.migrations.0090_concepts_repo_list_ids_idx',
+    'core.mappings.migrations.0061_mappings_repo_list_ids_idx',
+)
+
+
+class ListCoveringIndexTest(TestCase):
+    def test_list_indexes_cover_id(self):
+        with connection.cursor() as cursor:
+            for table in ('concepts', 'mappings'):
+                cursor.execute('select indexdef from pg_indexes where indexname = %s', [f'{table}_repo_list_ids_idx'])
+                definition = cursor.fetchone()[0]
+                self.assertIn('(parent_id, updated_at DESC) INCLUDE (id)', definition)
+                self.assertIn('WHERE ((NOT retired) AND is_active AND (id = versioned_object_id))', definition)
+
+                # the old index stays until EXPLAIN in production shows the new one in use (a later migration)
+                cursor.execute('select to_regclass(%s)', [f'{table}_repo_list_idx'])
+                self.assertIsNotNone(cursor.fetchone()[0])
+
+
+class ListCoveringIndexMigrationTest(SimpleTestCase):
+    """The migrations never build on a production-sized table, and never accept an invalid index."""
+    @staticmethod
+    def run_migration(module, *answers):
+        cursor = MagicMock()
+        cursor.fetchone.side_effect = answers
+        schema_editor = MagicMock()
+        schema_editor.connection.cursor.return_value.__enter__.return_value = cursor
+        error = None
+        try:
+            importlib.import_module(module).add_index(None, schema_editor)
+        except RuntimeError as ex:
+            error = str(ex)
+        return error, [call.args[0] for call in cursor.execute.call_args_list]
+
+    def test_refuses_to_build_on_a_large_table(self):
+        for module in MIGRATIONS:
+            error, statements = self.run_migration(module, (False,), (10_000_000,))
+            self.assertIn('by hand before deploying', error)
+            self.assertFalse([sql for sql in statements if 'create index' in sql])
+
+    def test_uses_an_index_built_by_hand(self):
+        for module in MIGRATIONS:
+            error, statements = self.run_migration(module, (True,), (True,))
+            self.assertIsNone(error)
+            self.assertFalse([sql for sql in statements if 'create index' in sql])
+
+    def test_stops_on_an_invalid_index(self):
+        for module in MIGRATIONS:
+            error, _ = self.run_migration(module, (True,), (False,))
+            self.assertIn('INVALID', error)
+
+    def test_never_drops_an_index_going_forward(self):
+        for module in MIGRATIONS:
+            for answers in (((False,), (1_000,), (True,)), ((True,), (True,)), ((True,), (False,))):
+                _, statements = self.run_migration(module, *answers)
+                self.assertFalse([sql for sql in statements if 'drop index' in sql])

core/mappings/migrations/0061_mappings_repo_list_ids_idx.py

@@ -0,0 +1,55 @@
+"""
+Adds mappings_repo_list_ids_idx: mappings_repo_list_idx's key and predicate with INCLUDE (id), so a deep page of a source's HEAD
+mappings list can walk its OFFSET in the index alone, without reading every skipped row from the table
+(OpenConceptLab/ocl_online#476). mappings_repo_list_idx stays until EXPLAIN on staging and production shows the new index in use;
+a later migration drops it.
+
+On a large table, build the index by hand before deploying: every API task runs `migrate` before it serves, and a
+build that outlasts the health check's grace leaves an INVALID index behind. This migration then only checks it. It
+builds the index itself only on a small table (a fresh, test or dev database).
+"""
+from django.db import migrations
+
+TABLE = 'mappings'
+NEW_INDEX = 'mappings_repo_list_ids_idx'
+CREATE_NEW_INDEX = (
+    f'create index concurrently if not exists {NEW_INDEX} on {TABLE} (parent_id, updated_at desc) include (id) '
+    'where not retired and is_active and id = versioned_object_id;'
+)
+MAX_ROWS_TO_BUILD_HERE = 3_000_000
+
+
+def add_index(apps, schema_editor):  # pylint: disable=unused-argument
+    with schema_editor.connection.cursor() as cursor:
+        cursor.execute('select to_regclass(%s) is not null', [NEW_INDEX])
+        if not cursor.fetchone()[0]:
+            cursor.execute('select reltuples from pg_class where oid = %s::regclass', [TABLE])
+            rows = cursor.fetchone()[0]
+            if rows > MAX_ROWS_TO_BUILD_HERE:
+                raise RuntimeError(
+                    f'{TABLE} has about {rows:,.0f} rows: build {NEW_INDEX} by hand before deploying, '
+                    f'then migrate again. {CREATE_NEW_INDEX}')
+            cursor.execute(CREATE_NEW_INDEX)
+
+        cursor.execute('select indisvalid from pg_index where indexrelid = %s::regclass', [NEW_INDEX])
+        if not cursor.fetchone()[0]:
+            raise RuntimeError(
+                f'{NEW_INDEX} is INVALID (an interrupted build). Run `drop index concurrently {NEW_INDEX};`, '
+                f'build it again, then migrate again. {CREATE_NEW_INDEX}')
+
+
+def drop_index(apps, schema_editor):  # pylint: disable=unused-argument
+    with schema_editor.connection.cursor() as cursor:
+        cursor.execute(f'drop index concurrently if exists {NEW_INDEX};')
+
+
+class Migration(migrations.Migration):
+    atomic = False  # CREATE/DROP INDEX CONCURRENTLY can't run in a transaction
+
+    dependencies = [
+        ('mappings', '0060_retire_public_edit_access'),
+    ]
+
+    operations = [
+        migrations.RunPython(add_index, drop_index),
+    ]