diff --git a/core/concepts/migrations/0090_concepts_repo_list_ids_idx.py b/core/concepts/migrations/0090_concepts_repo_list_ids_idx.py new file mode 100644 index 000000000..df6479bf0 --- /dev/null +++ b/core/concepts/migrations/0090_concepts_repo_list_ids_idx.py @@ -0,0 +1,55 @@ +""" +Adds concepts_repo_list_ids_idx: concepts_repo_list_idx's key and predicate with INCLUDE (id), so a deep page of a source's HEAD +concepts list can walk its OFFSET in the index alone, without reading every skipped row from the table +(OpenConceptLab/ocl_online#476). concepts_repo_list_idx stays until EXPLAIN on staging and production shows the new index in use; +a later migration drops it. + +On a large table, build the index by hand before deploying: every API task runs `migrate` before it serves, and a +build that outlasts the health check's grace leaves an INVALID index behind. This migration then only checks it. It +builds the index itself only on a small table (a fresh, test or dev database). +""" +from django.db import migrations + +TABLE = 'concepts' +NEW_INDEX = 'concepts_repo_list_ids_idx' +CREATE_NEW_INDEX = ( + f'create index concurrently if not exists {NEW_INDEX} on {TABLE} (parent_id, updated_at desc) include (id) ' + 'where not retired and is_active and id = versioned_object_id;' +) +MAX_ROWS_TO_BUILD_HERE = 3_000_000 + + +def add_index(apps, schema_editor): # pylint: disable=unused-argument + with schema_editor.connection.cursor() as cursor: + cursor.execute('select to_regclass(%s) is not null', [NEW_INDEX]) + if not cursor.fetchone()[0]: + cursor.execute('select reltuples from pg_class where oid = %s::regclass', [TABLE]) + rows = cursor.fetchone()[0] + if rows > MAX_ROWS_TO_BUILD_HERE: + raise RuntimeError( + f'{TABLE} has about {rows:,.0f} rows: build {NEW_INDEX} by hand before deploying, ' + f'then migrate again. {CREATE_NEW_INDEX}') + cursor.execute(CREATE_NEW_INDEX) + + cursor.execute('select indisvalid from pg_index where indexrelid = %s::regclass', [NEW_INDEX]) + if not cursor.fetchone()[0]: + raise RuntimeError( + f'{NEW_INDEX} is INVALID (an interrupted build). Run `drop index concurrently {NEW_INDEX};`, ' + f'build it again, then migrate again. {CREATE_NEW_INDEX}') + + +def drop_index(apps, schema_editor): # pylint: disable=unused-argument + with schema_editor.connection.cursor() as cursor: + cursor.execute(f'drop index concurrently if exists {NEW_INDEX};') + + +class Migration(migrations.Migration): + atomic = False # CREATE/DROP INDEX CONCURRENTLY can't run in a transaction + + dependencies = [ + ('concepts', '0089_merge_20261005_0816'), + ] + + operations = [ + migrations.RunPython(add_index, drop_index), + ] diff --git a/core/integration_tests/tests_list_indexes.py b/core/integration_tests/tests_list_indexes.py new file mode 100644 index 000000000..c385ff45e --- /dev/null +++ b/core/integration_tests/tests_list_indexes.py @@ -0,0 +1,63 @@ +import importlib +from unittest.mock import MagicMock + +from django.db import connection +from django.test import SimpleTestCase, TestCase + +MIGRATIONS = ( + 'core.concepts.migrations.0090_concepts_repo_list_ids_idx', + 'core.mappings.migrations.0061_mappings_repo_list_ids_idx', +) + + +class ListCoveringIndexTest(TestCase): + def test_list_indexes_cover_id(self): + with connection.cursor() as cursor: + for table in ('concepts', 'mappings'): + cursor.execute('select indexdef from pg_indexes where indexname = %s', [f'{table}_repo_list_ids_idx']) + definition = cursor.fetchone()[0] + self.assertIn('(parent_id, updated_at DESC) INCLUDE (id)', definition) + self.assertIn('WHERE ((NOT retired) AND is_active AND (id = versioned_object_id))', definition) + + # the old index stays until EXPLAIN in production shows the new one in use (a later migration) + cursor.execute('select to_regclass(%s)', [f'{table}_repo_list_idx']) + self.assertIsNotNone(cursor.fetchone()[0]) + + +class ListCoveringIndexMigrationTest(SimpleTestCase): + """The migrations never build on a production-sized table, and never accept an invalid index.""" + @staticmethod + def run_migration(module, *answers): + cursor = MagicMock() + cursor.fetchone.side_effect = answers + schema_editor = MagicMock() + schema_editor.connection.cursor.return_value.__enter__.return_value = cursor + error = None + try: + importlib.import_module(module).add_index(None, schema_editor) + except RuntimeError as ex: + error = str(ex) + return error, [call.args[0] for call in cursor.execute.call_args_list] + + def test_refuses_to_build_on_a_large_table(self): + for module in MIGRATIONS: + error, statements = self.run_migration(module, (False,), (10_000_000,)) + self.assertIn('by hand before deploying', error) + self.assertFalse([sql for sql in statements if 'create index' in sql]) + + def test_uses_an_index_built_by_hand(self): + for module in MIGRATIONS: + error, statements = self.run_migration(module, (True,), (True,)) + self.assertIsNone(error) + self.assertFalse([sql for sql in statements if 'create index' in sql]) + + def test_stops_on_an_invalid_index(self): + for module in MIGRATIONS: + error, _ = self.run_migration(module, (True,), (False,)) + self.assertIn('INVALID', error) + + def test_never_drops_an_index_going_forward(self): + for module in MIGRATIONS: + for answers in (((False,), (1_000,), (True,)), ((True,), (True,)), ((True,), (False,))): + _, statements = self.run_migration(module, *answers) + self.assertFalse([sql for sql in statements if 'drop index' in sql]) diff --git a/core/mappings/migrations/0061_mappings_repo_list_ids_idx.py b/core/mappings/migrations/0061_mappings_repo_list_ids_idx.py new file mode 100644 index 000000000..d97b4c8d8 --- /dev/null +++ b/core/mappings/migrations/0061_mappings_repo_list_ids_idx.py @@ -0,0 +1,55 @@ +""" +Adds mappings_repo_list_ids_idx: mappings_repo_list_idx's key and predicate with INCLUDE (id), so a deep page of a source's HEAD +mappings list can walk its OFFSET in the index alone, without reading every skipped row from the table +(OpenConceptLab/ocl_online#476). mappings_repo_list_idx stays until EXPLAIN on staging and production shows the new index in use; +a later migration drops it. + +On a large table, build the index by hand before deploying: every API task runs `migrate` before it serves, and a +build that outlasts the health check's grace leaves an INVALID index behind. This migration then only checks it. It +builds the index itself only on a small table (a fresh, test or dev database). +""" +from django.db import migrations + +TABLE = 'mappings' +NEW_INDEX = 'mappings_repo_list_ids_idx' +CREATE_NEW_INDEX = ( + f'create index concurrently if not exists {NEW_INDEX} on {TABLE} (parent_id, updated_at desc) include (id) ' + 'where not retired and is_active and id = versioned_object_id;' +) +MAX_ROWS_TO_BUILD_HERE = 3_000_000 + + +def add_index(apps, schema_editor): # pylint: disable=unused-argument + with schema_editor.connection.cursor() as cursor: + cursor.execute('select to_regclass(%s) is not null', [NEW_INDEX]) + if not cursor.fetchone()[0]: + cursor.execute('select reltuples from pg_class where oid = %s::regclass', [TABLE]) + rows = cursor.fetchone()[0] + if rows > MAX_ROWS_TO_BUILD_HERE: + raise RuntimeError( + f'{TABLE} has about {rows:,.0f} rows: build {NEW_INDEX} by hand before deploying, ' + f'then migrate again. {CREATE_NEW_INDEX}') + cursor.execute(CREATE_NEW_INDEX) + + cursor.execute('select indisvalid from pg_index where indexrelid = %s::regclass', [NEW_INDEX]) + if not cursor.fetchone()[0]: + raise RuntimeError( + f'{NEW_INDEX} is INVALID (an interrupted build). Run `drop index concurrently {NEW_INDEX};`, ' + f'build it again, then migrate again. {CREATE_NEW_INDEX}') + + +def drop_index(apps, schema_editor): # pylint: disable=unused-argument + with schema_editor.connection.cursor() as cursor: + cursor.execute(f'drop index concurrently if exists {NEW_INDEX};') + + +class Migration(migrations.Migration): + atomic = False # CREATE/DROP INDEX CONCURRENTLY can't run in a transaction + + dependencies = [ + ('mappings', '0060_retire_public_edit_access'), + ] + + operations = [ + migrations.RunPython(add_index, drop_index), + ]