fix(liquibase): keep populated Oracle observed instants on upgrade

.github/workflows/liquibase-upgrade.yml

@@ -9,6 +9,7 @@ on:
       - 'tools/tests/test_liquibase_1.2.7_to_head_upgrade.sh'
       - 'tools/tests/test_liquibase_codelist_auth_mode_swap.sh'
       - 'tools/tests/test_prepare_extracted_liquibase.py'
+      - 'tools/tests/test_service_check_observed_rebuild_order.py'
       - 'tools/bin/prepare_extracted_liquibase.py'
       - '.github/workflows/liquibase-upgrade.yml'
   workflow_dispatch:
@@ -29,6 +30,9 @@ jobs:
       - name: Extracted-tree helper unit tests
         run: python3 tools/tests/test_prepare_extracted_liquibase.py
 
+      - name: Service-check observed rebuild order
+        run: python3 tools/tests/test_service_check_observed_rebuild_order.py
+
   postgres-1-2-7-to-head:
     runs-on: ubuntu-latest
     timeout-minutes: 20

CHANGELOG.md

@@ -101,6 +101,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/),
 #### Profile-level
 
 - **Liquibase**: Development changelog 89, and PostgreSQL and Oracle changelog 33, alter `STM_SERVICE_CHECK.SEC_OBSERVED` and `STM_SERVICE_CHECK_SAMPLE.SCS_OBSERVED` to `TIMESTAMP(6) WITH TIME ZONE` on Oracle ([sitmun-admin-app#516](https://github.com/sitmun/sitmun-admin-app/issues/516)).
+- **Liquibase**: Development changelog 91, and PostgreSQL and Oracle changelog 35, copy populated `SEC_OBSERVED` and `SCS_OBSERVED` values as UTC before that `ALTER`. The `ALTER` changesets are unchanged ([#135](https://github.com/sitmun/sitmun-application-stack/issues/135)).
 
 ### Security
 

profiles/development/backend/liquibase/changelog/91_service_check_observed_timestamptz_rebuild.sql

@@ -0,0 +1,103 @@
+--liquibase formatted sql
+
+-- Oracle raises ORA-01439 when ALTER MODIFY changes TIMESTAMP to
+-- TIMESTAMP WITH TIME ZONE and the column already has values.
+-- Copy each value onto a new column as UTC. The backend stores these
+-- instants with hibernate.jdbc.time_zone set to UTC.
+-- The later ALTER changeset sees TIMESTAMP(6) WITH TIME ZONE and is skipped.
+
+--changeset sitmun:91-service-check-sec-observed-rebuild-oracle dbms:oracle splitStatements:false endDelimiter:/
+DECLARE
+  l_type VARCHAR2(128);
+  l_tz NUMBER;
+  l_nullable VARCHAR2(1);
+BEGIN
+  BEGIN
+    SELECT data_type, nullable INTO l_type, l_nullable
+    FROM user_tab_columns
+    WHERE table_name = 'STM_SERVICE_CHECK' AND column_name = 'SEC_OBSERVED';
+  EXCEPTION
+    WHEN NO_DATA_FOUND THEN
+      l_type := NULL;
+      l_nullable := NULL;
+  END;
+
+  SELECT COUNT(*) INTO l_tz
+  FROM user_tab_columns
+  WHERE table_name = 'STM_SERVICE_CHECK' AND column_name = 'SEC_OBSERVED_TZ';
+
+  IF l_type LIKE '%WITH TIME ZONE' THEN
+    IF l_nullable = 'Y' THEN
+      EXECUTE IMMEDIATE 'ALTER TABLE STM_SERVICE_CHECK MODIFY (SEC_OBSERVED NOT NULL)';
+    END IF;
+    IF l_tz = 1 THEN
+      EXECUTE IMMEDIATE 'ALTER TABLE STM_SERVICE_CHECK DROP COLUMN SEC_OBSERVED_TZ';
+    END IF;
+  ELSE
+    IF l_tz = 0 THEN
+      EXECUTE IMMEDIATE 'ALTER TABLE STM_SERVICE_CHECK ADD (SEC_OBSERVED_TZ TIMESTAMP(6) WITH TIME ZONE)';
+    END IF;
+    IF l_type IS NOT NULL THEN
+      EXECUTE IMMEDIATE 'UPDATE STM_SERVICE_CHECK SET SEC_OBSERVED_TZ = FROM_TZ(SEC_OBSERVED, ''UTC'') WHERE SEC_OBSERVED_TZ IS NULL';
+      EXECUTE IMMEDIATE 'ALTER TABLE STM_SERVICE_CHECK MODIFY (SEC_OBSERVED_TZ NOT NULL)';
+      EXECUTE IMMEDIATE 'ALTER TABLE STM_SERVICE_CHECK DROP COLUMN SEC_OBSERVED';
+    ELSE
+      EXECUTE IMMEDIATE 'ALTER TABLE STM_SERVICE_CHECK MODIFY (SEC_OBSERVED_TZ NOT NULL)';
+    END IF;
+    EXECUTE IMMEDIATE 'ALTER TABLE STM_SERVICE_CHECK RENAME COLUMN SEC_OBSERVED_TZ TO SEC_OBSERVED';
+  END IF;
+END;
+/
+
+--changeset sitmun:91-service-check-scs-observed-rebuild-oracle dbms:oracle splitStatements:false endDelimiter:/
+DECLARE
+  l_type VARCHAR2(128);
+  l_tz NUMBER;
+  l_idx NUMBER;
+  l_nullable VARCHAR2(1);
+BEGIN
+  BEGIN
+    SELECT data_type, nullable INTO l_type, l_nullable
+    FROM user_tab_columns
+    WHERE table_name = 'STM_SERVICE_CHECK_SAMPLE' AND column_name = 'SCS_OBSERVED';
+  EXCEPTION
+    WHEN NO_DATA_FOUND THEN
+      l_type := NULL;
+      l_nullable := NULL;
+  END;
+
+  SELECT COUNT(*) INTO l_tz
+  FROM user_tab_columns
+  WHERE table_name = 'STM_SERVICE_CHECK_SAMPLE' AND column_name = 'SCS_OBSERVED_TZ';
+
+  IF l_type LIKE '%WITH TIME ZONE' THEN
+    IF l_nullable = 'Y' THEN
+      EXECUTE IMMEDIATE 'ALTER TABLE STM_SERVICE_CHECK_SAMPLE MODIFY (SCS_OBSERVED NOT NULL)';
+    END IF;
+    IF l_tz = 1 THEN
+      EXECUTE IMMEDIATE 'ALTER TABLE STM_SERVICE_CHECK_SAMPLE DROP COLUMN SCS_OBSERVED_TZ';
+    END IF;
+  ELSE
+    SELECT COUNT(*) INTO l_idx FROM user_indexes WHERE index_name = 'STM_SCS_IX_SER_OBS';
+    IF l_idx = 1 THEN
+      EXECUTE IMMEDIATE 'DROP INDEX STM_SCS_IX_SER_OBS';
+    END IF;
+    IF l_tz = 0 THEN
+      EXECUTE IMMEDIATE 'ALTER TABLE STM_SERVICE_CHECK_SAMPLE ADD (SCS_OBSERVED_TZ TIMESTAMP(6) WITH TIME ZONE)';
+    END IF;
+    IF l_type IS NOT NULL THEN
+      EXECUTE IMMEDIATE 'UPDATE STM_SERVICE_CHECK_SAMPLE SET SCS_OBSERVED_TZ = FROM_TZ(SCS_OBSERVED, ''UTC'') WHERE SCS_OBSERVED_TZ IS NULL';
+      EXECUTE IMMEDIATE 'ALTER TABLE STM_SERVICE_CHECK_SAMPLE MODIFY (SCS_OBSERVED_TZ NOT NULL)';
+      EXECUTE IMMEDIATE 'ALTER TABLE STM_SERVICE_CHECK_SAMPLE DROP COLUMN SCS_OBSERVED';
+    ELSE
+      EXECUTE IMMEDIATE 'ALTER TABLE STM_SERVICE_CHECK_SAMPLE MODIFY (SCS_OBSERVED_TZ NOT NULL)';
+    END IF;
+    EXECUTE IMMEDIATE 'ALTER TABLE STM_SERVICE_CHECK_SAMPLE RENAME COLUMN SCS_OBSERVED_TZ TO SCS_OBSERVED';
+  END IF;
+
+  SELECT COUNT(*) INTO l_idx FROM user_indexes WHERE index_name = 'STM_SCS_IX_SER_OBS';
+  IF l_idx = 0 THEN
+    EXECUTE IMMEDIATE 'CREATE INDEX STM_SCS_IX_SER_OBS ON STM_SERVICE_CHECK_SAMPLE (SCS_SERID, SCS_OBSERVED)';
+  END IF;
+END;
+/

profiles/development/backend/liquibase/master.xml

@@ -95,6 +95,8 @@
   <include file="changelog/86_dev_service_parameter_info_olparam_oracle.yaml" relativeToChangelogFile="true"/>
   <include file="changelog/87_restrict_service_types.yaml" relativeToChangelogFile="true"/>
   <include file="changelog/88_geometry_code_lists.yaml" relativeToChangelogFile="true"/>
+  <!-- 91 runs before 89. ORA-01439 if SEC_OBSERVED already has rows. -->
+  <include file="changelog/91_service_check_observed_timestamptz_rebuild.sql" relativeToChangelogFile="true"/>
   <include file="changelog/89_service_check_observed_timestamptz.yaml" relativeToChangelogFile="true"/>
   <include file="changelog/90_service_check_detail_nullable.yaml" relativeToChangelogFile="true"/>
 </databaseChangeLog>

profiles/oracle/liquibase/changelog/35_service_check_observed_timestamptz_rebuild.sql

@@ -0,0 +1,103 @@
+--liquibase formatted sql
+
+-- Oracle raises ORA-01439 when ALTER MODIFY changes TIMESTAMP to
+-- TIMESTAMP WITH TIME ZONE and the column already has values.
+-- Copy each value onto a new column as UTC. The backend stores these
+-- instants with hibernate.jdbc.time_zone set to UTC.
+-- The later ALTER changeset sees TIMESTAMP(6) WITH TIME ZONE and is skipped.
+
+--changeset sitmun:35-service-check-sec-observed-rebuild-oracle dbms:oracle splitStatements:false endDelimiter:/
+DECLARE
+  l_type VARCHAR2(128);
+  l_tz NUMBER;
+  l_nullable VARCHAR2(1);
+BEGIN
+  BEGIN
+    SELECT data_type, nullable INTO l_type, l_nullable
+    FROM user_tab_columns
+    WHERE table_name = 'STM_SERVICE_CHECK' AND column_name = 'SEC_OBSERVED';
+  EXCEPTION
+    WHEN NO_DATA_FOUND THEN
+      l_type := NULL;
+      l_nullable := NULL;
+  END;
+
+  SELECT COUNT(*) INTO l_tz
+  FROM user_tab_columns
+  WHERE table_name = 'STM_SERVICE_CHECK' AND column_name = 'SEC_OBSERVED_TZ';
+
+  IF l_type LIKE '%WITH TIME ZONE' THEN
+    IF l_nullable = 'Y' THEN
+      EXECUTE IMMEDIATE 'ALTER TABLE STM_SERVICE_CHECK MODIFY (SEC_OBSERVED NOT NULL)';
+    END IF;
+    IF l_tz = 1 THEN
+      EXECUTE IMMEDIATE 'ALTER TABLE STM_SERVICE_CHECK DROP COLUMN SEC_OBSERVED_TZ';
+    END IF;
+  ELSE
+    IF l_tz = 0 THEN
+      EXECUTE IMMEDIATE 'ALTER TABLE STM_SERVICE_CHECK ADD (SEC_OBSERVED_TZ TIMESTAMP(6) WITH TIME ZONE)';
+    END IF;
+    IF l_type IS NOT NULL THEN
+      EXECUTE IMMEDIATE 'UPDATE STM_SERVICE_CHECK SET SEC_OBSERVED_TZ = FROM_TZ(SEC_OBSERVED, ''UTC'') WHERE SEC_OBSERVED_TZ IS NULL';
+      EXECUTE IMMEDIATE 'ALTER TABLE STM_SERVICE_CHECK MODIFY (SEC_OBSERVED_TZ NOT NULL)';
+      EXECUTE IMMEDIATE 'ALTER TABLE STM_SERVICE_CHECK DROP COLUMN SEC_OBSERVED';
+    ELSE
+      EXECUTE IMMEDIATE 'ALTER TABLE STM_SERVICE_CHECK MODIFY (SEC_OBSERVED_TZ NOT NULL)';
+    END IF;
+    EXECUTE IMMEDIATE 'ALTER TABLE STM_SERVICE_CHECK RENAME COLUMN SEC_OBSERVED_TZ TO SEC_OBSERVED';
+  END IF;
+END;
+/
+
+--changeset sitmun:35-service-check-scs-observed-rebuild-oracle dbms:oracle splitStatements:false endDelimiter:/
+DECLARE
+  l_type VARCHAR2(128);
+  l_tz NUMBER;
+  l_idx NUMBER;
+  l_nullable VARCHAR2(1);
+BEGIN
+  BEGIN
+    SELECT data_type, nullable INTO l_type, l_nullable
+    FROM user_tab_columns
+    WHERE table_name = 'STM_SERVICE_CHECK_SAMPLE' AND column_name = 'SCS_OBSERVED';
+  EXCEPTION
+    WHEN NO_DATA_FOUND THEN
+      l_type := NULL;
+      l_nullable := NULL;
+  END;
+
+  SELECT COUNT(*) INTO l_tz
+  FROM user_tab_columns
+  WHERE table_name = 'STM_SERVICE_CHECK_SAMPLE' AND column_name = 'SCS_OBSERVED_TZ';
+
+  IF l_type LIKE '%WITH TIME ZONE' THEN
+    IF l_nullable = 'Y' THEN
+      EXECUTE IMMEDIATE 'ALTER TABLE STM_SERVICE_CHECK_SAMPLE MODIFY (SCS_OBSERVED NOT NULL)';
+    END IF;
+    IF l_tz = 1 THEN
+      EXECUTE IMMEDIATE 'ALTER TABLE STM_SERVICE_CHECK_SAMPLE DROP COLUMN SCS_OBSERVED_TZ';
+    END IF;
+  ELSE
+    SELECT COUNT(*) INTO l_idx FROM user_indexes WHERE index_name = 'STM_SCS_IX_SER_OBS';
+    IF l_idx = 1 THEN
+      EXECUTE IMMEDIATE 'DROP INDEX STM_SCS_IX_SER_OBS';
+    END IF;
+    IF l_tz = 0 THEN
+      EXECUTE IMMEDIATE 'ALTER TABLE STM_SERVICE_CHECK_SAMPLE ADD (SCS_OBSERVED_TZ TIMESTAMP(6) WITH TIME ZONE)';
+    END IF;
+    IF l_type IS NOT NULL THEN
+      EXECUTE IMMEDIATE 'UPDATE STM_SERVICE_CHECK_SAMPLE SET SCS_OBSERVED_TZ = FROM_TZ(SCS_OBSERVED, ''UTC'') WHERE SCS_OBSERVED_TZ IS NULL';
+      EXECUTE IMMEDIATE 'ALTER TABLE STM_SERVICE_CHECK_SAMPLE MODIFY (SCS_OBSERVED_TZ NOT NULL)';
+      EXECUTE IMMEDIATE 'ALTER TABLE STM_SERVICE_CHECK_SAMPLE DROP COLUMN SCS_OBSERVED';
+    ELSE
+      EXECUTE IMMEDIATE 'ALTER TABLE STM_SERVICE_CHECK_SAMPLE MODIFY (SCS_OBSERVED_TZ NOT NULL)';
+    END IF;
+    EXECUTE IMMEDIATE 'ALTER TABLE STM_SERVICE_CHECK_SAMPLE RENAME COLUMN SCS_OBSERVED_TZ TO SCS_OBSERVED';
+  END IF;
+
+  SELECT COUNT(*) INTO l_idx FROM user_indexes WHERE index_name = 'STM_SCS_IX_SER_OBS';
+  IF l_idx = 0 THEN
+    EXECUTE IMMEDIATE 'CREATE INDEX STM_SCS_IX_SER_OBS ON STM_SERVICE_CHECK_SAMPLE (SCS_SERID, SCS_OBSERVED)';
+  END IF;
+END;
+/

profiles/oracle/liquibase/master.xml

@@ -37,6 +37,8 @@
   <include file="changelog/30_cartography_service_parameter.yaml" relativeToChangelogFile="true"/>
   <include file="changelog/31_restrict_service_types.yaml" relativeToChangelogFile="true"/>
   <include file="changelog/32_geometry_code_lists.yaml" relativeToChangelogFile="true"/>
+  <!-- 35 runs before 33. ORA-01439 if SEC_OBSERVED already has rows. -->
+  <include file="changelog/35_service_check_observed_timestamptz_rebuild.sql" relativeToChangelogFile="true"/>
   <include file="changelog/33_service_check_observed_timestamptz.yaml" relativeToChangelogFile="true"/>
   <include file="changelog/34_service_check_detail_nullable.yaml" relativeToChangelogFile="true"/>
 </databaseChangeLog>

profiles/postgres/liquibase/changelog/35_service_check_observed_timestamptz_rebuild.sql

@@ -0,0 +1,103 @@
+--liquibase formatted sql
+
+-- Oracle raises ORA-01439 when ALTER MODIFY changes TIMESTAMP to
+-- TIMESTAMP WITH TIME ZONE and the column already has values.
+-- Copy each value onto a new column as UTC. The backend stores these
+-- instants with hibernate.jdbc.time_zone set to UTC.
+-- The later ALTER changeset sees TIMESTAMP(6) WITH TIME ZONE and is skipped.
+
+--changeset sitmun:35-service-check-sec-observed-rebuild-oracle dbms:oracle splitStatements:false endDelimiter:/
+DECLARE
+  l_type VARCHAR2(128);
+  l_tz NUMBER;
+  l_nullable VARCHAR2(1);
+BEGIN
+  BEGIN
+    SELECT data_type, nullable INTO l_type, l_nullable
+    FROM user_tab_columns
+    WHERE table_name = 'STM_SERVICE_CHECK' AND column_name = 'SEC_OBSERVED';
+  EXCEPTION
+    WHEN NO_DATA_FOUND THEN
+      l_type := NULL;
+      l_nullable := NULL;
+  END;
+
+  SELECT COUNT(*) INTO l_tz
+  FROM user_tab_columns
+  WHERE table_name = 'STM_SERVICE_CHECK' AND column_name = 'SEC_OBSERVED_TZ';
+
+  IF l_type LIKE '%WITH TIME ZONE' THEN
+    IF l_nullable = 'Y' THEN
+      EXECUTE IMMEDIATE 'ALTER TABLE STM_SERVICE_CHECK MODIFY (SEC_OBSERVED NOT NULL)';
+    END IF;
+    IF l_tz = 1 THEN
+      EXECUTE IMMEDIATE 'ALTER TABLE STM_SERVICE_CHECK DROP COLUMN SEC_OBSERVED_TZ';
+    END IF;
+  ELSE
+    IF l_tz = 0 THEN
+      EXECUTE IMMEDIATE 'ALTER TABLE STM_SERVICE_CHECK ADD (SEC_OBSERVED_TZ TIMESTAMP(6) WITH TIME ZONE)';
+    END IF;
+    IF l_type IS NOT NULL THEN
+      EXECUTE IMMEDIATE 'UPDATE STM_SERVICE_CHECK SET SEC_OBSERVED_TZ = FROM_TZ(SEC_OBSERVED, ''UTC'') WHERE SEC_OBSERVED_TZ IS NULL';
+      EXECUTE IMMEDIATE 'ALTER TABLE STM_SERVICE_CHECK MODIFY (SEC_OBSERVED_TZ NOT NULL)';
+      EXECUTE IMMEDIATE 'ALTER TABLE STM_SERVICE_CHECK DROP COLUMN SEC_OBSERVED';
+    ELSE
+      EXECUTE IMMEDIATE 'ALTER TABLE STM_SERVICE_CHECK MODIFY (SEC_OBSERVED_TZ NOT NULL)';
+    END IF;
+    EXECUTE IMMEDIATE 'ALTER TABLE STM_SERVICE_CHECK RENAME COLUMN SEC_OBSERVED_TZ TO SEC_OBSERVED';
+  END IF;
+END;
+/
+
+--changeset sitmun:35-service-check-scs-observed-rebuild-oracle dbms:oracle splitStatements:false endDelimiter:/
+DECLARE
+  l_type VARCHAR2(128);
+  l_tz NUMBER;
+  l_idx NUMBER;
+  l_nullable VARCHAR2(1);
+BEGIN
+  BEGIN
+    SELECT data_type, nullable INTO l_type, l_nullable
+    FROM user_tab_columns
+    WHERE table_name = 'STM_SERVICE_CHECK_SAMPLE' AND column_name = 'SCS_OBSERVED';
+  EXCEPTION
+    WHEN NO_DATA_FOUND THEN
+      l_type := NULL;
+      l_nullable := NULL;
+  END;
+
+  SELECT COUNT(*) INTO l_tz
+  FROM user_tab_columns
+  WHERE table_name = 'STM_SERVICE_CHECK_SAMPLE' AND column_name = 'SCS_OBSERVED_TZ';
+
+  IF l_type LIKE '%WITH TIME ZONE' THEN
+    IF l_nullable = 'Y' THEN
+      EXECUTE IMMEDIATE 'ALTER TABLE STM_SERVICE_CHECK_SAMPLE MODIFY (SCS_OBSERVED NOT NULL)';
+    END IF;
+    IF l_tz = 1 THEN
+      EXECUTE IMMEDIATE 'ALTER TABLE STM_SERVICE_CHECK_SAMPLE DROP COLUMN SCS_OBSERVED_TZ';
+    END IF;
+  ELSE
+    SELECT COUNT(*) INTO l_idx FROM user_indexes WHERE index_name = 'STM_SCS_IX_SER_OBS';
+    IF l_idx = 1 THEN
+      EXECUTE IMMEDIATE 'DROP INDEX STM_SCS_IX_SER_OBS';
+    END IF;
+    IF l_tz = 0 THEN
+      EXECUTE IMMEDIATE 'ALTER TABLE STM_SERVICE_CHECK_SAMPLE ADD (SCS_OBSERVED_TZ TIMESTAMP(6) WITH TIME ZONE)';
+    END IF;
+    IF l_type IS NOT NULL THEN
+      EXECUTE IMMEDIATE 'UPDATE STM_SERVICE_CHECK_SAMPLE SET SCS_OBSERVED_TZ = FROM_TZ(SCS_OBSERVED, ''UTC'') WHERE SCS_OBSERVED_TZ IS NULL';
+      EXECUTE IMMEDIATE 'ALTER TABLE STM_SERVICE_CHECK_SAMPLE MODIFY (SCS_OBSERVED_TZ NOT NULL)';
+      EXECUTE IMMEDIATE 'ALTER TABLE STM_SERVICE_CHECK_SAMPLE DROP COLUMN SCS_OBSERVED';
+    ELSE
+      EXECUTE IMMEDIATE 'ALTER TABLE STM_SERVICE_CHECK_SAMPLE MODIFY (SCS_OBSERVED_TZ NOT NULL)';
+    END IF;
+    EXECUTE IMMEDIATE 'ALTER TABLE STM_SERVICE_CHECK_SAMPLE RENAME COLUMN SCS_OBSERVED_TZ TO SCS_OBSERVED';
+  END IF;
+
+  SELECT COUNT(*) INTO l_idx FROM user_indexes WHERE index_name = 'STM_SCS_IX_SER_OBS';
+  IF l_idx = 0 THEN
+    EXECUTE IMMEDIATE 'CREATE INDEX STM_SCS_IX_SER_OBS ON STM_SERVICE_CHECK_SAMPLE (SCS_SERID, SCS_OBSERVED)';
+  END IF;
+END;
+/

profiles/postgres/liquibase/master.xml

@@ -35,6 +35,8 @@
   <include file="changelog/30_cartography_service_parameter.yaml" relativeToChangelogFile="true"/>
   <include file="changelog/31_restrict_service_types.yaml" relativeToChangelogFile="true"/>
   <include file="changelog/32_geometry_code_lists.yaml" relativeToChangelogFile="true"/>
+  <!-- 35 runs before 33. ORA-01439 if SEC_OBSERVED already has rows. -->
+  <include file="changelog/35_service_check_observed_timestamptz_rebuild.sql" relativeToChangelogFile="true"/>
   <include file="changelog/33_service_check_observed_timestamptz.yaml" relativeToChangelogFile="true"/>
   <include file="changelog/34_service_check_detail_nullable.yaml" relativeToChangelogFile="true"/>
 </databaseChangeLog>