diff --git a/delivery.yaml b/delivery.yaml index 12ea76a70..e66675f26 100644 --- a/delivery.yaml +++ b/delivery.yaml @@ -3,8 +3,8 @@ allow_concurrent_steps: true build_env: &BUILD_ENV BASE_IMAGE: container-registry.zalando.net/library/ubuntu-22.04 - PGVERSION: 18 - PGOLDVERSIONS: "16 17" + PGVERSION: 19 + PGOLDVERSIONS: "17 18" MULTI_ARCH_REGISTRY: container-registry-test.zalando.net/acid pipeline: diff --git a/postgres-appliance/Dockerfile b/postgres-appliance/Dockerfile index 846eaf80b..4e1f8be36 100644 --- a/postgres-appliance/Dockerfile +++ b/postgres-appliance/Dockerfile @@ -1,5 +1,5 @@ ARG BASE_IMAGE=ubuntu:22.04 -ARG PGVERSION=18 +ARG PGVERSION=19 ARG DEMO=false ARG COMPRESS=false ARG ADDITIONAL_LOCALES= @@ -46,18 +46,18 @@ ARG PGVERSION ARG TIMESCALEDB_APACHE_ONLY=true ARG TIMESCALEDB_TOOLKIT=true ARG COMPRESS -ARG PGOLDVERSIONS="14 15 16 17" +ARG PGOLDVERSIONS="15 16 17 18" ARG WITH_PERL=false ARG DEB_PG_SUPPORTED_VERSIONS="$PGOLDVERSIONS $PGVERSION" # Install PostgreSQL, extensions and contribs ENV POSTGIS_VERSION=3.6 \ - BG_MON_COMMIT=a73c6bcd10dfdf9feaf5eabab7eb6b12d167680d \ - PG_AUTH_MON_COMMIT=fe099eef7662cbc85b0b79191f47f52f1e96b779 \ - PG_MON_COMMIT=88ac7b58348aa061c814982defc170644f368f39 \ + BG_MON_COMMIT=794f0ea14031880e195a9bdc52d2209a33f1bd56 \ + PG_AUTH_MON_COMMIT=b7e3f440205414610fd60e402d4f16c4cd59a683 \ + PG_MON_COMMIT=7b924e2a2d0095b72f4d11de262180c6cbd2c9f1 \ PLPROFILER=REL4_2_5 \ - PG_PROFILE=4.11 \ + PG_PROFILE=4.14 \ PAM_OAUTH2=v1.0.1 \ PG_TEXTSEARCH=v1.3.0 diff --git a/postgres-appliance/build_scripts/base.sh b/postgres-appliance/build_scripts/base.sh index 2f2b04a21..31c364623 100644 --- a/postgres-appliance/build_scripts/base.sh +++ b/postgres-appliance/build_scripts/base.sh @@ -76,42 +76,47 @@ for version in $DEB_PG_SUPPORTED_VERSIONS; do EXTRAS=("postgresql-pltcl-${version}" "postgresql-${version}-dirtyread" "postgresql-${version}-extra-window-functions" - "postgresql-${version}-first-last-agg" "postgresql-${version}-hll" "postgresql-${version}-hypopg" "postgresql-${version}-partman" "postgresql-${version}-plproxy" "postgresql-${version}-pgaudit" - "postgresql-${version}-pldebugger" "postgresql-${version}-pglogical" "postgresql-${version}-plpgsql-check" - "postgresql-${version}-pg-checksums" - "postgresql-${version}-pgq-node" "postgresql-${version}-postgis-${POSTGIS_VERSION%.*}" "postgresql-${version}-postgis-${POSTGIS_VERSION%.*}-scripts" "postgresql-${version}-repack" - "postgresql-${version}-wal2json" - "postgresql-${version}-decoderbufs" - "postgresql-${version}-pllua" "postgresql-${version}-pgvector" - "postgresql-${version}-roaringbitmap" - "postgresql-${version}-pgfaceting") + "postgresql-${version}-roaringbitmap") - if [ "$version" != "18" ]; then + if [ "$version" -lt 18 ]; then EXTRAS+=("postgresql-${version}-pgl-ddl-deploy" "postgresql-${version}-pglogical-ticker") fi - if [ "$WITH_PERL" = "true" ]; then - EXTRAS+=("postgresql-plperl-${version}") + if [ "$version" -lt 19 ]; then + EXTRAS+=("postgresql-${version}-first-last-agg" + "postgresql-${version}-pldebugger" + "postgresql-${version}-pg-checksums" + "postgresql-${version}-pgq-node" + "postgresql-${version}-wal2json" + "postgresql-${version}-decoderbufs" + "postgresql-${version}-pllua" + "postgresql-${version}-pgfaceting") + fi + + if [ "$WITH_PERL" = "true" ]; then + EXTRAS+=("postgresql-plperl-${version}") fi fi - if [ "${TIMESCALEDB_APACHE_ONLY}" = "true" ]; then - EXTRAS+=("timescaledb-2-oss-postgresql-${version}") - else - EXTRAS+=("timescaledb-2-postgresql-${version}") + if [ "$version" -lt 19 ]; then + if [ "${TIMESCALEDB_APACHE_ONLY}" = "true" ]; then + EXTRAS+=("timescaledb-2-oss-postgresql-${version}") + else + EXTRAS+=("timescaledb-2-postgresql-${version}") + fi fi # Install PostgreSQL binaries, contrib, plproxy and multiple pl's @@ -121,12 +126,16 @@ for version in $DEB_PG_SUPPORTED_VERSIONS; do "postgresql-${version}-pgextwlist" \ "postgresql-plpython3-${version}" \ "postgresql-server-dev-${version}" \ - "postgresql-${version}-pgq3" \ "postgresql-${version}-pg-stat-kcache" \ "postgresql-${version}-pg-permissions" \ "postgresql-${version}-set-user" \ "${EXTRAS[@]}" + if [ "$version" -lt 19 ]; then + apt-get install --allow-downgrades -y \ + "postgresql-${version}-pgq3" + fi + # Clean up timescaledb versions - keep at least 5 minor versions, but ensure compatibility with the lowest/oldest PG version (where possible) exclude_patterns=() @@ -179,10 +188,13 @@ for version in $DEB_PG_SUPPORTED_VERSIONS; do EXTRA_EXTENSIONS=() if [ "$DEMO" != "true" ]; then - EXTRA_EXTENSIONS+=("plprofiler" "pg_mon-${PG_MON_COMMIT}") - if [ "$version" -ge 17 ]; then - EXTRA_EXTENSIONS+=("pg_textsearch") + if [ "$version" -lt 19 ]; then + EXTRA_EXTENSIONS+=("plprofiler") + if [ "$version" -ge 17 ]; then + EXTRA_EXTENSIONS+=("pg_textsearch") + fi fi + EXTRA_EXTENSIONS+=("pg_mon-${PG_MON_COMMIT}") fi for n in bg_mon-${BG_MON_COMMIT} \ @@ -273,7 +285,7 @@ if [ "$DEMO" != "true" ]; then for e in pgq pgq_node plproxy address_standardizer address_standardizer_data_us; do orig=$(basename "$(find . -maxdepth 1 -type f -name "$e--*--*.sql" | head -n1)") - if [ "x$orig" != "x" ]; then + if [ -n "$orig" ]; then for f in "$e"--*--*.sql; do if [ "$f" != "$orig" ] && [ ! -L "$f" ] && diff "$f" "$orig" > /dev/null; then echo "creating symlink $f -> $orig" diff --git a/postgres-appliance/build_scripts/patroni.sh b/postgres-appliance/build_scripts/patroni.sh index e9fe60a79..8fc505419 100644 --- a/postgres-appliance/build_scripts/patroni.sh +++ b/postgres-appliance/build_scripts/patroni.sh @@ -41,7 +41,7 @@ else EXTRAS="" fi -pip3 install "patroni[kubernetes$EXTRAS]==$PATRONIVERSION" +pip3 install "patroni[kubernetes$EXTRAS] @ git+https://github.com/patroni/patroni.git@master" for d in /usr/local/lib/python3.10 /usr/lib/python3; do cd $d/dist-packages diff --git a/postgres-appliance/scripts/spilo_commons.py b/postgres-appliance/scripts/spilo_commons.py index 0877fab36..912ecd682 100644 --- a/postgres-appliance/scripts/spilo_commons.py +++ b/postgres-appliance/scripts/spilo_commons.py @@ -13,12 +13,12 @@ # (min_version, max_version, shared_preload_libraries, extwlist.extensions) extensions = { 'timescaledb': (9.6, 18, True, True), - 'pg_cron': (9.5, 18, True, False), - 'pg_stat_kcache': (9.4, 18, True, False), - 'pg_partman': (9.4, 18, False, True) + 'pg_cron': (9.5, 19, True, False), + 'pg_stat_kcache': (9.4, 19, True, False), + 'pg_partman': (9.4, 19, False, True) } if os.environ.get('ENABLE_PG_MON') == 'true': - extensions['pg_mon'] = (11, 18, True, False) + extensions['pg_mon'] = (11, 19, True, False) if os.environ.get('ENABLE_PG_TEXTSEARCH') == 'true': extensions['pg_textsearch'] = (17, 18, True, True) diff --git a/postgres-appliance/tests/docker-compose.yml b/postgres-appliance/tests/docker-compose.yml index 2dc660bdd..f09ebeb5e 100644 --- a/postgres-appliance/tests/docker-compose.yml +++ b/postgres-appliance/tests/docker-compose.yml @@ -51,7 +51,7 @@ services: postgresql: parameters: shared_buffers: 32MB - PGVERSION: '14' + PGVERSION: '15' # Just to test upgrade with clone. Without CLONE_SCOPE they don't work CLONE_WAL_S3_BUCKET: *bucket CLONE_AWS_ACCESS_KEY_ID: *access_key diff --git a/postgres-appliance/tests/test_spilo.sh b/postgres-appliance/tests/test_spilo.sh index a68c86a7f..85bd1d571 100755 --- a/postgres-appliance/tests/test_spilo.sh +++ b/postgres-appliance/tests/test_spilo.sh @@ -124,15 +124,15 @@ function drop_timescaledb() { } function test_inplace_upgrade_wrong_version() { - docker_exec "$1" "PGVERSION=14 $UPGRADE_SCRIPT 3" 2>&1 | grep 'Upgrade is not required' + docker_exec "$1" "PGVERSION=15 $UPGRADE_SCRIPT 3" 2>&1 | grep 'Upgrade is not required' } function test_inplace_upgrade_wrong_capacity() { - docker_exec "$1" "PGVERSION=15 $UPGRADE_SCRIPT 4" 2>&1 | grep 'number of replicas does not match' + docker_exec "$1" "PGVERSION=16 $UPGRADE_SCRIPT 4" 2>&1 | grep 'number of replicas does not match' } -function test_successful_inplace_upgrade_to_15() { - docker_exec "$1" "PGVERSION=15 $UPGRADE_SCRIPT 3" +function test_successful_inplace_upgrade_to_16() { + docker_exec "$1" "PGVERSION=16 $UPGRADE_SCRIPT 3" } function test_envdir_suffix() { @@ -146,11 +146,7 @@ function test_envdir_updated_to_x() { } function test_failed_inplace_upgrade_big_replication_lag() { - ! test_successful_inplace_upgrade_to_15 "$1" -} - -function test_successful_inplace_upgrade_to_16() { - docker_exec "$1" "PGVERSION=16 $UPGRADE_SCRIPT 3" + ! test_successful_inplace_upgrade_to_16 "$1" } function test_successful_inplace_upgrade_to_17() { @@ -161,8 +157,12 @@ function test_successful_inplace_upgrade_to_18() { docker_exec "$1" "PGVERSION=18 $UPGRADE_SCRIPT 3" } -function test_pg_upgrade_to_18_check_failed() { - ! test_successful_inplace_upgrade_to_18 "$1" +function test_successful_inplace_upgrade_to_19() { + docker_exec "$1" "PGVERSION=19 $UPGRADE_SCRIPT 3" +} + +function test_pg_upgrade_to_19_check_failed() { + ! test_successful_inplace_upgrade_to_19 "$1" } function start_clone_with_walg_upgrade_container() { @@ -170,7 +170,7 @@ function start_clone_with_walg_upgrade_container() { docker-compose run \ -e SCOPE=upgrade \ - -e PGVERSION=15 \ + -e PGVERSION=16 \ -e CLONE_SCOPE=demo \ -e CLONE_METHOD=CLONE_WITH_WALG \ -e CLONE_TARGET_TIME="$(next_minute)" \ @@ -183,27 +183,27 @@ function start_clone_with_walg_upgrade_replica_container() { start_clone_with_walg_upgrade_container 2 } -function start_clone_with_walg_upgrade_to_18_container() { +function start_clone_with_walg_upgrade_to_19_container() { docker-compose run \ -e SCOPE=upgrade3 \ - -e PGVERSION=18 \ + -e PGVERSION=19 \ -e CLONE_SCOPE=demo \ - -e CLONE_PGVERSION=14 \ + -e CLONE_PGVERSION=15 \ -e CLONE_METHOD=CLONE_WITH_WALG \ -e CLONE_TARGET_TIME="$(next_minute)" \ --name "${PREFIX}upgrade4" \ -d "spilo3" } -function start_clone_with_walg_18_container() { +function start_clone_with_walg_19_container() { docker-compose run \ - -e SCOPE=clone17 \ - -e PGVERSION=18 \ + -e SCOPE=clone18 \ + -e PGVERSION=19 \ -e CLONE_SCOPE=upgrade3 \ - -e CLONE_PGVERSION=18 \ + -e CLONE_PGVERSION=19 \ -e CLONE_METHOD=CLONE_WITH_WALG \ -e CLONE_TARGET_TIME="$(next_hour)" \ - --name "${PREFIX}clone17" \ + --name "${PREFIX}clone18" \ -d "spilo3" } @@ -211,7 +211,7 @@ function start_clone_with_basebackup_upgrade_container() { local container=$1 docker-compose run \ -e SCOPE=upgrade2 \ - -e PGVERSION=16 \ + -e PGVERSION=17 \ -e CLONE_SCOPE=upgrade \ -e CLONE_METHOD=CLONE_WITH_BASEBACKUP \ -e CLONE_HOST="$(docker_exec "$container" "hostname --ip-address")" \ @@ -225,10 +225,10 @@ function start_clone_with_basebackup_upgrade_container() { function start_clone_with_hourly_log_rotation() { docker-compose run \ -e SCOPE=hourlylogs \ - -e PGVERSION=18 \ + -e PGVERSION=19 \ -e LOG_SHIP_HOURLY="true" \ -e CLONE_SCOPE=upgrade2 \ - -e CLONE_PGVERSION=16 \ + -e CLONE_PGVERSION=17 \ -e CLONE_METHOD=CLONE_WITH_WALG \ -e CLONE_TARGET_TIME="$(next_minute)" \ --name "${PREFIX}hourlylogs" \ @@ -260,18 +260,18 @@ function verify_hourly_log_rotation() { [ "$log_rotation_age" = "1h" ] && [ "$log_filename" = "postgresql-%u-%H.log" ] && [ "$postgres_log_ftables" -eq 192 ] && [ "$postgres_log_views" -eq 8 ] && [ "$postgres_failed_auth_views" -eq 200 ] } -# TEST SUITE 1 - In-place major upgrade 14->15->...->18 -# TEST SUITE 2 - Major upgrade 14->18 after wal-g clone (with CLONE_PGVERSION set) -# TEST SUITE 3 - PITR (clone with wal-g) with unreachable target (15+) -# TEST SUITE 4 - Major upgrade 14->15 after wal-g clone (no CLONE_PGVERSION) +# TEST SUITE 1 - In-place major upgrade 15->16->...->19 +# TEST SUITE 2 - Major upgrade 15->19 after wal-g clone (with CLONE_PGVERSION set) +# TEST SUITE 3 - PITR (clone with wal-g) with unreachable target (16+) +# TEST SUITE 4 - Major upgrade 15->16 after wal-g clone (no CLONE_PGVERSION) # TEST SUITE 5 - Replica bootstrap with wal-g -# TEST SUITE 6 - Major upgrade 15->16 after clone with basebackup +# TEST SUITE 6 - Major upgrade 16->17 after clone with basebackup # TEST SUITE 7 - Hourly log rotation function test_spilo() { # TEST SUITE 1 local container=$1 - run_test test_envdir_suffix "$container" 14 + run_test test_envdir_suffix "$container" 15 log_info "[TS1] Testing wrong upgrade setups" run_test test_inplace_upgrade_wrong_version "$container" @@ -288,66 +288,66 @@ function test_spilo() { # TEST SUITE 2 local upgrade3_container - upgrade3_container=$(start_clone_with_walg_upgrade_to_18_container) # SCOPE=upgrade3 PGVERSION=18 CLONE: _SCOPE=demo _PGVERSION=14 _TARGET_TIME= - log_info "[TS2] Started $upgrade3_container for testing major upgrade 14->18 after clone with wal-g" + upgrade3_container=$(start_clone_with_walg_upgrade_to_19_container) # SCOPE=upgrade3 PGVERSION=19 CLONE: _SCOPE=demo _PGVERSION=15 _TARGET_TIME= + log_info "[TS2] Started $upgrade3_container for testing major upgrade 15->19 after clone with wal-g" # TEST SUITE 4 local upgrade_container - upgrade_container=$(start_clone_with_walg_upgrade_container) # SCOPE=upgrade PGVERSION=15 CLONE: _SCOPE=demo _TARGET_TIME= - log_info "[TS4] Started $upgrade_container for testing major upgrade 14->15 after clone with wal-g" + upgrade_container=$(start_clone_with_walg_upgrade_container) # SCOPE=upgrade PGVERSION=16 CLONE: _SCOPE=demo _TARGET_TIME= + log_info "[TS4] Started $upgrade_container for testing major upgrade 15->16 after clone with wal-g" # TEST SUITE 1 # wait clone to finish and prevent timescale installation gets cloned find_leader "$upgrade3_container" find_leader "$upgrade_container" - create_timescaledb "$container" # we don't install it at the beginning, as we do 14->18 in a clone + create_timescaledb "$container" # we don't install it at the beginning, as we do 15->19 in a clone - log_info "[TS1] Testing in-place major upgrade 14->15" + log_info "[TS1] Testing in-place major upgrade 15->16" wait_zero_lag "$container" - run_test test_successful_inplace_upgrade_to_15 "$container" + run_test test_successful_inplace_upgrade_to_16 "$container" wait_all_streaming "$container" - run_test test_envdir_updated_to_x 15 + run_test test_envdir_updated_to_x 16 # TEST SUITE 2 - log_info "[TS2] Testing in-place major upgrade 14->18 after wal-g clone" - run_test verify_clone_upgrade "$upgrade3_container" "wal-g" 14 18 + log_info "[TS2] Testing in-place major upgrade 15->19 after wal-g clone" + run_test verify_clone_upgrade "$upgrade3_container" "wal-g" 15 19 run_test verify_archive_mode_is_on "$upgrade3_container" wait_backup "$upgrade3_container" # TEST SUITE 3 - local clone18_container - clone18_container=$(start_clone_with_walg_18_container) # SCOPE=clone18 CLONE: _SCOPE=upgrade3 _PGVERSION=18 _TARGET_TIME= - log_info "[TS3] Started $clone18_container for testing point-in-time recovery (clone with wal-g) with unreachable target on 15+" + local clone19_container + clone19_container=$(start_clone_with_walg_19_container) # SCOPE=clone19 CLONE: _SCOPE=upgrade3 _PGVERSION=19 _TARGET_TIME= + log_info "[TS3] Started $clone19_container for testing point-in-time recovery (clone with wal-g) with unreachable target on 16+" # TEST SUITE 1 - log_info "[TS1] Testing in-place major upgrade 15->16" - run_test test_successful_inplace_upgrade_to_16 "$container" + log_info "[TS1] Testing in-place major upgrade 16->17" + run_test test_successful_inplace_upgrade_to_17 "$container" wait_all_streaming "$container" - run_test test_envdir_updated_to_x 16 + run_test test_envdir_updated_to_x 17 # TEST SUITE 3 - find_leader "$clone18_container" - run_test verify_archive_mode_is_on "$clone18_container" + find_leader "$clone19_container" + run_test verify_archive_mode_is_on "$clone19_container" # TEST SUITE 1 wait_backup "$container" - log_info "[TS1] Testing in-place major upgrade to 16->17" - run_test test_successful_inplace_upgrade_to_17 "$container" + log_info "[TS1] Testing in-place major upgrade to 17->18" + run_test test_successful_inplace_upgrade_to_18 "$container" wait_all_streaming "$container" - run_test test_envdir_updated_to_x 17 + run_test test_envdir_updated_to_x 18 # TEST SUITE 4 - log_info "[TS4] Testing in-place major upgrade 14->15 after clone with wal-g" - run_test verify_clone_upgrade "$upgrade_container" "wal-g" 14 15 + log_info "[TS4] Testing in-place major upgrade 15->16 after clone with wal-g" + run_test verify_clone_upgrade "$upgrade_container" "wal-g" 15 16 run_test verify_archive_mode_is_on "$upgrade_container" wait_backup "$upgrade_container" @@ -361,20 +361,20 @@ function test_spilo() { # TEST SUITE 6 local basebackup_container - basebackup_container=$(start_clone_with_basebackup_upgrade_container "$upgrade_container") # SCOPE=upgrade2 PGVERSION=16 CLONE: _SCOPE=upgrade - log_info "[TS6] Started $basebackup_container for testing major upgrade 15->16 after clone with basebackup" + basebackup_container=$(start_clone_with_basebackup_upgrade_container "$upgrade_container") # SCOPE=upgrade2 PGVERSION=17 CLONE: _SCOPE=upgrade + log_info "[TS6] Started $basebackup_container for testing major upgrade 16->17 after clone with basebackup" wait_backup "$basebackup_container" # TEST SUITE 1 - # run_test test_pg_upgrade_to_18_check_failed "$container" # pg_upgrade --check complains about timescaledb + run_test test_pg_upgrade_to_19_check_failed "$container" # pg_upgrade --check complains about timescaledb wait_backup "$container" drop_timescaledb "$container" - log_info "[TS1] Testing in-place major upgrade 17->18" - run_test test_successful_inplace_upgrade_to_18 "$container" + log_info "[TS1] Testing in-place major upgrade 18->19" + run_test test_successful_inplace_upgrade_to_19 "$container" wait_all_streaming "$container" - run_test test_envdir_updated_to_x 18 + run_test test_envdir_updated_to_x 19 # TEST SUITE 5 @@ -387,8 +387,8 @@ function test_spilo() { log_info "[TS7] Started $hourlylogs_container for testing hourly log rotation" # TEST SUITE 6 - log_info "[TS6] Testing in-place major upgrade 15->16 after clone with basebackup" - run_test verify_clone_upgrade "$basebackup_container" "basebackup" 15 16 + log_info "[TS6] Testing in-place major upgrade 16->17 after clone with basebackup" + run_test verify_clone_upgrade "$basebackup_container" "basebackup" 16 17 run_test verify_archive_mode_is_on "$basebackup_container" # TEST SUITE 7