From eda2f4850c470381098fda900e54055934c9b339 Mon Sep 17 00:00:00 2001 From: Maxim Smyatkin Date: Tue, 7 Jul 2026 18:35:33 +0300 Subject: [PATCH 1/2] Support GPDB varlena layout This commit allows cloudbery use GPDB varlena layout, otherwise upgrading from greenplum is blocked. Additionally bump catalog version and write varlena layout metadata (gpdb6 compatible or native Little Endian) in controldata. This allows us to prohibit using wrong binaries with wrong data format, preventing data corruption. --- src/backend/access/transam/xlog.c | 11 ++++++ src/bin/pg_controldata/pg_controldata.c | 2 ++ src/bin/pg_resetwal/pg_resetwal.c | 3 ++ src/bin/pg_upgrade/controldata.c | 31 +++++++++++++++++ src/bin/pg_upgrade/pg_upgrade.h | 7 ++++ src/include/catalog/pg_control.h | 22 +++++++++++- src/include/postgres.h | 46 ++++++++++++++++++++----- 7 files changed, 113 insertions(+), 9 deletions(-) diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index 17ebce7303e..123803488e3 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -4784,6 +4784,8 @@ WriteControlFile(void) ControlFile->float8ByVal = FLOAT8PASSBYVAL; + ControlFile->bigendian_varlena = BIGENDIAN_VARLENA_LAYOUT; + /* Contents are protected with a CRC */ INIT_CRC32C(ControlFile->crc); COMP_CRC32C(ControlFile->crc, @@ -5001,6 +5003,15 @@ ReadControlFile(void) errhint("It looks like you need to recompile or initdb."))); #endif + if (ControlFile->bigendian_varlena != BIGENDIAN_VARLENA_LAYOUT) + ereport(FATAL, + (errmsg("database files are incompatible with server"), + errdetail("The database cluster was initialized with %s varlena headers," + " but the server was compiled for %s.", + ControlFile->bigendian_varlena ? "network-byte-order (GPDB6-compatible)" : "native", + BIGENDIAN_VARLENA_LAYOUT ? "network-byte-order (GPDB6-compatible)" : "native"), + errhint("It looks like you need to recompile or initdb."))); + wal_segment_size = ControlFile->xlog_seg_size; if (!IsValidWalSegSize(wal_segment_size)) diff --git a/src/bin/pg_controldata/pg_controldata.c b/src/bin/pg_controldata/pg_controldata.c index ca5d3de023d..8e348b859ee 100644 --- a/src/bin/pg_controldata/pg_controldata.c +++ b/src/bin/pg_controldata/pg_controldata.c @@ -336,6 +336,8 @@ main(int argc, char *argv[]) _("64-bit integers")); printf(_("Float8 argument passing: %s\n"), (ControlFile->float8ByVal ? _("by value") : _("by reference"))); + printf(_("Varlena header byte order: %s\n"), + (ControlFile->bigendian_varlena ? _("network (GPDB6-compatible)") : _("native"))); printf(_("Data page checksum version: %u\n"), ControlFile->data_checksum_version); printf(_("Mock authentication nonce: %s\n"), diff --git a/src/bin/pg_resetwal/pg_resetwal.c b/src/bin/pg_resetwal/pg_resetwal.c index 7b660a75e49..d59284054e2 100644 --- a/src/bin/pg_resetwal/pg_resetwal.c +++ b/src/bin/pg_resetwal/pg_resetwal.c @@ -900,6 +900,7 @@ GuessControlValues(void) ControlFile.toast_max_chunk_size = TOAST_MAX_CHUNK_SIZE; ControlFile.loblksize = LOBLKSIZE; ControlFile.float8ByVal = FLOAT8PASSBYVAL; + ControlFile.bigendian_varlena = BIGENDIAN_VARLENA_LAYOUT; ControlFile.data_checksum_version = PG_DATA_CHECKSUM_VERSION; /* @@ -984,6 +985,8 @@ PrintControlValues(bool guessed) _("64-bit integers")); printf(_("Float8 argument passing: %s\n"), (ControlFile.float8ByVal ? _("by value") : _("by reference"))); + printf(_("Varlena header byte order: %s\n"), + (ControlFile.bigendian_varlena ? _("network (GPDB6-compatible)") : _("native"))); printf(_("Data page checksum version: %u\n"), ControlFile.data_checksum_version); printf(_("File encryption method: %s\n"), diff --git a/src/bin/pg_upgrade/controldata.c b/src/bin/pg_upgrade/controldata.c index 2b300819678..e51bbb3d3d3 100644 --- a/src/bin/pg_upgrade/controldata.c +++ b/src/bin/pg_upgrade/controldata.c @@ -66,6 +66,7 @@ get_control_data(ClusterInfo *cluster, bool live_check) bool got_toast = false; bool got_large_object = false; bool got_date_is_int = false; + bool got_varlena_layout = false; bool got_data_checksum_version = false; bool got_cluster_state = false; int got_file_encryption_method = false; @@ -544,6 +545,17 @@ get_control_data(ClusterInfo *cluster, bool live_check) cluster->controldata.date_is_int = strstr(p, "64-bit integers") != NULL; got_date_is_int = true; } + else if ((p = strstr(bufin, "Varlena header byte order:")) != NULL) + { + p = strchr(p, ':'); + + if (p == NULL || strlen(p) <= 1) + pg_fatal("%d: controldata retrieval problem\n", __LINE__); + + p++; /* remove ':' char */ + cluster->controldata.varlena_bigendian = strstr(p, "network") != NULL; + got_varlena_layout = true; + } else if ((p = strstr(bufin, "checksum")) != NULL) { p = strchr(p, ':'); @@ -645,6 +657,8 @@ get_control_data(ClusterInfo *cluster, bool live_check) !got_index || /* !got_toast || */ (!got_large_object && cluster->controldata.ctrl_ver >= LARGE_OBJECT_SIZE_PG_CONTROL_VER) || + (!got_varlena_layout && + cluster->controldata.ctrl_ver >= VARLENA_LAYOUT_PG_CONTROL_VER) || !got_date_is_int || !got_data_checksum_version || !got_file_encryption_method) { @@ -713,6 +727,10 @@ get_control_data(ClusterInfo *cluster, bool live_check) cluster->controldata.ctrl_ver >= LARGE_OBJECT_SIZE_PG_CONTROL_VER) pg_log(PG_REPORT, " large-object chunk size\n"); + if (!got_varlena_layout && + cluster->controldata.ctrl_ver >= VARLENA_LAYOUT_PG_CONTROL_VER) + pg_log(PG_REPORT, " varlena header byte order\n"); + if (!got_date_is_int) pg_log(PG_REPORT, " dates/times are integers?\n"); @@ -769,6 +787,19 @@ check_control_data(ControlData *oldctrl, oldctrl->large_object != newctrl->large_object) pg_fatal("old and new pg_controldata large-object chunk sizes are invalid or do not match\n"); + /* + * On-disk varlena header layout (bigendian_varlena) is recorded in + * pg_control only since VARLENA_LAYOUT_PG_CONTROL_VER. Older clusters + * (GPDB6, pre-feature Cloudberry) don't report it, so only enforce the + * match when the old cluster actually carries the field. A mismatch means + * every variable-length datum would be misread, so this must be fatal. + */ + if (oldctrl->ctrl_ver >= VARLENA_LAYOUT_PG_CONTROL_VER && + oldctrl->varlena_bigendian != newctrl->varlena_bigendian) + pg_fatal("old and new clusters use different on-disk varlena header layouts\n" + "The new cluster must be built to match the old cluster's layout\n" + "(rebuild with or without -DFORCE_BIGENDIAN_VARLENA to match).\n"); + /* * GPDB, since 9.5, pg_upgrade removed the support for 8.3, however, GPDB * still keep it to support upgrading from GPDB 5 diff --git a/src/bin/pg_upgrade/pg_upgrade.h b/src/bin/pg_upgrade/pg_upgrade.h index 2938ea76845..256d07d4db2 100644 --- a/src/bin/pg_upgrade/pg_upgrade.h +++ b/src/bin/pg_upgrade/pg_upgrade.h @@ -193,6 +193,12 @@ typedef enum */ #define LARGE_OBJECT_SIZE_PG_CONTROL_VER 942 +/* + * varlena header byte order (bigendian_varlena) added to pg_control in + * Cloudberry to guard GPDB6-compatible vs native on-disk varlena layout. + */ +#define VARLENA_LAYOUT_PG_CONTROL_VER 13000701 + /* * change in JSONB format during 9.4 beta */ @@ -323,6 +329,7 @@ typedef struct uint32 large_object; bool date_is_int; bool float8_pass_by_value; + bool varlena_bigendian; uint32 data_checksum_version; int file_encryption_method; } ControlData; diff --git a/src/include/catalog/pg_control.h b/src/include/catalog/pg_control.h index c3f87cfd14c..43bfdb37f9e 100644 --- a/src/include/catalog/pg_control.h +++ b/src/include/catalog/pg_control.h @@ -27,11 +27,25 @@ * The first four digits is the PostgreSQL version number. The last * four digits indicates the GPDB version. */ -#define PG_CONTROL_VERSION 13000700 +#define PG_CONTROL_VERSION 13000701 /* Nonce key length, see below */ #define MOCK_AUTH_NONCE_LEN 32 +/* + * On-disk varlena header byte order produced by this build, stamped into + * ControlFileData.bigendian_varlena. 1 == big-endian/network order (real + * big-endian hardware, or -DFORCE_BIGENDIAN_VARLENA for GPDB6 compatibility; + * see the varlena macros in postgres.h); 0 == upstream native little-endian. + * Defined here rather than in postgres.h so the frontend tools (pg_resetwal, + * pg_controldata, pg_upgrade) that only include pg_control.h can see it. + */ +#if defined(WORDS_BIGENDIAN) || defined(FORCE_BIGENDIAN_VARLENA) +#define BIGENDIAN_VARLENA_LAYOUT 1 +#else +#define BIGENDIAN_VARLENA_LAYOUT 0 +#endif + /* * Body of CheckPoint XLOG records. This is declared here because we keep * a copy of the latest one in pg_control for possible disaster recovery. @@ -230,6 +244,12 @@ typedef struct ControlFileData bool float8ByVal; /* float8, int8, etc pass-by-value? */ + /* + * On-disk varlena header byte order (GPDB6-compatible big-endian/network + * order vs upstream native). See BIGENDIAN_VARLENA_LAYOUT above. + */ + bool bigendian_varlena; + /* Are data pages protected by checksums? Zero if no checksum version */ uint32 data_checksum_version; diff --git a/src/include/postgres.h b/src/include/postgres.h index 90fd1f29d39..92b2e13c07d 100644 --- a/src/include/postgres.h +++ b/src/include/postgres.h @@ -211,12 +211,15 @@ typedef struct * the specific type and length of the pointer datum. * * NOTE: - * Greenplum differs from PostgreSQL here... In Postgres, it use different - * macros for big-endian and little-endian machines, so the length is contiguous, - * while the 4 byte lengths are stored in native endian format. + * PostgreSQL uses different macros for big-endian and little-endian machines: + * flag bits sit in the physically first byte in both cases, and 4-byte lengths + * are stored in native endian format. * - * Greenplum stored the 4 byte varlena header in network byte order, so it always - * look big-endian in the tuple. + * GPDB 6 (and earlier) stored the 4-byte varlena header in network byte order + * unconditionally (htonl), so it always looked big-endian in the tuple. Stock + * Cloudberry on little-endian follows upstream's native layout. Building with + * -DFORCE_BIGENDIAN_VARLENA restores the GPDB6 encoding for binary pg_upgrade + * compatibility (see below). * */ @@ -229,7 +232,22 @@ typedef struct * checking for IS_1B. */ -#ifdef WORDS_BIGENDIAN +#if defined(FORCE_BIGENDIAN_VARLENA) || defined(WORDS_BIGENDIAN) + +/* + * Big-endian on-disk layout (real big-endian hardware, or little-endian with + * -DFORCE_BIGENDIAN_VARLENA for GPDB6 compatibility). + * + * Flag bits and 1-byte (short) / external headers are byte-level and shared. + * On big-endian hardware the native uint32 already stores the 4-byte length in + * network byte order, so no swap is needed. On little-endian with + * FORCE_BIGENDIAN_VARLENA the native uint32 is not in network order, so the + * 4-byte length word must be swapped with htonl/ntohl. That also makes newly + * written data GPDB6-format, so the entire cluster must be built this way for + * self-consistency. The compressed second word (va_tcinfo) needs no swap: + * GPDB6 stored it natively as well, and its compression-method bits are 0 + * (== PGLZ), which is what Cloudberry expects. + */ #define VARATT_IS_4B(PTR) \ ((((varattrib_1b *) (PTR))->va_header & 0x80) == 0x00) @@ -245,17 +263,29 @@ typedef struct (*((uint8 *) (PTR)) != 0) /* VARSIZE_4B() should only be used on known-aligned data */ +#ifdef FORCE_BIGENDIAN_VARLENA +#define VARSIZE_4B(PTR) \ + (ntohl(((varattrib_4b *) (PTR))->va_4byte.va_header) & 0x3FFFFFFF) +#else /* WORDS_BIGENDIAN */ #define VARSIZE_4B(PTR) \ (((varattrib_4b *) (PTR))->va_4byte.va_header & 0x3FFFFFFF) +#endif #define VARSIZE_1B(PTR) \ (((varattrib_1b *) (PTR))->va_header & 0x7F) #define VARTAG_1B_E(PTR) \ (((varattrib_1b_e *) (PTR))->va_tag) +#ifdef FORCE_BIGENDIAN_VARLENA +#define SET_VARSIZE_4B(PTR,len) \ + (((varattrib_4b *) (PTR))->va_4byte.va_header = htonl((len) & 0x3FFFFFFF)) +#define SET_VARSIZE_4B_C(PTR,len) \ + (((varattrib_4b *) (PTR))->va_4byte.va_header = htonl(((len) & 0x3FFFFFFF) | 0x40000000)) +#else /* WORDS_BIGENDIAN */ #define SET_VARSIZE_4B(PTR,len) \ (((varattrib_4b *) (PTR))->va_4byte.va_header = (len) & 0x3FFFFFFF) #define SET_VARSIZE_4B_C(PTR,len) \ (((varattrib_4b *) (PTR))->va_4byte.va_header = ((len) & 0x3FFFFFFF) | 0x40000000) +#endif #define SET_VARSIZE_1B(PTR,len) \ (((varattrib_1b *) (PTR))->va_header = (len) | 0x80) #define SET_VARTAG_1B_E(PTR,tag) \ @@ -263,7 +293,7 @@ typedef struct ((varattrib_1b_e *) (PTR))->va_tag = (tag)) #define VARSIZE_TO_SHORT(PTR) ((char)(VARSIZE(PTR)-VARHDRSZ+VARHDRSZ_SHORT) | 0x80) -#else /* !WORDS_BIGENDIAN */ +#else /* !WORDS_BIGENDIAN && !FORCE_BIGENDIAN_VARLENA */ #define VARATT_IS_4B(PTR) \ ((((varattrib_1b *) (PTR))->va_header & 0x01) == 0x00) @@ -297,7 +327,7 @@ typedef struct ((varattrib_1b_e *) (PTR))->va_tag = (tag)) #define VARSIZE_TO_SHORT(PTR) ((char)((VARSIZE(PTR)-VARHDRSZ+VARHDRSZ_SHORT) << 1) | 0x01) -#endif /* WORDS_BIGENDIAN */ +#endif /* WORDS_BIGENDIAN / FORCE_BIGENDIAN_VARLENA */ #define VARDATA_4B(PTR) (((varattrib_4b *) (PTR))->va_4byte.va_data) #define VARDATA_4B_C(PTR) (((varattrib_4b *) (PTR))->va_compressed.va_data) From dc3ff3ca34523f69b4f2e6aacf011b68ca31c397 Mon Sep 17 00:00:00 2001 From: Maxim Smyatkin Date: Tue, 28 Jul 2026 21:27:01 +0300 Subject: [PATCH 2/2] Add legacy varlena option to configure And default it to true for our builds --- configure | 37 +++++++++++++++++++ configure.ac | 16 ++++++++ .../scripts/configure-cloudberry.sh | 1 + src/include/pg_config.h.in | 3 ++ 4 files changed, 57 insertions(+) diff --git a/configure b/configure index 586e4d0cf6a..2eaf177b03a 100755 --- a/configure +++ b/configure @@ -725,6 +725,7 @@ with_zstd with_yezzey PROTOC with_gp_stats_collector +with_varlena_gpdb_layout with_diskquota with_zstd with_libbz2 @@ -949,6 +950,7 @@ with_libbz2 with_zstd with_diskquota with_gp_stats_collector +with_varlena_gpdb_layout with_yezzey with_rt with_libcurl @@ -11455,6 +11457,41 @@ else fi +# +# varlena_gpdb_layout +# + + + +# Check whether --with-varlena_gpdb_layout was given. +if test "${with_varlena_gpdb_layout+set}" = set; then :\ + withval=$with_varlena_gpdb_layout; + case $withval in + yes) + : + ;; + no) + : + ;; + *) + as_fn_error $? "no argument expected for --with-varlena_gpdb_layout option" "$LINENO" 5 + ;; + esac + +else $as_nop + with_varlena_gpdb_layout=no + +fi + + + + +if test "$with_varlena_gpdb_layout" = yes; then + printf "%s\n" "#define FORCE_BIGENDIAN_VARLENA 1" >>confdefs.h + CFLAGS="$CFLAGS -DFORCE_BIGENDIAN_VARLENA" + CXXFLAGS="$CXXFLAGS -DFORCE_BIGENDIAN_VARLENA" +fi + # # yezzey # diff --git a/configure.ac b/configure.ac index 58fa651ee2e..2b70af608c7 100644 --- a/configure.ac +++ b/configure.ac @@ -1419,6 +1419,22 @@ PGAC_ARG_BOOL(with, gp_stats_collector, yes, [build with gp_stats_collector extension]) AC_SUBST(with_gp_stats_collector) + +# +# varlena_gpdb_layout +# +PGAC_ARG_BOOL(with, varlena_gpdb_layout, no, + [force varlena headers to use legacy gpdb layout]) +AC_SUBST(with_varlena_gpdb_layout) + +if test "$with_varlena_gpdb_layout" = yes; then + AC_DEFINE(FORCE_BIGENDIAN_VARLENA, 1, + [Define to use GPDB6-compatible big-endian/network-order varlena headers.]) + # Keep -D in both flag sets for any TU that does not include pg_config.h. + CFLAGS="$CFLAGS -DFORCE_BIGENDIAN_VARLENA" + CXXFLAGS="$CXXFLAGS -DFORCE_BIGENDIAN_VARLENA" +fi + # # Realtime library # diff --git a/devops/build/automation/cloudberry/scripts/configure-cloudberry.sh b/devops/build/automation/cloudberry/scripts/configure-cloudberry.sh index 89cc9f721f2..7ec0302821d 100755 --- a/devops/build/automation/cloudberry/scripts/configure-cloudberry.sh +++ b/devops/build/automation/cloudberry/scripts/configure-cloudberry.sh @@ -181,6 +181,7 @@ execute_cmd ./configure --prefix=${BUILD_DESTINATION} \ ${CONFIGURE_MDBLOCALES_OPTS} \ --with-includes=/usr/local/xerces-c/include \ --with-libraries=${BUILD_DESTINATION}/lib \ + --with-varlena_gpdb_layout \ ${CONFIGURE_EXTRA_OPTS:-""} || exit 4 log_section_end "Configure" diff --git a/src/include/pg_config.h.in b/src/include/pg_config.h.in index b037fc11501..1233958cab4 100644 --- a/src/include/pg_config.h.in +++ b/src/include/pg_config.h.in @@ -1127,6 +1127,9 @@ # endif #endif +/* Define to use GPDB6-compatible big-endian/network-order varlena headers. */ +#undef FORCE_BIGENDIAN_VARLENA + /* Size of a WAL file block. This need have no particular relation to BLCKSZ. XLOG_BLCKSZ must be a power of 2, and if your system supports O_DIRECT I/O, XLOG_BLCKSZ must be a multiple of the alignment requirement for direct-I/O