diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index 17ebce7303e..123803488e3 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -4784,6 +4784,8 @@ WriteControlFile(void) ControlFile->float8ByVal = FLOAT8PASSBYVAL; + ControlFile->bigendian_varlena = BIGENDIAN_VARLENA_LAYOUT; + /* Contents are protected with a CRC */ INIT_CRC32C(ControlFile->crc); COMP_CRC32C(ControlFile->crc, @@ -5001,6 +5003,15 @@ ReadControlFile(void) errhint("It looks like you need to recompile or initdb."))); #endif + if (ControlFile->bigendian_varlena != BIGENDIAN_VARLENA_LAYOUT) + ereport(FATAL, + (errmsg("database files are incompatible with server"), + errdetail("The database cluster was initialized with %s varlena headers," + " but the server was compiled for %s.", + ControlFile->bigendian_varlena ? "network-byte-order (GPDB6-compatible)" : "native", + BIGENDIAN_VARLENA_LAYOUT ? "network-byte-order (GPDB6-compatible)" : "native"), + errhint("It looks like you need to recompile or initdb."))); + wal_segment_size = ControlFile->xlog_seg_size; if (!IsValidWalSegSize(wal_segment_size)) diff --git a/src/bin/pg_controldata/pg_controldata.c b/src/bin/pg_controldata/pg_controldata.c index ca5d3de023d..8e348b859ee 100644 --- a/src/bin/pg_controldata/pg_controldata.c +++ b/src/bin/pg_controldata/pg_controldata.c @@ -336,6 +336,8 @@ main(int argc, char *argv[]) _("64-bit integers")); printf(_("Float8 argument passing: %s\n"), (ControlFile->float8ByVal ? _("by value") : _("by reference"))); + printf(_("Varlena header byte order: %s\n"), + (ControlFile->bigendian_varlena ? _("network (GPDB6-compatible)") : _("native"))); printf(_("Data page checksum version: %u\n"), ControlFile->data_checksum_version); printf(_("Mock authentication nonce: %s\n"), diff --git a/src/bin/pg_resetwal/pg_resetwal.c b/src/bin/pg_resetwal/pg_resetwal.c index 7b660a75e49..d59284054e2 100644 --- a/src/bin/pg_resetwal/pg_resetwal.c +++ b/src/bin/pg_resetwal/pg_resetwal.c @@ -900,6 +900,7 @@ GuessControlValues(void) ControlFile.toast_max_chunk_size = TOAST_MAX_CHUNK_SIZE; ControlFile.loblksize = LOBLKSIZE; ControlFile.float8ByVal = FLOAT8PASSBYVAL; + ControlFile.bigendian_varlena = BIGENDIAN_VARLENA_LAYOUT; ControlFile.data_checksum_version = PG_DATA_CHECKSUM_VERSION; /* @@ -984,6 +985,8 @@ PrintControlValues(bool guessed) _("64-bit integers")); printf(_("Float8 argument passing: %s\n"), (ControlFile.float8ByVal ? _("by value") : _("by reference"))); + printf(_("Varlena header byte order: %s\n"), + (ControlFile.bigendian_varlena ? _("network (GPDB6-compatible)") : _("native"))); printf(_("Data page checksum version: %u\n"), ControlFile.data_checksum_version); printf(_("File encryption method: %s\n"), diff --git a/src/bin/pg_upgrade/controldata.c b/src/bin/pg_upgrade/controldata.c index 2b300819678..e51bbb3d3d3 100644 --- a/src/bin/pg_upgrade/controldata.c +++ b/src/bin/pg_upgrade/controldata.c @@ -66,6 +66,7 @@ get_control_data(ClusterInfo *cluster, bool live_check) bool got_toast = false; bool got_large_object = false; bool got_date_is_int = false; + bool got_varlena_layout = false; bool got_data_checksum_version = false; bool got_cluster_state = false; int got_file_encryption_method = false; @@ -544,6 +545,17 @@ get_control_data(ClusterInfo *cluster, bool live_check) cluster->controldata.date_is_int = strstr(p, "64-bit integers") != NULL; got_date_is_int = true; } + else if ((p = strstr(bufin, "Varlena header byte order:")) != NULL) + { + p = strchr(p, ':'); + + if (p == NULL || strlen(p) <= 1) + pg_fatal("%d: controldata retrieval problem\n", __LINE__); + + p++; /* remove ':' char */ + cluster->controldata.varlena_bigendian = strstr(p, "network") != NULL; + got_varlena_layout = true; + } else if ((p = strstr(bufin, "checksum")) != NULL) { p = strchr(p, ':'); @@ -645,6 +657,8 @@ get_control_data(ClusterInfo *cluster, bool live_check) !got_index || /* !got_toast || */ (!got_large_object && cluster->controldata.ctrl_ver >= LARGE_OBJECT_SIZE_PG_CONTROL_VER) || + (!got_varlena_layout && + cluster->controldata.ctrl_ver >= VARLENA_LAYOUT_PG_CONTROL_VER) || !got_date_is_int || !got_data_checksum_version || !got_file_encryption_method) { @@ -713,6 +727,10 @@ get_control_data(ClusterInfo *cluster, bool live_check) cluster->controldata.ctrl_ver >= LARGE_OBJECT_SIZE_PG_CONTROL_VER) pg_log(PG_REPORT, " large-object chunk size\n"); + if (!got_varlena_layout && + cluster->controldata.ctrl_ver >= VARLENA_LAYOUT_PG_CONTROL_VER) + pg_log(PG_REPORT, " varlena header byte order\n"); + if (!got_date_is_int) pg_log(PG_REPORT, " dates/times are integers?\n"); @@ -769,6 +787,19 @@ check_control_data(ControlData *oldctrl, oldctrl->large_object != newctrl->large_object) pg_fatal("old and new pg_controldata large-object chunk sizes are invalid or do not match\n"); + /* + * On-disk varlena header layout (bigendian_varlena) is recorded in + * pg_control only since VARLENA_LAYOUT_PG_CONTROL_VER. Older clusters + * (GPDB6, pre-feature Cloudberry) don't report it, so only enforce the + * match when the old cluster actually carries the field. A mismatch means + * every variable-length datum would be misread, so this must be fatal. + */ + if (oldctrl->ctrl_ver >= VARLENA_LAYOUT_PG_CONTROL_VER && + oldctrl->varlena_bigendian != newctrl->varlena_bigendian) + pg_fatal("old and new clusters use different on-disk varlena header layouts\n" + "The new cluster must be built to match the old cluster's layout\n" + "(rebuild with or without -DFORCE_BIGENDIAN_VARLENA to match).\n"); + /* * GPDB, since 9.5, pg_upgrade removed the support for 8.3, however, GPDB * still keep it to support upgrading from GPDB 5 diff --git a/src/bin/pg_upgrade/pg_upgrade.h b/src/bin/pg_upgrade/pg_upgrade.h index 2938ea76845..256d07d4db2 100644 --- a/src/bin/pg_upgrade/pg_upgrade.h +++ b/src/bin/pg_upgrade/pg_upgrade.h @@ -193,6 +193,12 @@ typedef enum */ #define LARGE_OBJECT_SIZE_PG_CONTROL_VER 942 +/* + * varlena header byte order (bigendian_varlena) added to pg_control in + * Cloudberry to guard GPDB6-compatible vs native on-disk varlena layout. + */ +#define VARLENA_LAYOUT_PG_CONTROL_VER 13000701 + /* * change in JSONB format during 9.4 beta */ @@ -323,6 +329,7 @@ typedef struct uint32 large_object; bool date_is_int; bool float8_pass_by_value; + bool varlena_bigendian; uint32 data_checksum_version; int file_encryption_method; } ControlData; diff --git a/src/include/catalog/pg_control.h b/src/include/catalog/pg_control.h index c3f87cfd14c..43bfdb37f9e 100644 --- a/src/include/catalog/pg_control.h +++ b/src/include/catalog/pg_control.h @@ -27,11 +27,25 @@ * The first four digits is the PostgreSQL version number. The last * four digits indicates the GPDB version. */ -#define PG_CONTROL_VERSION 13000700 +#define PG_CONTROL_VERSION 13000701 /* Nonce key length, see below */ #define MOCK_AUTH_NONCE_LEN 32 +/* + * On-disk varlena header byte order produced by this build, stamped into + * ControlFileData.bigendian_varlena. 1 == big-endian/network order (real + * big-endian hardware, or -DFORCE_BIGENDIAN_VARLENA for GPDB6 compatibility; + * see the varlena macros in postgres.h); 0 == upstream native little-endian. + * Defined here rather than in postgres.h so the frontend tools (pg_resetwal, + * pg_controldata, pg_upgrade) that only include pg_control.h can see it. + */ +#if defined(WORDS_BIGENDIAN) || defined(FORCE_BIGENDIAN_VARLENA) +#define BIGENDIAN_VARLENA_LAYOUT 1 +#else +#define BIGENDIAN_VARLENA_LAYOUT 0 +#endif + /* * Body of CheckPoint XLOG records. This is declared here because we keep * a copy of the latest one in pg_control for possible disaster recovery. @@ -230,6 +244,12 @@ typedef struct ControlFileData bool float8ByVal; /* float8, int8, etc pass-by-value? */ + /* + * On-disk varlena header byte order (GPDB6-compatible big-endian/network + * order vs upstream native). See BIGENDIAN_VARLENA_LAYOUT above. + */ + bool bigendian_varlena; + /* Are data pages protected by checksums? Zero if no checksum version */ uint32 data_checksum_version; diff --git a/src/include/postgres.h b/src/include/postgres.h index 90fd1f29d39..92b2e13c07d 100644 --- a/src/include/postgres.h +++ b/src/include/postgres.h @@ -211,12 +211,15 @@ typedef struct * the specific type and length of the pointer datum. * * NOTE: - * Greenplum differs from PostgreSQL here... In Postgres, it use different - * macros for big-endian and little-endian machines, so the length is contiguous, - * while the 4 byte lengths are stored in native endian format. + * PostgreSQL uses different macros for big-endian and little-endian machines: + * flag bits sit in the physically first byte in both cases, and 4-byte lengths + * are stored in native endian format. * - * Greenplum stored the 4 byte varlena header in network byte order, so it always - * look big-endian in the tuple. + * GPDB 6 (and earlier) stored the 4-byte varlena header in network byte order + * unconditionally (htonl), so it always looked big-endian in the tuple. Stock + * Cloudberry on little-endian follows upstream's native layout. Building with + * -DFORCE_BIGENDIAN_VARLENA restores the GPDB6 encoding for binary pg_upgrade + * compatibility (see below). * */ @@ -229,7 +232,22 @@ typedef struct * checking for IS_1B. */ -#ifdef WORDS_BIGENDIAN +#if defined(FORCE_BIGENDIAN_VARLENA) || defined(WORDS_BIGENDIAN) + +/* + * Big-endian on-disk layout (real big-endian hardware, or little-endian with + * -DFORCE_BIGENDIAN_VARLENA for GPDB6 compatibility). + * + * Flag bits and 1-byte (short) / external headers are byte-level and shared. + * On big-endian hardware the native uint32 already stores the 4-byte length in + * network byte order, so no swap is needed. On little-endian with + * FORCE_BIGENDIAN_VARLENA the native uint32 is not in network order, so the + * 4-byte length word must be swapped with htonl/ntohl. That also makes newly + * written data GPDB6-format, so the entire cluster must be built this way for + * self-consistency. The compressed second word (va_tcinfo) needs no swap: + * GPDB6 stored it natively as well, and its compression-method bits are 0 + * (== PGLZ), which is what Cloudberry expects. + */ #define VARATT_IS_4B(PTR) \ ((((varattrib_1b *) (PTR))->va_header & 0x80) == 0x00) @@ -245,17 +263,29 @@ typedef struct (*((uint8 *) (PTR)) != 0) /* VARSIZE_4B() should only be used on known-aligned data */ +#ifdef FORCE_BIGENDIAN_VARLENA +#define VARSIZE_4B(PTR) \ + (ntohl(((varattrib_4b *) (PTR))->va_4byte.va_header) & 0x3FFFFFFF) +#else /* WORDS_BIGENDIAN */ #define VARSIZE_4B(PTR) \ (((varattrib_4b *) (PTR))->va_4byte.va_header & 0x3FFFFFFF) +#endif #define VARSIZE_1B(PTR) \ (((varattrib_1b *) (PTR))->va_header & 0x7F) #define VARTAG_1B_E(PTR) \ (((varattrib_1b_e *) (PTR))->va_tag) +#ifdef FORCE_BIGENDIAN_VARLENA +#define SET_VARSIZE_4B(PTR,len) \ + (((varattrib_4b *) (PTR))->va_4byte.va_header = htonl((len) & 0x3FFFFFFF)) +#define SET_VARSIZE_4B_C(PTR,len) \ + (((varattrib_4b *) (PTR))->va_4byte.va_header = htonl(((len) & 0x3FFFFFFF) | 0x40000000)) +#else /* WORDS_BIGENDIAN */ #define SET_VARSIZE_4B(PTR,len) \ (((varattrib_4b *) (PTR))->va_4byte.va_header = (len) & 0x3FFFFFFF) #define SET_VARSIZE_4B_C(PTR,len) \ (((varattrib_4b *) (PTR))->va_4byte.va_header = ((len) & 0x3FFFFFFF) | 0x40000000) +#endif #define SET_VARSIZE_1B(PTR,len) \ (((varattrib_1b *) (PTR))->va_header = (len) | 0x80) #define SET_VARTAG_1B_E(PTR,tag) \ @@ -263,7 +293,7 @@ typedef struct ((varattrib_1b_e *) (PTR))->va_tag = (tag)) #define VARSIZE_TO_SHORT(PTR) ((char)(VARSIZE(PTR)-VARHDRSZ+VARHDRSZ_SHORT) | 0x80) -#else /* !WORDS_BIGENDIAN */ +#else /* !WORDS_BIGENDIAN && !FORCE_BIGENDIAN_VARLENA */ #define VARATT_IS_4B(PTR) \ ((((varattrib_1b *) (PTR))->va_header & 0x01) == 0x00) @@ -297,7 +327,7 @@ typedef struct ((varattrib_1b_e *) (PTR))->va_tag = (tag)) #define VARSIZE_TO_SHORT(PTR) ((char)((VARSIZE(PTR)-VARHDRSZ+VARHDRSZ_SHORT) << 1) | 0x01) -#endif /* WORDS_BIGENDIAN */ +#endif /* WORDS_BIGENDIAN / FORCE_BIGENDIAN_VARLENA */ #define VARDATA_4B(PTR) (((varattrib_4b *) (PTR))->va_4byte.va_data) #define VARDATA_4B_C(PTR) (((varattrib_4b *) (PTR))->va_compressed.va_data)