New header varatt.h split off from postgres.h

This new header contains all the variable-length data types support
(TOAST support) from postgres.h, which isn't needed by large parts of
the backend code.

Reviewed-by: Tom Lane <tgl@sss.pgh.pa.us>
Discussion: https://www.postgresql.org/message-id/flat/ddcce239-0f29-6e62-4b47-1f8ca742addf%40enterprisedb.com
This commit is contained in:
Peter Eisentraut 2023-01-10 05:48:59 +01:00
parent cd06ccd78f
commit d952373a98
50 changed files with 412 additions and 353 deletions

View File

@ -8,6 +8,7 @@
#include "utils/builtins.h"
#include "utils/formatting.h"
#include "utils/varlena.h"
#include "varatt.h"
PG_MODULE_MAGIC;

View File

@ -9,6 +9,7 @@
#include "cubedata.h"
#include "nodes/miscnodes.h"
#include "utils/float.h"
#include "varatt.h"
/* All grammar constructs return strings */
#define YYSTYPE char *

View File

@ -43,6 +43,7 @@
#include "mb/pg_wchar.h"
#include "utils/builtins.h"
#include "utils/varlena.h"
#include "varatt.h"
PG_MODULE_MAGIC;

View File

@ -11,6 +11,7 @@
#include "libpq/pqformat.h"
#include "ltree.h"
#include "utils/memutils.h"
#include "varatt.h"
typedef struct

View File

@ -12,6 +12,7 @@
#include "ltree.h"
#include "miscadmin.h"
#include "nodes/miscnodes.h"
#include "varatt.h"
/* parser's states */

View File

@ -7,6 +7,7 @@
#include "access/stratnum.h"
#include "fmgr.h"
#include "trgm.h"
#include "varatt.h"
PG_FUNCTION_INFO_V1(gin_extract_trgm);
PG_FUNCTION_INFO_V1(gin_extract_value_trgm);

View File

@ -8,6 +8,7 @@
#include "fmgr.h"
#include "port/pg_bitutils.h"
#include "trgm.h"
#include "varatt.h"
/* gist_trgm_ops opclass options */
typedef struct

View File

@ -196,6 +196,7 @@
#include "tsearch/ts_locale.h"
#include "utils/hsearch.h"
#include "utils/memutils.h"
#include "varatt.h"
/*
* Uncomment (or use -DTRGM_REGEXP_DEBUG) to print debug info,

View File

@ -39,6 +39,7 @@
#include "px.h"
#include "utils/builtins.h"
#include "utils/uuid.h"
#include "varatt.h"
PG_MODULE_MAGIC;

View File

@ -19,6 +19,7 @@
#include "port/pg_bswap.h"
#include "utils/builtins.h"
#include "utils/uuid.h"
#include "varatt.h"
/*
* It's possible that there's more than one uuid.h header file present.

View File

@ -2390,6 +2390,7 @@ PG_FUNCTION_INFO_V1(funcname);
#include <string.h>
#include "fmgr.h"
#include "utils/geo_decls.h"
#include "varatt.h"
PG_MODULE_MAGIC;

View File

@ -22,6 +22,7 @@
#include "common/pg_lzcompress.h"
#include "fmgr.h"
#include "utils/builtins.h"
#include "varatt.h"
/* GUC */
int default_toast_compression = TOAST_PGLZ_COMPRESSION;

View File

@ -32,6 +32,7 @@
#include "utils/builtins.h"
#include "utils/float.h"
#include "utils/pg_locale.h"
#include "varatt.h"
/*
* Datatype-specific hash functions.

View File

@ -46,6 +46,7 @@
#include "utils/datum.h"
#include "utils/pg_locale.h"
#include "utils/varlena.h"
#include "varatt.h"
/*

View File

@ -19,6 +19,7 @@
#include "access/toast_helper.h"
#include "access/toast_internals.h"
#include "catalog/pg_type_d.h"
#include "varatt.h"
/*

View File

@ -53,6 +53,7 @@
#include "utils/builtins.h"
#include "utils/memutils.h"
#include "utils/snapmgr.h"
#include "varatt.h"
/* define this to enable debug logging */
/* #define FSDB 1 */

View File

@ -77,6 +77,7 @@
#include "libpq/pqformat.h"
#include "mb/pg_wchar.h"
#include "port/pg_bswap.h"
#include "varatt.h"
/* --------------------------------

View File

@ -16,6 +16,7 @@
#include "tsearch/ts_cache.h"
#include "tsearch/ts_utils.h"
#include "varatt.h"
#define IGNORE_LONGLEXEME 1

View File

@ -19,6 +19,7 @@
#include "common/hashfn.h"
#include "tsearch/ts_type.h"
#include "utils/builtins.h"
#include "varatt.h"
/* A hash key for lexemes */

View File

@ -14,6 +14,7 @@
#include "mb/pg_wchar.h"
#include "utils/ascii.h"
#include "utils/builtins.h"
#include "varatt.h"
static void pg_to_ascii(unsigned char *src, unsigned char *src_end,
unsigned char *dest, int enc);

View File

@ -19,6 +19,7 @@
#include "libpq/pqformat.h"
#include "utils/builtins.h"
#include "varatt.h"
#define ISOCTAL(c) (((c) >= '0') && ((c) <= '7'))
#define TOOCTAL(c) ((c) + '0')

View File

@ -17,6 +17,7 @@
#include "common/md5.h"
#include "common/sha2.h"
#include "utils/builtins.h"
#include "varatt.h"
/*

View File

@ -18,6 +18,7 @@
#include "mb/pg_wchar.h"
#include "utils/builtins.h"
#include "utils/memutils.h"
#include "varatt.h"
/*

View File

@ -88,6 +88,7 @@
#include "utils/memutils.h"
#include "utils/numeric.h"
#include "utils/pg_locale.h"
#include "varatt.h"
/* ----------

View File

@ -35,6 +35,7 @@
#include "utils/float.h"
#include "utils/fmgrprotos.h"
#include "utils/geo_decls.h"
#include "varatt.h"
/*
* * Type constructors have this form:

View File

@ -24,6 +24,7 @@
#include "miscadmin.h"
#include "utils/builtins.h"
#include "utils/pg_locale.h"
#include "varatt.h"
#define LIKE_TRUE 1

View File

@ -51,6 +51,7 @@
#include "access/stratnum.h"
#include "utils/builtins.h"
#include "utils/inet.h"
#include "varatt.h"
/*
* Operator strategy numbers used in the GiST inet_ops opclass

View File

@ -37,6 +37,7 @@
#include "catalog/pg_type.h"
#include "utils/builtins.h"
#include "utils/inet.h"
#include "varatt.h"
static int inet_spg_node_number(const inet *val, int commonbits);

View File

@ -21,6 +21,7 @@
#include "utils/builtins.h"
#include "utils/formatting.h"
#include "utils/memutils.h"
#include "varatt.h"
static text *dotrim(const char *string, int stringlen,

View File

@ -14,6 +14,7 @@
#include "postgres.h"
#include "utils/builtins.h"
#include "varatt.h"
/*

View File

@ -42,6 +42,7 @@
#include "utils/lsyscache.h"
#include "utils/rangetypes.h"
#include "utils/timestamp.h"
#include "varatt.h"
/* fn_extra cache entry for one of the range I/O functions */

View File

@ -31,6 +31,7 @@
#include "utils/lsyscache.h"
#include "utils/rangetypes.h"
#include "utils/multirangetypes.h"
#include "varatt.h"
static int float8_qsort_cmp(const void *a1, const void *a2, void *arg);
static int range_bound_qsort_cmp(const void *a1, const void *a2, void *arg);

View File

@ -19,6 +19,7 @@
#include "tsearch/ts_type.h"
#include "tsearch/ts_utils.h"
#include "utils/builtins.h"
#include "varatt.h"
Datum

View File

@ -23,6 +23,7 @@
#include "utils/builtins.h"
#include "utils/memutils.h"
#include "utils/pg_crc.h"
#include "varatt.h"
/* FTS operator priorities, see ts_type.h */
const int tsearch_op_priority[OP_COUNT] =

View File

@ -17,6 +17,7 @@
#include "miscadmin.h"
#include "tsearch/ts_utils.h"
#include "varatt.h"
typedef struct NODE
{

View File

@ -17,6 +17,7 @@
#include "lib/qunique.h"
#include "tsearch/ts_utils.h"
#include "utils/builtins.h"
#include "varatt.h"
Datum
tsquery_numnode(PG_FUNCTION_ARGS)

View File

@ -16,6 +16,7 @@
#include "miscadmin.h"
#include "tsearch/ts_utils.h"
#include "varatt.h"
/*
* Build QTNode tree for a tsquery given in QueryItem array format.

View File

@ -20,6 +20,7 @@
#include "tsearch/ts_utils.h"
#include "utils/builtins.h"
#include "utils/memutils.h"
#include "varatt.h"
typedef struct
{

View File

@ -22,6 +22,7 @@
#include "utils/hsearch.h"
#include "utils/inval.h"
#include "utils/syscache.h"
#include "varatt.h"
/* Hash table for information about each attribute's options */

View File

@ -29,6 +29,7 @@
#include "utils/inval.h"
#include "utils/spccache.h"
#include "utils/syscache.h"
#include "varatt.h"
/* Hash table for information about each tablespace */

View File

@ -40,6 +40,7 @@
#include "utils/builtins.h"
#include "utils/memutils.h"
#include "utils/syscache.h"
#include "varatt.h"
/*
* We maintain a simple linked list caching the fmgr lookup info for the

View File

@ -19,6 +19,7 @@
#include "access/tupdesc.h"
#include "access/tupmacs.h"
#include "storage/bufpage.h"
#include "varatt.h"
/*
* MaxTupleAttributeNumber limits the number of (user) columns in a tuple.

View File

@ -113,6 +113,7 @@ install_headers(
'postgres.h',
'postgres_ext.h',
'postgres_fe.h',
'varatt.h',
'windowapi.h',
pg_config_ext,
pg_config_os,

View File

@ -23,9 +23,8 @@
*
* section description
* ------- ------------------------------------------------
* 1) variable-length datatypes (TOAST support)
* 2) Datum type + support functions
* 3) miscellaneous
* 1) Datum type + support functions
* 2) miscellaneous
*
* NOTES
*
@ -36,8 +35,7 @@
* postgres_fe.h. We do that since those type definitions are needed by
* frontend modules that want to deal with binary data transmission to or
* from the backend. Type definitions in this file should be for
* representations that never escape the backend, such as Datum or
* TOASTed varlena objects.
* representations that never escape the backend, such as Datum.
*
*----------------------------------------------------------------
*/
@ -49,353 +47,7 @@
#include "utils/palloc.h"
/* ----------------------------------------------------------------
* Section 1: variable-length datatypes (TOAST support)
* ----------------------------------------------------------------
*/
/*
* struct varatt_external is a traditional "TOAST pointer", that is, the
* information needed to fetch a Datum stored out-of-line in a TOAST table.
* The data is compressed if and only if the external size stored in
* va_extinfo is less than va_rawsize - VARHDRSZ.
*
* This struct must not contain any padding, because we sometimes compare
* these pointers using memcmp.
*
* Note that this information is stored unaligned within actual tuples, so
* you need to memcpy from the tuple into a local struct variable before
* you can look at these fields! (The reason we use memcmp is to avoid
* having to do that just to detect equality of two TOAST pointers...)
*/
typedef struct varatt_external
{
int32 va_rawsize; /* Original data size (includes header) */
uint32 va_extinfo; /* External saved size (without header) and
* compression method */
Oid va_valueid; /* Unique ID of value within TOAST table */
Oid va_toastrelid; /* RelID of TOAST table containing it */
} varatt_external;
/*
* These macros define the "saved size" portion of va_extinfo. Its remaining
* two high-order bits identify the compression method.
*/
#define VARLENA_EXTSIZE_BITS 30
#define VARLENA_EXTSIZE_MASK ((1U << VARLENA_EXTSIZE_BITS) - 1)
/*
* struct varatt_indirect is a "TOAST pointer" representing an out-of-line
* Datum that's stored in memory, not in an external toast relation.
* The creator of such a Datum is entirely responsible that the referenced
* storage survives for as long as referencing pointer Datums can exist.
*
* Note that just as for struct varatt_external, this struct is stored
* unaligned within any containing tuple.
*/
typedef struct varatt_indirect
{
struct varlena *pointer; /* Pointer to in-memory varlena */
} varatt_indirect;
/*
* struct varatt_expanded is a "TOAST pointer" representing an out-of-line
* Datum that is stored in memory, in some type-specific, not necessarily
* physically contiguous format that is convenient for computation not
* storage. APIs for this, in particular the definition of struct
* ExpandedObjectHeader, are in src/include/utils/expandeddatum.h.
*
* Note that just as for struct varatt_external, this struct is stored
* unaligned within any containing tuple.
*/
typedef struct ExpandedObjectHeader ExpandedObjectHeader;
typedef struct varatt_expanded
{
ExpandedObjectHeader *eohptr;
} varatt_expanded;
/*
* Type tag for the various sorts of "TOAST pointer" datums. The peculiar
* value for VARTAG_ONDISK comes from a requirement for on-disk compatibility
* with a previous notion that the tag field was the pointer datum's length.
*/
typedef enum vartag_external
{
VARTAG_INDIRECT = 1,
VARTAG_EXPANDED_RO = 2,
VARTAG_EXPANDED_RW = 3,
VARTAG_ONDISK = 18
} vartag_external;
/* this test relies on the specific tag values above */
#define VARTAG_IS_EXPANDED(tag) \
(((tag) & ~1) == VARTAG_EXPANDED_RO)
#define VARTAG_SIZE(tag) \
((tag) == VARTAG_INDIRECT ? sizeof(varatt_indirect) : \
VARTAG_IS_EXPANDED(tag) ? sizeof(varatt_expanded) : \
(tag) == VARTAG_ONDISK ? sizeof(varatt_external) : \
(AssertMacro(false), 0))
/*
* These structs describe the header of a varlena object that may have been
* TOASTed. Generally, don't reference these structs directly, but use the
* macros below.
*
* We use separate structs for the aligned and unaligned cases because the
* compiler might otherwise think it could generate code that assumes
* alignment while touching fields of a 1-byte-header varlena.
*/
typedef union
{
struct /* Normal varlena (4-byte length) */
{
uint32 va_header;
char va_data[FLEXIBLE_ARRAY_MEMBER];
} va_4byte;
struct /* Compressed-in-line format */
{
uint32 va_header;
uint32 va_tcinfo; /* Original data size (excludes header) and
* compression method; see va_extinfo */
char va_data[FLEXIBLE_ARRAY_MEMBER]; /* Compressed data */
} va_compressed;
} varattrib_4b;
typedef struct
{
uint8 va_header;
char va_data[FLEXIBLE_ARRAY_MEMBER]; /* Data begins here */
} varattrib_1b;
/* TOAST pointers are a subset of varattrib_1b with an identifying tag byte */
typedef struct
{
uint8 va_header; /* Always 0x80 or 0x01 */
uint8 va_tag; /* Type of datum */
char va_data[FLEXIBLE_ARRAY_MEMBER]; /* Type-specific data */
} varattrib_1b_e;
/*
* Bit layouts for varlena headers on big-endian machines:
*
* 00xxxxxx 4-byte length word, aligned, uncompressed data (up to 1G)
* 01xxxxxx 4-byte length word, aligned, *compressed* data (up to 1G)
* 10000000 1-byte length word, unaligned, TOAST pointer
* 1xxxxxxx 1-byte length word, unaligned, uncompressed data (up to 126b)
*
* Bit layouts for varlena headers on little-endian machines:
*
* xxxxxx00 4-byte length word, aligned, uncompressed data (up to 1G)
* xxxxxx10 4-byte length word, aligned, *compressed* data (up to 1G)
* 00000001 1-byte length word, unaligned, TOAST pointer
* xxxxxxx1 1-byte length word, unaligned, uncompressed data (up to 126b)
*
* The "xxx" bits are the length field (which includes itself in all cases).
* In the big-endian case we mask to extract the length, in the little-endian
* case we shift. Note that in both cases the flag bits are in the physically
* first byte. Also, it is not possible for a 1-byte length word to be zero;
* this lets us disambiguate alignment padding bytes from the start of an
* unaligned datum. (We now *require* pad bytes to be filled with zero!)
*
* In TOAST pointers the va_tag field (see varattrib_1b_e) is used to discern
* the specific type and length of the pointer datum.
*/
/*
* Endian-dependent macros. These are considered internal --- use the
* external macros below instead of using these directly.
*
* Note: IS_1B is true for external toast records but VARSIZE_1B will return 0
* for such records. Hence you should usually check for IS_EXTERNAL before
* checking for IS_1B.
*/
#ifdef WORDS_BIGENDIAN
#define VARATT_IS_4B(PTR) \
((((varattrib_1b *) (PTR))->va_header & 0x80) == 0x00)
#define VARATT_IS_4B_U(PTR) \
((((varattrib_1b *) (PTR))->va_header & 0xC0) == 0x00)
#define VARATT_IS_4B_C(PTR) \
((((varattrib_1b *) (PTR))->va_header & 0xC0) == 0x40)
#define VARATT_IS_1B(PTR) \
((((varattrib_1b *) (PTR))->va_header & 0x80) == 0x80)
#define VARATT_IS_1B_E(PTR) \
((((varattrib_1b *) (PTR))->va_header) == 0x80)
#define VARATT_NOT_PAD_BYTE(PTR) \
(*((uint8 *) (PTR)) != 0)
/* VARSIZE_4B() should only be used on known-aligned data */
#define VARSIZE_4B(PTR) \
(((varattrib_4b *) (PTR))->va_4byte.va_header & 0x3FFFFFFF)
#define VARSIZE_1B(PTR) \
(((varattrib_1b *) (PTR))->va_header & 0x7F)
#define VARTAG_1B_E(PTR) \
(((varattrib_1b_e *) (PTR))->va_tag)
#define SET_VARSIZE_4B(PTR,len) \
(((varattrib_4b *) (PTR))->va_4byte.va_header = (len) & 0x3FFFFFFF)
#define SET_VARSIZE_4B_C(PTR,len) \
(((varattrib_4b *) (PTR))->va_4byte.va_header = ((len) & 0x3FFFFFFF) | 0x40000000)
#define SET_VARSIZE_1B(PTR,len) \
(((varattrib_1b *) (PTR))->va_header = (len) | 0x80)
#define SET_VARTAG_1B_E(PTR,tag) \
(((varattrib_1b_e *) (PTR))->va_header = 0x80, \
((varattrib_1b_e *) (PTR))->va_tag = (tag))
#else /* !WORDS_BIGENDIAN */
#define VARATT_IS_4B(PTR) \
((((varattrib_1b *) (PTR))->va_header & 0x01) == 0x00)
#define VARATT_IS_4B_U(PTR) \
((((varattrib_1b *) (PTR))->va_header & 0x03) == 0x00)
#define VARATT_IS_4B_C(PTR) \
((((varattrib_1b *) (PTR))->va_header & 0x03) == 0x02)
#define VARATT_IS_1B(PTR) \
((((varattrib_1b *) (PTR))->va_header & 0x01) == 0x01)
#define VARATT_IS_1B_E(PTR) \
((((varattrib_1b *) (PTR))->va_header) == 0x01)
#define VARATT_NOT_PAD_BYTE(PTR) \
(*((uint8 *) (PTR)) != 0)
/* VARSIZE_4B() should only be used on known-aligned data */
#define VARSIZE_4B(PTR) \
((((varattrib_4b *) (PTR))->va_4byte.va_header >> 2) & 0x3FFFFFFF)
#define VARSIZE_1B(PTR) \
((((varattrib_1b *) (PTR))->va_header >> 1) & 0x7F)
#define VARTAG_1B_E(PTR) \
(((varattrib_1b_e *) (PTR))->va_tag)
#define SET_VARSIZE_4B(PTR,len) \
(((varattrib_4b *) (PTR))->va_4byte.va_header = (((uint32) (len)) << 2))
#define SET_VARSIZE_4B_C(PTR,len) \
(((varattrib_4b *) (PTR))->va_4byte.va_header = (((uint32) (len)) << 2) | 0x02)
#define SET_VARSIZE_1B(PTR,len) \
(((varattrib_1b *) (PTR))->va_header = (((uint8) (len)) << 1) | 0x01)
#define SET_VARTAG_1B_E(PTR,tag) \
(((varattrib_1b_e *) (PTR))->va_header = 0x01, \
((varattrib_1b_e *) (PTR))->va_tag = (tag))
#endif /* WORDS_BIGENDIAN */
#define VARDATA_4B(PTR) (((varattrib_4b *) (PTR))->va_4byte.va_data)
#define VARDATA_4B_C(PTR) (((varattrib_4b *) (PTR))->va_compressed.va_data)
#define VARDATA_1B(PTR) (((varattrib_1b *) (PTR))->va_data)
#define VARDATA_1B_E(PTR) (((varattrib_1b_e *) (PTR))->va_data)
/*
* Externally visible TOAST macros begin here.
*/
#define VARHDRSZ_EXTERNAL offsetof(varattrib_1b_e, va_data)
#define VARHDRSZ_COMPRESSED offsetof(varattrib_4b, va_compressed.va_data)
#define VARHDRSZ_SHORT offsetof(varattrib_1b, va_data)
#define VARATT_SHORT_MAX 0x7F
#define VARATT_CAN_MAKE_SHORT(PTR) \
(VARATT_IS_4B_U(PTR) && \
(VARSIZE(PTR) - VARHDRSZ + VARHDRSZ_SHORT) <= VARATT_SHORT_MAX)
#define VARATT_CONVERTED_SHORT_SIZE(PTR) \
(VARSIZE(PTR) - VARHDRSZ + VARHDRSZ_SHORT)
/*
* In consumers oblivious to data alignment, call PG_DETOAST_DATUM_PACKED(),
* VARDATA_ANY(), VARSIZE_ANY() and VARSIZE_ANY_EXHDR(). Elsewhere, call
* PG_DETOAST_DATUM(), VARDATA() and VARSIZE(). Directly fetching an int16,
* int32 or wider field in the struct representing the datum layout requires
* aligned data. memcpy() is alignment-oblivious, as are most operations on
* datatypes, such as text, whose layout struct contains only char fields.
*
* Code assembling a new datum should call VARDATA() and SET_VARSIZE().
* (Datums begin life untoasted.)
*
* Other macros here should usually be used only by tuple assembly/disassembly
* code and code that specifically wants to work with still-toasted Datums.
*/
#define VARDATA(PTR) VARDATA_4B(PTR)
#define VARSIZE(PTR) VARSIZE_4B(PTR)
#define VARSIZE_SHORT(PTR) VARSIZE_1B(PTR)
#define VARDATA_SHORT(PTR) VARDATA_1B(PTR)
#define VARTAG_EXTERNAL(PTR) VARTAG_1B_E(PTR)
#define VARSIZE_EXTERNAL(PTR) (VARHDRSZ_EXTERNAL + VARTAG_SIZE(VARTAG_EXTERNAL(PTR)))
#define VARDATA_EXTERNAL(PTR) VARDATA_1B_E(PTR)
#define VARATT_IS_COMPRESSED(PTR) VARATT_IS_4B_C(PTR)
#define VARATT_IS_EXTERNAL(PTR) VARATT_IS_1B_E(PTR)
#define VARATT_IS_EXTERNAL_ONDISK(PTR) \
(VARATT_IS_EXTERNAL(PTR) && VARTAG_EXTERNAL(PTR) == VARTAG_ONDISK)
#define VARATT_IS_EXTERNAL_INDIRECT(PTR) \
(VARATT_IS_EXTERNAL(PTR) && VARTAG_EXTERNAL(PTR) == VARTAG_INDIRECT)
#define VARATT_IS_EXTERNAL_EXPANDED_RO(PTR) \
(VARATT_IS_EXTERNAL(PTR) && VARTAG_EXTERNAL(PTR) == VARTAG_EXPANDED_RO)
#define VARATT_IS_EXTERNAL_EXPANDED_RW(PTR) \
(VARATT_IS_EXTERNAL(PTR) && VARTAG_EXTERNAL(PTR) == VARTAG_EXPANDED_RW)
#define VARATT_IS_EXTERNAL_EXPANDED(PTR) \
(VARATT_IS_EXTERNAL(PTR) && VARTAG_IS_EXPANDED(VARTAG_EXTERNAL(PTR)))
#define VARATT_IS_EXTERNAL_NON_EXPANDED(PTR) \
(VARATT_IS_EXTERNAL(PTR) && !VARTAG_IS_EXPANDED(VARTAG_EXTERNAL(PTR)))
#define VARATT_IS_SHORT(PTR) VARATT_IS_1B(PTR)
#define VARATT_IS_EXTENDED(PTR) (!VARATT_IS_4B_U(PTR))
#define SET_VARSIZE(PTR, len) SET_VARSIZE_4B(PTR, len)
#define SET_VARSIZE_SHORT(PTR, len) SET_VARSIZE_1B(PTR, len)
#define SET_VARSIZE_COMPRESSED(PTR, len) SET_VARSIZE_4B_C(PTR, len)
#define SET_VARTAG_EXTERNAL(PTR, tag) SET_VARTAG_1B_E(PTR, tag)
#define VARSIZE_ANY(PTR) \
(VARATT_IS_1B_E(PTR) ? VARSIZE_EXTERNAL(PTR) : \
(VARATT_IS_1B(PTR) ? VARSIZE_1B(PTR) : \
VARSIZE_4B(PTR)))
/* Size of a varlena data, excluding header */
#define VARSIZE_ANY_EXHDR(PTR) \
(VARATT_IS_1B_E(PTR) ? VARSIZE_EXTERNAL(PTR)-VARHDRSZ_EXTERNAL : \
(VARATT_IS_1B(PTR) ? VARSIZE_1B(PTR)-VARHDRSZ_SHORT : \
VARSIZE_4B(PTR)-VARHDRSZ))
/* caution: this will not work on an external or compressed-in-line Datum */
/* caution: this will return a possibly unaligned pointer */
#define VARDATA_ANY(PTR) \
(VARATT_IS_1B(PTR) ? VARDATA_1B(PTR) : VARDATA_4B(PTR))
/* Decompressed size and compression method of a compressed-in-line Datum */
#define VARDATA_COMPRESSED_GET_EXTSIZE(PTR) \
(((varattrib_4b *) (PTR))->va_compressed.va_tcinfo & VARLENA_EXTSIZE_MASK)
#define VARDATA_COMPRESSED_GET_COMPRESS_METHOD(PTR) \
(((varattrib_4b *) (PTR))->va_compressed.va_tcinfo >> VARLENA_EXTSIZE_BITS)
/* Same for external Datums; but note argument is a struct varatt_external */
#define VARATT_EXTERNAL_GET_EXTSIZE(toast_pointer) \
((toast_pointer).va_extinfo & VARLENA_EXTSIZE_MASK)
#define VARATT_EXTERNAL_GET_COMPRESS_METHOD(toast_pointer) \
((toast_pointer).va_extinfo >> VARLENA_EXTSIZE_BITS)
#define VARATT_EXTERNAL_SET_SIZE_AND_COMPRESS_METHOD(toast_pointer, len, cm) \
do { \
Assert((cm) == TOAST_PGLZ_COMPRESSION_ID || \
(cm) == TOAST_LZ4_COMPRESSION_ID); \
((toast_pointer).va_extinfo = \
(len) | ((uint32) (cm) << VARLENA_EXTSIZE_BITS)); \
} while (0)
/*
* Testing whether an externally-stored value is compressed now requires
* comparing size stored in va_extinfo (the actual length of the external data)
* to rawsize (the original uncompressed datum's size). The latter includes
* VARHDRSZ overhead, the former doesn't. We never use compression unless it
* actually saves space, so we expect either equality or less-than.
*/
#define VARATT_EXTERNAL_IS_COMPRESSED(toast_pointer) \
(VARATT_EXTERNAL_GET_EXTSIZE(toast_pointer) < \
(toast_pointer).va_rawsize - VARHDRSZ)
/* ----------------------------------------------------------------
* Section 2: Datum type + support functions
* Section 1: Datum type + support functions
* ----------------------------------------------------------------
*/
@ -907,7 +559,7 @@ extern Datum Float8GetDatum(float8 X);
/* ----------------------------------------------------------------
* Section 3: miscellaneous
* Section 2: miscellaneous
* ----------------------------------------------------------------
*/

View File

@ -44,6 +44,8 @@
#ifndef EXPANDEDDATUM_H
#define EXPANDEDDATUM_H
#include "varatt.h"
/* Size of an EXTERNAL datum that contains a pointer to an expanded object */
#define EXPANDED_POINTER_SIZE (VARHDRSZ_EXTERNAL + sizeof(varatt_expanded))

358
src/include/varatt.h Normal file
View File

@ -0,0 +1,358 @@
/*-------------------------------------------------------------------------
*
* varatt.h
* variable-length datatypes (TOAST support)
*
*
* Portions Copyright (c) 1996-2023, PostgreSQL Global Development Group
* Portions Copyright (c) 1995, Regents of the University of California
*
* src/include/varatt.h
*
*-------------------------------------------------------------------------
*/
#ifndef VARATT_H
#define VARATT_H
/*
* struct varatt_external is a traditional "TOAST pointer", that is, the
* information needed to fetch a Datum stored out-of-line in a TOAST table.
* The data is compressed if and only if the external size stored in
* va_extinfo is less than va_rawsize - VARHDRSZ.
*
* This struct must not contain any padding, because we sometimes compare
* these pointers using memcmp.
*
* Note that this information is stored unaligned within actual tuples, so
* you need to memcpy from the tuple into a local struct variable before
* you can look at these fields! (The reason we use memcmp is to avoid
* having to do that just to detect equality of two TOAST pointers...)
*/
typedef struct varatt_external
{
int32 va_rawsize; /* Original data size (includes header) */
uint32 va_extinfo; /* External saved size (without header) and
* compression method */
Oid va_valueid; /* Unique ID of value within TOAST table */
Oid va_toastrelid; /* RelID of TOAST table containing it */
} varatt_external;
/*
* These macros define the "saved size" portion of va_extinfo. Its remaining
* two high-order bits identify the compression method.
*/
#define VARLENA_EXTSIZE_BITS 30
#define VARLENA_EXTSIZE_MASK ((1U << VARLENA_EXTSIZE_BITS) - 1)
/*
* struct varatt_indirect is a "TOAST pointer" representing an out-of-line
* Datum that's stored in memory, not in an external toast relation.
* The creator of such a Datum is entirely responsible that the referenced
* storage survives for as long as referencing pointer Datums can exist.
*
* Note that just as for struct varatt_external, this struct is stored
* unaligned within any containing tuple.
*/
typedef struct varatt_indirect
{
struct varlena *pointer; /* Pointer to in-memory varlena */
} varatt_indirect;
/*
* struct varatt_expanded is a "TOAST pointer" representing an out-of-line
* Datum that is stored in memory, in some type-specific, not necessarily
* physically contiguous format that is convenient for computation not
* storage. APIs for this, in particular the definition of struct
* ExpandedObjectHeader, are in src/include/utils/expandeddatum.h.
*
* Note that just as for struct varatt_external, this struct is stored
* unaligned within any containing tuple.
*/
typedef struct ExpandedObjectHeader ExpandedObjectHeader;
typedef struct varatt_expanded
{
ExpandedObjectHeader *eohptr;
} varatt_expanded;
/*
* Type tag for the various sorts of "TOAST pointer" datums. The peculiar
* value for VARTAG_ONDISK comes from a requirement for on-disk compatibility
* with a previous notion that the tag field was the pointer datum's length.
*/
typedef enum vartag_external
{
VARTAG_INDIRECT = 1,
VARTAG_EXPANDED_RO = 2,
VARTAG_EXPANDED_RW = 3,
VARTAG_ONDISK = 18
} vartag_external;
/* this test relies on the specific tag values above */
#define VARTAG_IS_EXPANDED(tag) \
(((tag) & ~1) == VARTAG_EXPANDED_RO)
#define VARTAG_SIZE(tag) \
((tag) == VARTAG_INDIRECT ? sizeof(varatt_indirect) : \
VARTAG_IS_EXPANDED(tag) ? sizeof(varatt_expanded) : \
(tag) == VARTAG_ONDISK ? sizeof(varatt_external) : \
(AssertMacro(false), 0))
/*
* These structs describe the header of a varlena object that may have been
* TOASTed. Generally, don't reference these structs directly, but use the
* macros below.
*
* We use separate structs for the aligned and unaligned cases because the
* compiler might otherwise think it could generate code that assumes
* alignment while touching fields of a 1-byte-header varlena.
*/
typedef union
{
struct /* Normal varlena (4-byte length) */
{
uint32 va_header;
char va_data[FLEXIBLE_ARRAY_MEMBER];
} va_4byte;
struct /* Compressed-in-line format */
{
uint32 va_header;
uint32 va_tcinfo; /* Original data size (excludes header) and
* compression method; see va_extinfo */
char va_data[FLEXIBLE_ARRAY_MEMBER]; /* Compressed data */
} va_compressed;
} varattrib_4b;
typedef struct
{
uint8 va_header;
char va_data[FLEXIBLE_ARRAY_MEMBER]; /* Data begins here */
} varattrib_1b;
/* TOAST pointers are a subset of varattrib_1b with an identifying tag byte */
typedef struct
{
uint8 va_header; /* Always 0x80 or 0x01 */
uint8 va_tag; /* Type of datum */
char va_data[FLEXIBLE_ARRAY_MEMBER]; /* Type-specific data */
} varattrib_1b_e;
/*
* Bit layouts for varlena headers on big-endian machines:
*
* 00xxxxxx 4-byte length word, aligned, uncompressed data (up to 1G)
* 01xxxxxx 4-byte length word, aligned, *compressed* data (up to 1G)
* 10000000 1-byte length word, unaligned, TOAST pointer
* 1xxxxxxx 1-byte length word, unaligned, uncompressed data (up to 126b)
*
* Bit layouts for varlena headers on little-endian machines:
*
* xxxxxx00 4-byte length word, aligned, uncompressed data (up to 1G)
* xxxxxx10 4-byte length word, aligned, *compressed* data (up to 1G)
* 00000001 1-byte length word, unaligned, TOAST pointer
* xxxxxxx1 1-byte length word, unaligned, uncompressed data (up to 126b)
*
* The "xxx" bits are the length field (which includes itself in all cases).
* In the big-endian case we mask to extract the length, in the little-endian
* case we shift. Note that in both cases the flag bits are in the physically
* first byte. Also, it is not possible for a 1-byte length word to be zero;
* this lets us disambiguate alignment padding bytes from the start of an
* unaligned datum. (We now *require* pad bytes to be filled with zero!)
*
* In TOAST pointers the va_tag field (see varattrib_1b_e) is used to discern
* the specific type and length of the pointer datum.
*/
/*
* Endian-dependent macros. These are considered internal --- use the
* external macros below instead of using these directly.
*
* Note: IS_1B is true for external toast records but VARSIZE_1B will return 0
* for such records. Hence you should usually check for IS_EXTERNAL before
* checking for IS_1B.
*/
#ifdef WORDS_BIGENDIAN
#define VARATT_IS_4B(PTR) \
((((varattrib_1b *) (PTR))->va_header & 0x80) == 0x00)
#define VARATT_IS_4B_U(PTR) \
((((varattrib_1b *) (PTR))->va_header & 0xC0) == 0x00)
#define VARATT_IS_4B_C(PTR) \
((((varattrib_1b *) (PTR))->va_header & 0xC0) == 0x40)
#define VARATT_IS_1B(PTR) \
((((varattrib_1b *) (PTR))->va_header & 0x80) == 0x80)
#define VARATT_IS_1B_E(PTR) \
((((varattrib_1b *) (PTR))->va_header) == 0x80)
#define VARATT_NOT_PAD_BYTE(PTR) \
(*((uint8 *) (PTR)) != 0)
/* VARSIZE_4B() should only be used on known-aligned data */
#define VARSIZE_4B(PTR) \
(((varattrib_4b *) (PTR))->va_4byte.va_header & 0x3FFFFFFF)
#define VARSIZE_1B(PTR) \
(((varattrib_1b *) (PTR))->va_header & 0x7F)
#define VARTAG_1B_E(PTR) \
(((varattrib_1b_e *) (PTR))->va_tag)
#define SET_VARSIZE_4B(PTR,len) \
(((varattrib_4b *) (PTR))->va_4byte.va_header = (len) & 0x3FFFFFFF)
#define SET_VARSIZE_4B_C(PTR,len) \
(((varattrib_4b *) (PTR))->va_4byte.va_header = ((len) & 0x3FFFFFFF) | 0x40000000)
#define SET_VARSIZE_1B(PTR,len) \
(((varattrib_1b *) (PTR))->va_header = (len) | 0x80)
#define SET_VARTAG_1B_E(PTR,tag) \
(((varattrib_1b_e *) (PTR))->va_header = 0x80, \
((varattrib_1b_e *) (PTR))->va_tag = (tag))
#else /* !WORDS_BIGENDIAN */
#define VARATT_IS_4B(PTR) \
((((varattrib_1b *) (PTR))->va_header & 0x01) == 0x00)
#define VARATT_IS_4B_U(PTR) \
((((varattrib_1b *) (PTR))->va_header & 0x03) == 0x00)
#define VARATT_IS_4B_C(PTR) \
((((varattrib_1b *) (PTR))->va_header & 0x03) == 0x02)
#define VARATT_IS_1B(PTR) \
((((varattrib_1b *) (PTR))->va_header & 0x01) == 0x01)
#define VARATT_IS_1B_E(PTR) \
((((varattrib_1b *) (PTR))->va_header) == 0x01)
#define VARATT_NOT_PAD_BYTE(PTR) \
(*((uint8 *) (PTR)) != 0)
/* VARSIZE_4B() should only be used on known-aligned data */
#define VARSIZE_4B(PTR) \
((((varattrib_4b *) (PTR))->va_4byte.va_header >> 2) & 0x3FFFFFFF)
#define VARSIZE_1B(PTR) \
((((varattrib_1b *) (PTR))->va_header >> 1) & 0x7F)
#define VARTAG_1B_E(PTR) \
(((varattrib_1b_e *) (PTR))->va_tag)
#define SET_VARSIZE_4B(PTR,len) \
(((varattrib_4b *) (PTR))->va_4byte.va_header = (((uint32) (len)) << 2))
#define SET_VARSIZE_4B_C(PTR,len) \
(((varattrib_4b *) (PTR))->va_4byte.va_header = (((uint32) (len)) << 2) | 0x02)
#define SET_VARSIZE_1B(PTR,len) \
(((varattrib_1b *) (PTR))->va_header = (((uint8) (len)) << 1) | 0x01)
#define SET_VARTAG_1B_E(PTR,tag) \
(((varattrib_1b_e *) (PTR))->va_header = 0x01, \
((varattrib_1b_e *) (PTR))->va_tag = (tag))
#endif /* WORDS_BIGENDIAN */
#define VARDATA_4B(PTR) (((varattrib_4b *) (PTR))->va_4byte.va_data)
#define VARDATA_4B_C(PTR) (((varattrib_4b *) (PTR))->va_compressed.va_data)
#define VARDATA_1B(PTR) (((varattrib_1b *) (PTR))->va_data)
#define VARDATA_1B_E(PTR) (((varattrib_1b_e *) (PTR))->va_data)
/*
* Externally visible TOAST macros begin here.
*/
#define VARHDRSZ_EXTERNAL offsetof(varattrib_1b_e, va_data)
#define VARHDRSZ_COMPRESSED offsetof(varattrib_4b, va_compressed.va_data)
#define VARHDRSZ_SHORT offsetof(varattrib_1b, va_data)
#define VARATT_SHORT_MAX 0x7F
#define VARATT_CAN_MAKE_SHORT(PTR) \
(VARATT_IS_4B_U(PTR) && \
(VARSIZE(PTR) - VARHDRSZ + VARHDRSZ_SHORT) <= VARATT_SHORT_MAX)
#define VARATT_CONVERTED_SHORT_SIZE(PTR) \
(VARSIZE(PTR) - VARHDRSZ + VARHDRSZ_SHORT)
/*
* In consumers oblivious to data alignment, call PG_DETOAST_DATUM_PACKED(),
* VARDATA_ANY(), VARSIZE_ANY() and VARSIZE_ANY_EXHDR(). Elsewhere, call
* PG_DETOAST_DATUM(), VARDATA() and VARSIZE(). Directly fetching an int16,
* int32 or wider field in the struct representing the datum layout requires
* aligned data. memcpy() is alignment-oblivious, as are most operations on
* datatypes, such as text, whose layout struct contains only char fields.
*
* Code assembling a new datum should call VARDATA() and SET_VARSIZE().
* (Datums begin life untoasted.)
*
* Other macros here should usually be used only by tuple assembly/disassembly
* code and code that specifically wants to work with still-toasted Datums.
*/
#define VARDATA(PTR) VARDATA_4B(PTR)
#define VARSIZE(PTR) VARSIZE_4B(PTR)
#define VARSIZE_SHORT(PTR) VARSIZE_1B(PTR)
#define VARDATA_SHORT(PTR) VARDATA_1B(PTR)
#define VARTAG_EXTERNAL(PTR) VARTAG_1B_E(PTR)
#define VARSIZE_EXTERNAL(PTR) (VARHDRSZ_EXTERNAL + VARTAG_SIZE(VARTAG_EXTERNAL(PTR)))
#define VARDATA_EXTERNAL(PTR) VARDATA_1B_E(PTR)
#define VARATT_IS_COMPRESSED(PTR) VARATT_IS_4B_C(PTR)
#define VARATT_IS_EXTERNAL(PTR) VARATT_IS_1B_E(PTR)
#define VARATT_IS_EXTERNAL_ONDISK(PTR) \
(VARATT_IS_EXTERNAL(PTR) && VARTAG_EXTERNAL(PTR) == VARTAG_ONDISK)
#define VARATT_IS_EXTERNAL_INDIRECT(PTR) \
(VARATT_IS_EXTERNAL(PTR) && VARTAG_EXTERNAL(PTR) == VARTAG_INDIRECT)
#define VARATT_IS_EXTERNAL_EXPANDED_RO(PTR) \
(VARATT_IS_EXTERNAL(PTR) && VARTAG_EXTERNAL(PTR) == VARTAG_EXPANDED_RO)
#define VARATT_IS_EXTERNAL_EXPANDED_RW(PTR) \
(VARATT_IS_EXTERNAL(PTR) && VARTAG_EXTERNAL(PTR) == VARTAG_EXPANDED_RW)
#define VARATT_IS_EXTERNAL_EXPANDED(PTR) \
(VARATT_IS_EXTERNAL(PTR) && VARTAG_IS_EXPANDED(VARTAG_EXTERNAL(PTR)))
#define VARATT_IS_EXTERNAL_NON_EXPANDED(PTR) \
(VARATT_IS_EXTERNAL(PTR) && !VARTAG_IS_EXPANDED(VARTAG_EXTERNAL(PTR)))
#define VARATT_IS_SHORT(PTR) VARATT_IS_1B(PTR)
#define VARATT_IS_EXTENDED(PTR) (!VARATT_IS_4B_U(PTR))
#define SET_VARSIZE(PTR, len) SET_VARSIZE_4B(PTR, len)
#define SET_VARSIZE_SHORT(PTR, len) SET_VARSIZE_1B(PTR, len)
#define SET_VARSIZE_COMPRESSED(PTR, len) SET_VARSIZE_4B_C(PTR, len)
#define SET_VARTAG_EXTERNAL(PTR, tag) SET_VARTAG_1B_E(PTR, tag)
#define VARSIZE_ANY(PTR) \
(VARATT_IS_1B_E(PTR) ? VARSIZE_EXTERNAL(PTR) : \
(VARATT_IS_1B(PTR) ? VARSIZE_1B(PTR) : \
VARSIZE_4B(PTR)))
/* Size of a varlena data, excluding header */
#define VARSIZE_ANY_EXHDR(PTR) \
(VARATT_IS_1B_E(PTR) ? VARSIZE_EXTERNAL(PTR)-VARHDRSZ_EXTERNAL : \
(VARATT_IS_1B(PTR) ? VARSIZE_1B(PTR)-VARHDRSZ_SHORT : \
VARSIZE_4B(PTR)-VARHDRSZ))
/* caution: this will not work on an external or compressed-in-line Datum */
/* caution: this will return a possibly unaligned pointer */
#define VARDATA_ANY(PTR) \
(VARATT_IS_1B(PTR) ? VARDATA_1B(PTR) : VARDATA_4B(PTR))
/* Decompressed size and compression method of a compressed-in-line Datum */
#define VARDATA_COMPRESSED_GET_EXTSIZE(PTR) \
(((varattrib_4b *) (PTR))->va_compressed.va_tcinfo & VARLENA_EXTSIZE_MASK)
#define VARDATA_COMPRESSED_GET_COMPRESS_METHOD(PTR) \
(((varattrib_4b *) (PTR))->va_compressed.va_tcinfo >> VARLENA_EXTSIZE_BITS)
/* Same for external Datums; but note argument is a struct varatt_external */
#define VARATT_EXTERNAL_GET_EXTSIZE(toast_pointer) \
((toast_pointer).va_extinfo & VARLENA_EXTSIZE_MASK)
#define VARATT_EXTERNAL_GET_COMPRESS_METHOD(toast_pointer) \
((toast_pointer).va_extinfo >> VARLENA_EXTSIZE_BITS)
#define VARATT_EXTERNAL_SET_SIZE_AND_COMPRESS_METHOD(toast_pointer, len, cm) \
do { \
Assert((cm) == TOAST_PGLZ_COMPRESSION_ID || \
(cm) == TOAST_LZ4_COMPRESSION_ID); \
((toast_pointer).va_extinfo = \
(len) | ((uint32) (cm) << VARLENA_EXTSIZE_BITS)); \
} while (0)
/*
* Testing whether an externally-stored value is compressed now requires
* comparing size stored in va_extinfo (the actual length of the external data)
* to rawsize (the original uncompressed datum's size). The latter includes
* VARHDRSZ overhead, the former doesn't. We never use compression unless it
* actually saves space, so we expect either equality or less-than.
*/
#define VARATT_EXTERNAL_IS_COMPRESSED(toast_pointer) \
(VARATT_EXTERNAL_GET_EXTSIZE(toast_pointer) < \
(toast_pointer).va_rawsize - VARHDRSZ)
#endif

View File

@ -16,6 +16,7 @@
#include "fmgr.h"
#include "utils/builtins.h"
#include "utils/bytea.h" /* for byteain & byteaout */
#include "varatt.h"
/* perl stuff */
#define PG_NEED_PERL_XSUB_H

View File

@ -24,6 +24,7 @@
#include "access/spgist.h"
#include "catalog/pg_type.h"
#include "utils/datum.h"
#include "varatt.h"
PG_MODULE_MAGIC;

View File

@ -22,6 +22,7 @@
#include "access/xloginsert.h"
#include "fmgr.h"
#include "utils/pg_lsn.h"
#include "varatt.h"
PG_MODULE_MAGIC;

View File

@ -16,6 +16,7 @@
#include "fmgr.h"
#include "miscadmin.h"
#include "pgstat.h"
#include "varatt.h"
#include "test_shm_mq.h"