database/postgres
1
0
Fork 0
mirror of https://github.com/postgres/postgres.git synced 2025-07-14 08:21:07 +03:00
Code Activity

Switch to CRC-32C in WAL and other places.

Browse Source 
The old algorithm was found to not be the usual CRC-32 algorithm, used by
Ethernet et al. We were using a non-reflected lookup table with code meant
for a reflected lookup table. That's a strange combination that AFAICS does
not correspond to any bit-wise CRC calculation, which makes it difficult to
reason about its properties. Although it has worked well in practice, seems
safer to use a well-known algorithm.

Since we're changing the algorithm anyway, we might as well choose a
different polynomial. The Castagnoli polynomial has better error-correcting
properties than the traditional CRC-32 polynomial, even if we had
implemented it correctly. Another reason for picking that is that some new
CPUs have hardware support for calculating CRC-32C, but not CRC-32, let
alone our strange variant of it. This patch doesn't add any support for such
hardware, but a future patch could now do that.

The old algorithm is kept around for tsquery and pg_trgm, which use the
values in indexes that need to remain compatible so that pg_upgrade works.
While we're at it, share the old lookup table for CRC-32 calculation
between hstore, ltree and core. They all use the same table, so might as
well.
This commit is contained in:
Heikki Linnakangas
2014-11-04 11:35:15 +02:00
parent 404bc51cde
commit 5028f22f6e
20 changed files with 299 additions and 340 deletions
Download Patch File Download Diff File Expand all files Collapse all files

93
src/include/utils/pg_crc.h
View File

@ -7,9 +7,22 @@
* A PAINLESS GUIDE TO CRC ERROR DETECTION ALGORITHMS, available from
* http://www.ross.net/crc/ or several other net sites.
*
* We use a normal (not "reflected", in Williams' terms) CRC, using initial
* all-ones register contents and a final bit inversion.
* We have three slightly different variants of a 32-bit CRC calculation:
* CRC-32C (Castagnoli polynomial), CRC-32 (Ethernet polynomial), and a legacy
* CRC-32 version that uses the lookup table in a funny way. They all consist
* of four macros:
*
* INIT_<variant>(crc)
* Initialize a CRC accumulator
*
* COMP_<variant>(crc, data, len)
* Accumulate some (more) bytes into a CRC
*
* FIN_<variant>(crc)
* Finish a CRC calculation
*
* EQ_<variant>(c1, c2)
* Check for equality of two CRCs.
*
* Portions Copyright (c) 1996-2014, PostgreSQL Global Development Group
* Portions Copyright (c) 1994, Regents of the University of California
@ -28,29 +41,81 @@
typedef uint32 pg_crc32;
/* Initialize a CRC accumulator */
#define INIT_CRC32(crc) ((crc) = 0xFFFFFFFF)
/*
* CRC calculation using the CRC-32C (Castagnoli) polynomial.
*
* We use all-ones as the initial register contents and final bit inversion.
* This is the same algorithm used e.g. in iSCSI. See RFC 3385 for more
* details on the choice of polynomial.
*/
#define INIT_CRC32C(crc) ((crc) = 0xFFFFFFFF)
#define FIN_CRC32C(crc) ((crc) ^= 0xFFFFFFFF)
#define COMP_CRC32C(crc, data, len) \
COMP_CRC32_NORMAL_TABLE(crc, data, len, pg_crc32c_table)
#define EQ_CRC32C(c1, c2) ((c1) == (c2))
/* Finish a CRC calculation */
#define FIN_CRC32(crc) ((crc) ^= 0xFFFFFFFF)
/*
* CRC-32, the same used e.g. in Ethernet.
*
* This is currently only used in ltree and hstore contrib modules. It uses
* the same lookup table as the legacy algorithm below. New code should
* use the Castagnoli version instead.
*/
#define INIT_TRADITIONAL_CRC32(crc) ((crc) = 0xFFFFFFFF)
#define FIN_TRADITIONAL_CRC32(crc) ((crc) ^= 0xFFFFFFFF)
#define COMP_TRADITIONAL_CRC32(crc, data, len) \
COMP_CRC32_NORMAL_TABLE(crc, data, len, pg_crc32_table)
#define EQ_TRADITIONAL_CRC32(c1, c2) ((c1) == (c2))
/* Accumulate some (more) bytes into a CRC */
#define COMP_CRC32(crc, data, len) \
do { \
/*
* The CRC algorithm used for WAL et al in pre-9.5 versions.
*
* This closely resembles the normal CRC-32 algorithm, but is subtly
* different. Using Williams' terms, we use the "normal" table, but with
* "reflected" code. That's bogus, but it was like that for years before
* anyone noticed. It does not correspond to any polynomial in a normal CRC
* algorithm, so it's not clear what the error-detection properties of this
* algorithm actually are.
*
* We still need to carry this around because it is used in a few on-disk
* structures that need to be pg_upgradeable. It should not be used in new
* code.
*/
#define INIT_LEGACY_CRC32(crc) ((crc) = 0xFFFFFFFF)
#define FIN_LEGACY_CRC32(crc) ((crc) ^= 0xFFFFFFFF)
#define COMP_LEGACY_CRC32(crc, data, len) \
COMP_CRC32_REFLECTED_TABLE(crc, data, len, pg_crc32_table)
#define EQ_LEGACY_CRC32(c1, c2) ((c1) == (c2))
/*
* Common code for CRC computation using a lookup table.
*/
#define COMP_CRC32_NORMAL_TABLE(crc, data, len, table) \
do { \
const unsigned char *__data = (const unsigned char *) (data); \
uint32 __len = (len); \
\
while (__len-- > 0) \
{ \
int __tab_index = ((int) ((crc) >> 24) ^ *__data++) & 0xFF; \
(crc) = pg_crc32_table[__tab_index] ^ ((crc) << 8); \
int __tab_index = ((int) (crc) ^ *__data++) & 0xFF; \
(crc) = table[__tab_index] ^ ((crc) >> 8); \
} \
} while (0)
/* Check for equality of two CRCs */
#define EQ_CRC32(c1,c2) ((c1) == (c2))
#define COMP_CRC32_REFLECTED_TABLE(crc, data, len, table) \
do { \
const unsigned char *__data = (const unsigned char *) (data); \
uint32 __len = (len); \
\
while (__len-- > 0) \
{ \
int __tab_index = ((int) ((crc) >> 24) ^ *__data++) & 0xFF; \
(crc) = table[__tab_index] ^ ((crc) << 8); \
} \
} while (0)
/* Constant table for CRC calculation */
/* Constant tables for CRC-32C and CRC-32 polynomials */
extern CRCDLLIMPORT const uint32 pg_crc32c_table[];
extern CRCDLLIMPORT const uint32 pg_crc32_table[];
#endif /* PG_CRC_H */