Skip to content

faster CRC32 implementation #5023

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
UplinkCoder merged 1 commit into from
Jan 10, 2017
Merged

faster CRC32 implementation #5023

Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
117 changes: 75 additions & 42 deletions std/digest/crc.d
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -100,46 +100,39 @@ unittest
hash = crc.finish();
}

private immutable uint[256] crc32_table =
[
0x00000000,0x77073096,0xee0e612c,0x990951ba,0x076dc419,0x706af48f,0xe963a535,
0x9e6495a3,0x0edb8832,0x79dcb8a4,0xe0d5e91e,0x97d2d988,0x09b64c2b,0x7eb17cbd,
0xe7b82d07,0x90bf1d91,0x1db71064,0x6ab020f2,0xf3b97148,0x84be41de,0x1adad47d,
0x6ddde4eb,0xf4d4b551,0x83d385c7,0x136c9856,0x646ba8c0,0xfd62f97a,0x8a65c9ec,
0x14015c4f,0x63066cd9,0xfa0f3d63,0x8d080df5,0x3b6e20c8,0x4c69105e,0xd56041e4,
0xa2677172,0x3c03e4d1,0x4b04d447,0xd20d85fd,0xa50ab56b,0x35b5a8fa,0x42b2986c,
0xdbbbc9d6,0xacbcf940,0x32d86ce3,0x45df5c75,0xdcd60dcf,0xabd13d59,0x26d930ac,
0x51de003a,0xc8d75180,0xbfd06116,0x21b4f4b5,0x56b3c423,0xcfba9599,0xb8bda50f,
0x2802b89e,0x5f058808,0xc60cd9b2,0xb10be924,0x2f6f7c87,0x58684c11,0xc1611dab,
0xb6662d3d,0x76dc4190,0x01db7106,0x98d220bc,0xefd5102a,0x71b18589,0x06b6b51f,
0x9fbfe4a5,0xe8b8d433,0x7807c9a2,0x0f00f934,0x9609a88e,0xe10e9818,0x7f6a0dbb,
0x086d3d2d,0x91646c97,0xe6635c01,0x6b6b51f4,0x1c6c6162,0x856530d8,0xf262004e,
0x6c0695ed,0x1b01a57b,0x8208f4c1,0xf50fc457,0x65b0d9c6,0x12b7e950,0x8bbeb8ea,
0xfcb9887c,0x62dd1ddf,0x15da2d49,0x8cd37cf3,0xfbd44c65,0x4db26158,0x3ab551ce,
0xa3bc0074,0xd4bb30e2,0x4adfa541,0x3dd895d7,0xa4d1c46d,0xd3d6f4fb,0x4369e96a,
0x346ed9fc,0xad678846,0xda60b8d0,0x44042d73,0x33031de5,0xaa0a4c5f,0xdd0d7cc9,
0x5005713c,0x270241aa,0xbe0b1010,0xc90c2086,0x5768b525,0x206f85b3,0xb966d409,
0xce61e49f,0x5edef90e,0x29d9c998,0xb0d09822,0xc7d7a8b4,0x59b33d17,0x2eb40d81,
0xb7bd5c3b,0xc0ba6cad,0xedb88320,0x9abfb3b6,0x03b6e20c,0x74b1d29a,0xead54739,
0x9dd277af,0x04db2615,0x73dc1683,0xe3630b12,0x94643b84,0x0d6d6a3e,0x7a6a5aa8,
0xe40ecf0b,0x9309ff9d,0x0a00ae27,0x7d079eb1,0xf00f9344,0x8708a3d2,0x1e01f268,
0x6906c2fe,0xf762575d,0x806567cb,0x196c3671,0x6e6b06e7,0xfed41b76,0x89d32be0,
0x10da7a5a,0x67dd4acc,0xf9b9df6f,0x8ebeeff9,0x17b7be43,0x60b08ed5,0xd6d6a3e8,
0xa1d1937e,0x38d8c2c4,0x4fdff252,0xd1bb67f1,0xa6bc5767,0x3fb506dd,0x48b2364b,
0xd80d2bda,0xaf0a1b4c,0x36034af6,0x41047a60,0xdf60efc3,0xa867df55,0x316e8eef,
0x4669be79,0xcb61b38c,0xbc66831a,0x256fd2a0,0x5268e236,0xcc0c7795,0xbb0b4703,
0x220216b9,0x5505262f,0xc5ba3bbe,0xb2bd0b28,0x2bb45a92,0x5cb36a04,0xc2d7ffa7,
0xb5d0cf31,0x2cd99e8b,0x5bdeae1d,0x9b64c2b0,0xec63f226,0x756aa39c,0x026d930a,
0x9c0906a9,0xeb0e363f,0x72076785,0x05005713,0x95bf4a82,0xe2b87a14,0x7bb12bae,
0x0cb61b38,0x92d28e9b,0xe5d5be0d,0x7cdcefb7,0x0bdbdf21,0x86d3d2d4,0xf1d4e242,
0x68ddb3f8,0x1fda836e,0x81be16cd,0xf6b9265b,0x6fb077e1,0x18b74777,0x88085ae6,
0xff0f6a70,0x66063bca,0x11010b5c,0x8f659eff,0xf862ae69,0x616bffd3,0x166ccf45,
0xa00ae278,0xd70dd2ee,0x4e048354,0x3903b3c2,0xa7672661,0xd06016f7,0x4969474d,
0x3e6e77db,0xaed16a4a,0xd9d65adc,0x40df0b66,0x37d83bf0,0xa9bcae53,0xdebb9ec5,
0x47b2cf7f,0x30b5ffe9,0xbdbdf21c,0xcabac28a,0x53b39330,0x24b4a3a6,0xbad03605,
0xcdd70693,0x54de5729,0x23d967bf,0xb3667a2e,0xc4614ab8,0x5d681b02,0x2a6f2b94,
0xb40bbe37,0xc30c8ea1,0x5a05df1b,0x2d02ef8d
];
private uint[256][8] genTables(uint polynomial)
{
uint[256][8] res = void;

foreach (i; 0 .. 0x100)
{
uint crc = i;
foreach (_; 0 .. 8)
crc = (crc >> 1) ^ (-int(crc & 1) & polynomial);
res[0][i] = crc;
}

foreach (i; 0 .. 0x100)
{
res[1][i] = (res[0][i] >> 8) ^ res[0][res[0][i] & 0xFF];
res[2][i] = (res[1][i] >> 8) ^ res[0][res[1][i] & 0xFF];
res[3][i] = (res[2][i] >> 8) ^ res[0][res[2][i] & 0xFF];

res[4][i] = (res[3][i] >> 8) ^ res[0][res[3][i] & 0xFF];
res[5][i] = (res[4][i] >> 8) ^ res[0][res[4][i] & 0xFF];
res[6][i] = (res[5][i] >> 8) ^ res[0][res[5][i] & 0xFF];
res[7][i] = (res[6][i] >> 8) ^ res[0][res[6][i] & 0xFF];
}
return res;
}

private static immutable uint[256][8] crc32Tables = genTables(0xEDB88320);

unittest
{
auto tables = genTables(0xEDB88320);
assert(tables[0][0] == 0x00000000 && tables[0][$ - 1] == 0x2d02ef8d && tables[7][$ - 1] == 0x264b06e6);
}

/**
* Template API CRC32 implementation.
Expand All @@ -159,8 +152,48 @@ struct CRC32
*/
void put(scope const(ubyte)[] data...) @trusted pure nothrow @nogc
{
foreach (val; data)
_state = (_state >> 8) ^ crc32_table[cast(ubyte)_state ^ val];
uint crc = _state;
// process eight bytes at once
while (data.length >= 8)
{
// Use byte-wise reads to support architectures without HW support
// for unaligned reads. This can be optimized by compilers to a single
// 32-bit read if unaligned reads are supported.
// DMD is not able to do this optimization though, so explicitly
// do unaligned reads for DMD's architectures.
version (X86)
enum hasLittleEndianUnalignedReads = true;
else version (X86_64)
enum hasLittleEndianUnalignedReads = true;
else
enum hasLittleEndianUnalignedReads = false; // leave decision to optimizer
static if (hasLittleEndianUnalignedReads)
{
uint one = (cast(uint*) data.ptr)[0] ^ crc;
uint two = (cast(uint*) data.ptr)[1];
}
else
{
uint one = (data.ptr[3] << 24 | data.ptr[2] << 16 | data.ptr[1] << 8 | data.ptr[0]) ^ crc;
uint two = (data.ptr[7] << 24 | data.ptr[6] << 16 | data.ptr[5] << 8 | data.ptr[4]);
}

crc =
crc32Tables[0][two >> 24] ^
crc32Tables[1][(two >> 16) & 0xFF] ^
crc32Tables[2][(two >> 8) & 0xFF] ^
crc32Tables[3][two & 0xFF] ^
crc32Tables[4][one >> 24] ^
crc32Tables[5][(one >> 16) & 0xFF] ^
crc32Tables[6][(one >> 8) & 0xFF] ^
crc32Tables[7][one & 0xFF];

data = data[8 .. $];
}
// remaining 1 to 7 bytes
foreach (d; data)
crc = (crc >> 8) ^ crc32Tables[0][(crc & 0xFF) ^ d];
_state = crc;
}
///
unittest
Expand Down