lib/zstd
1
0
Fork 0
mirror of https://github.com/facebook/zstd.git synced 2025-08-08 17:22:10 +03:00
Code Activity

Rename and remove unneeded files

Browse Source
This commit is contained in:
Stella Lau
2017-07-25 15:17:36 -07:00
parent 0295a27133
commit 629c300118
7 changed files with 65 additions and 389 deletions
Download Patch File Download Diff File Expand all files Collapse all files

13
contrib/long_distance_matching/Makefile
View File

@@ -1,5 +1,5 @@
# ################################################################
# Copyright (c) 2016-present, Yann Collet, Facebook, Inc.
# Copyright (c) 2016-present, Facebook, Inc.
# All rights reserved.
#
# This source code is licensed under the BSD-style license found in the
@@ -25,19 +25,16 @@ LDFLAGS += -lzstd
default: all
all: main-circular-buffer main-integrated main-64
all: main-hash32 main-hash64
main-circular-buffer: ldm_common.c circular_buffer_table.c ldm.c main-ldm.c
main-hash64: ldm_common.c ldm_hash64.c main.c
$(CC) $(CPPFLAGS) $(CFLAGS) $^ $(LDFLAGS) -o $@
main-64: ldm_common.c ldm_64_hash.c main-ldm.c
$(CC) $(CPPFLAGS) $(CFLAGS) $^ $(LDFLAGS) -o $@
main-integrated: ldm_common.c ldm_integrated.c main-ldm.c
main-hash32: ldm_common.c ldm_hash32.c main.c
$(CC) $(CPPFLAGS) $(CFLAGS) $^ $(LDFLAGS) -o $@
clean:
@rm -f core *.o tmp* result* *.ldm *.ldm.dec \
main-circular-buffer main-64 main-integrated
main-hash64 main-hash32
@echo Cleaning completed

256
contrib/long_distance_matching/circular_buffer_table.c
View File

@@ -1,256 +0,0 @@
#include <stdlib.h>
#include <stdio.h>
#include "ldm.h"
#include "ldm_hashtable.h"
#include "mem.h"
// THe number of elements per hash bucket.
// HASH_BUCKET_SIZE_LOG is defined in ldm.h.
#define HASH_BUCKET_SIZE (1 << (HASH_BUCKET_SIZE_LOG))
// The number of hash buckets.
#define LDM_HASHLOG ((LDM_MEMORY_USAGE)-(LDM_HASH_ENTRY_SIZE_LOG)-(HASH_BUCKET_SIZE_LOG))
// If ZSTD_SKIP is defined, then the first entry is returned in HASH_getBestEntry
// (without looking at other entries in the bucket).
//#define ZSTD_SKIP
struct LDM_hashTable {
U32 numBuckets; // The number of buckets.
U32 numEntries; // numBuckets * HASH_BUCKET_SIZE.
LDM_hashEntry *entries;
BYTE *bucketOffsets; // A pointer (per bucket) to the next insert position.
const BYTE *offsetBase; // Corresponds to offset=0 in LDM_hashEntry.
U32 minMatchLength;
U32 maxWindowSize;
};
LDM_hashTable *HASH_createTable(U32 size, const BYTE *offsetBase,
U32 minMatchLength, U32 maxWindowSize) {
LDM_hashTable *table = malloc(sizeof(LDM_hashTable));
table->numBuckets = size >> HASH_BUCKET_SIZE_LOG;
table->numEntries = size;
table->entries = calloc(size, sizeof(LDM_hashEntry));
table->bucketOffsets = calloc(size >> HASH_BUCKET_SIZE_LOG, sizeof(BYTE));
table->offsetBase = offsetBase;
table->minMatchLength = minMatchLength;
table->maxWindowSize = maxWindowSize;
return table;
}
static LDM_hashEntry *getBucket(const LDM_hashTable *table, const hash_t hash) {
return table->entries + (hash << HASH_BUCKET_SIZE_LOG);
}
// From lib/compress/zstd_compress.c
static unsigned ZSTD_NbCommonBytes (register size_t val)
{
if (MEM_isLittleEndian()) {
if (MEM_64bits()) {
# if defined(_MSC_VER) && defined(_WIN64)
unsigned long r = 0;
_BitScanForward64( &r, (U64)val );
return (unsigned)(r>>3);
# elif defined(__GNUC__) && (__GNUC__ >= 3)
return (__builtin_ctzll((U64)val) >> 3);
# else
static const int DeBruijnBytePos[64] = { 0, 0, 0, 0, 0, 1, 1, 2,
0, 3, 1, 3, 1, 4, 2, 7,
0, 2, 3, 6, 1, 5, 3, 5,
1, 3, 4, 4, 2, 5, 6, 7,
7, 0, 1, 2, 3, 3, 4, 6,
2, 6, 5, 5, 3, 4, 5, 6,
7, 1, 2, 4, 6, 4, 4, 5,
7, 2, 6, 5, 7, 6, 7, 7 };
return DeBruijnBytePos[((U64)((val & -(long long)val) * 0x0218A392CDABBD3FULL)) >> 58];
# endif
} else { /* 32 bits */
# if defined(_MSC_VER)
unsigned long r=0;
_BitScanForward( &r, (U32)val );
return (unsigned)(r>>3);
# elif defined(__GNUC__) && (__GNUC__ >= 3)
return (__builtin_ctz((U32)val) >> 3);
# else
static const int DeBruijnBytePos[32] = { 0, 0, 3, 0, 3, 1, 3, 0,
3, 2, 2, 1, 3, 2, 0, 1,
3, 3, 1, 2, 2, 2, 2, 0,
3, 1, 2, 0, 1, 0, 1, 1 };
return DeBruijnBytePos[((U32)((val & -(S32)val) * 0x077CB531U)) >> 27];
# endif
}
} else { /* Big Endian CPU */
if (MEM_64bits()) {
# if defined(_MSC_VER) && defined(_WIN64)
unsigned long r = 0;
_BitScanReverse64( &r, val );
return (unsigned)(r>>3);
# elif defined(__GNUC__) && (__GNUC__ >= 3)
return (__builtin_clzll(val) >> 3);
# else
unsigned r;
const unsigned n32 = sizeof(size_t)*4; /* calculate this way due to compiler complaining in 32-bits mode */
if (!(val>>n32)) { r=4; } else { r=0; val>>=n32; }
if (!(val>>16)) { r+=2; val>>=8; } else { val>>=24; }
r += (!val);
return r;
# endif
} else { /* 32 bits */
# if defined(_MSC_VER)
unsigned long r = 0;
_BitScanReverse( &r, (unsigned long)val );
return (unsigned)(r>>3);
# elif defined(__GNUC__) && (__GNUC__ >= 3)
return (__builtin_clz((U32)val) >> 3);
# else
unsigned r;
if (!(val>>16)) { r=2; val>>=8; } else { r=0; val>>=24; }
r += (!val);
return r;
# endif
} }
}
/**
* From lib/compress/zstd_compress.c
* Returns the number of bytes (consecutively) in common between pIn and pMatch
* up to pInLimit.
*/
static size_t ZSTD_count(const BYTE *pIn, const BYTE *pMatch,
const BYTE *const pInLimit) {
const BYTE * const pStart = pIn;
const BYTE * const pInLoopLimit = pInLimit - (sizeof(size_t)-1);
while (pIn < pInLoopLimit) {
size_t const diff = MEM_readST(pMatch) ^ MEM_readST(pIn);
if (!diff) {
pIn += sizeof(size_t);
pMatch += sizeof(size_t);
continue;
}
pIn += ZSTD_NbCommonBytes(diff);
return (size_t)(pIn - pStart);
}
if (MEM_64bits()) {
if ((pIn < (pInLimit - 3)) && (MEM_read32(pMatch) == MEM_read32(pIn))) {
pIn += 4;
pMatch += 4;
}
}
if ((pIn < (pInLimit - 1)) && (MEM_read16(pMatch) == MEM_read16(pIn))) {
pIn += 2;
pMatch += 2;
}
if ((pIn < pInLimit) && (*pMatch == *pIn)) {
pIn++;
}
return (size_t)(pIn - pStart);
}
/**
* Returns the number of bytes in common between pIn and pMatch,
* counting backwards, with pIn having a lower limit of pAnchor and
* pMatch having a lower limit of pBase.
*/
static size_t countBackwardsMatch(const BYTE *pIn, const BYTE *pAnchor,
const BYTE *pMatch, const BYTE *pBase) {
size_t matchLength = 0;
while (pIn > pAnchor && pMatch > pBase && pIn[-1] == pMatch[-1]) {
pIn--;
pMatch--;
matchLength++;
}
return matchLength;
}
LDM_hashEntry *HASH_getBestEntry(const LDM_hashTable *table,
const hash_t hash,
const U32 checksum,
const BYTE *pIn,
const BYTE *pEnd,
const BYTE *pAnchor,
U64 *pForwardMatchLength,
U64 *pBackwardMatchLength) {
LDM_hashEntry *bucket = getBucket(table, hash);
LDM_hashEntry *cur = bucket;
LDM_hashEntry *bestEntry = NULL;
U64 bestMatchLength = 0;
for (; cur < bucket + HASH_BUCKET_SIZE; ++cur) {
const BYTE *pMatch = cur->offset + table->offsetBase;
// Check checksum for faster check.
if (cur->checksum == checksum && pIn - pMatch <= table->maxWindowSize) {
U64 forwardMatchLength = ZSTD_count(pIn, pMatch, pEnd);
U64 backwardMatchLength, totalMatchLength;
// Only take matches where the forwardMatchLength is large enough
// for speed.
if (forwardMatchLength < table->minMatchLength) {
continue;
}
backwardMatchLength =
countBackwardsMatch(pIn, pAnchor, cur->offset + table->offsetBase,
table->offsetBase);
totalMatchLength = forwardMatchLength + backwardMatchLength;
if (totalMatchLength >= bestMatchLength) {
bestMatchLength = totalMatchLength;
*pForwardMatchLength = forwardMatchLength;
*pBackwardMatchLength = backwardMatchLength;
bestEntry = cur;
#ifdef ZSTD_SKIP
return cur;
#endif
}
}
}
if (bestEntry != NULL) {
return bestEntry;
}
return NULL;
}
hash_t HASH_hashU32(U32 value) {
return ((value * 2654435761U) >> (32 - LDM_HASHLOG));
}
void HASH_insert(LDM_hashTable *table,
const hash_t hash, const LDM_hashEntry entry) {
// Circular buffer.
*(getBucket(table, hash) + table->bucketOffsets[hash]) = entry;
table->bucketOffsets[hash]++;
table->bucketOffsets[hash] &= HASH_BUCKET_SIZE - 1;
}
U32 HASH_getSize(const LDM_hashTable *table) {
return table->numBuckets;
}
void HASH_destroyTable(LDM_hashTable *table) {
free(table->entries);
free(table->bucketOffsets);
free(table);
}
void HASH_outputTableOccupancy(const LDM_hashTable *table) {
U32 ctr = 0;
LDM_hashEntry *cur = table->entries;
LDM_hashEntry *end = table->entries + (table->numBuckets * HASH_BUCKET_SIZE);
for (; cur < end; ++cur) {
if (cur->offset == 0) {
ctr++;
}
}
printf("Num buckets, bucket size: %d, %d\n",
table->numBuckets, HASH_BUCKET_SIZE);
printf("Hash table size, empty slots, %% empty: %u, %u, %.3f\n",
table->numEntries, ctr,
100.0 * (double)(ctr) / table->numEntries);
}

14
contrib/long_distance_matching/ldm.h
View File

@@ -17,17 +17,22 @@
// THe number of bytes storing the offset.
#define LDM_OFFSET_SIZE 4
// =============================================================================
// User parameters.
// =============================================================================
// Defines the size of the hash table.
// Note that this is not the number of buckets.
// Currently this should be less than WINDOW_SIZE_LOG + 4?
#define LDM_MEMORY_USAGE 23
#define LDM_MEMORY_USAGE 25
// The number of entries in a hash bucket.
#define HASH_BUCKET_SIZE_LOG 0 // The maximum is 4 for now.
#define HASH_BUCKET_SIZE_LOG 3 // The maximum is 4 for now.
// Defines the lag in inserting elements into the hash table.
#define LDM_LAG 0
// The maximum window size.
#define LDM_WINDOW_SIZE_LOG 28 // Max value is 30
#define LDM_WINDOW_SIZE (1 << (LDM_WINDOW_SIZE_LOG))
@@ -37,10 +42,11 @@
// Experimental.
//#define TMP_EVICTION // Experiment with eviction policies.
#define TMP_TAG_INSERT // Insertion policy based on hash.
#define INSERT_BY_TAG // Insertion policy based on hash.
#define USE_CHECKSUM 1
//#define USE_CHECKSUM (HASH_BUCKET_SIZE_LOG)
// =============================================================================
typedef struct LDM_compressStats LDM_compressStats;
typedef struct LDM_CCtx LDM_CCtx;

0
contrib/long_distance_matching/ldm_integrated.c → contrib/long_distance_matching/ldm_hash32.c
View File

65
contrib/long_distance_matching/ldm_64_hash.c → contrib/long_distance_matching/ldm_hash64.c
View File

@@ -489,7 +489,7 @@ void LDM_printCompressStats(const LDM_compressStats *stats) {
(double) stats->numMatches);
}
printf("\n");
#ifdef TMP_TAG_INSERT
#ifdef INSERT_BY_TAG
/*
printf("Lower bit distribution\n");
for (i = 0; i < (1 << HASH_ONLY_EVERY_LOG); i++) {
@@ -524,7 +524,7 @@ static U32 getChecksum(U64 hash) {
return (hash >> (64 - 32 - LDM_HASHLOG)) & 0xFFFFFFFF;
}
#ifdef TMP_TAG_INSERT
#ifdef INSERT_BY_TAG
static U32 lowerBitsFromHfHash(U64 hash) {
// The number of bits used so far is LDM_HASHLOG + 32.
// So there are 32 - LDM_HASHLOG bits left.
@@ -611,7 +611,7 @@ static void setNextHash(LDM_CCtx *cctx) {
cctx->lastPosHashed[LDM_HASH_LENGTH]);
cctx->nextPosHashed = cctx->nextIp;
#ifdef TMP_TAG_INSERT
#ifdef INSERT_BY_TAG
{
U32 hashEveryMask = lowerBitsFromHfHash(cctx->nextHash);
cctx->stats.TMP_totalHashCount++;
@@ -647,9 +647,13 @@ static void putHashOfCurrentPositionFromHash(LDM_CCtx *cctx, U64 hash) {
// Hash only every HASH_ONLY_EVERY times, based on cctx->ip.
// Note: this works only when cctx->step is 1.
#if LDM_LAG
if (((cctx->ip - cctx->ibase) & HASH_ONLY_EVERY) == HASH_ONLY_EVERY) {
// TODO: Off by one, but not important.
if (cctx->lagIp - cctx->ibase > 0) {
if (cctx -> lagIp - cctx->ibase > 0) {
#ifdef INSERT_BY_TAG
U32 hashEveryMask = lowerBitsFromHfHash(cctx->lagHash);
if (hashEveryMask == HASH_ONLY_EVERY) {
#else
if (((cctx->ip - cctx->ibase) & HASH_ONLY_EVERY) == HASH_ONLY_EVERY) {
#endif
U32 smallHash = getSmallHash(cctx->lagHash);
# if USE_CHECKSUM
@@ -664,23 +668,32 @@ static void putHashOfCurrentPositionFromHash(LDM_CCtx *cctx, U64 hash) {
# else
HASH_insert(cctx->hashTable, smallHash, entry);
# endif
} else {
# if USE_CHECKSUM
U32 checksum = getChecksum(hash);
const LDM_hashEntry entry = { cctx->lagIp - cctx->ibase, checksum };
# else
const LDM_hashEntry entry = { cctx->lagIp - cctx->ibase };
# endif
}
} else {
#ifdef INSERT_BY_TAG
U32 hashEveryMask = lowerBitsFromHfHash(hash);
if (hashEveryMask == HASH_ONLY_EVERY) {
#else
if (((cctx->ip - cctx->ibase) & HASH_ONLY_EVERY) == HASH_ONLY_EVERY) {
#endif
U32 smallHash = getSmallHash(hash);
# ifdef TMP_EVICTION
#if USE_CHECKSUM
U32 checksum = getChecksum(hash);
const LDM_hashEntry entry = { cctx->ip - cctx->ibase, checksum };
#else
const LDM_hashEntry entry = { cctx->ip - cctx->ibase };
#endif
#ifdef TMP_EVICTION
HASH_insert(cctx->hashTable, smallHash, entry, cctx);
# else
#else
HASH_insert(cctx->hashTable, smallHash, entry);
# endif
#endif
}
}
#else
#ifdef TMP_TAG_INSERT
#ifdef INSERT_BY_TAG
U32 hashEveryMask = lowerBitsFromHfHash(hash);
if (hashEveryMask == HASH_ONLY_EVERY) {
#else
@@ -799,7 +812,7 @@ static int LDM_findBestMatch(LDM_CCtx *cctx, const BYTE **match,
U64 hash;
hash_t smallHash;
U32 checksum;
#ifdef TMP_TAG_INSERT
#ifdef INSERT_BY_TAG
U32 hashEveryMask;
#endif
setNextHash(cctx);
@@ -807,7 +820,7 @@ static int LDM_findBestMatch(LDM_CCtx *cctx, const BYTE **match,
hash = cctx->nextHash;
smallHash = getSmallHash(hash);
checksum = getChecksum(hash);
#ifdef TMP_TAG_INSERT
#ifdef INSERT_BY_TAG
hashEveryMask = lowerBitsFromHfHash(hash);
#endif
@@ -817,7 +830,7 @@ static int LDM_findBestMatch(LDM_CCtx *cctx, const BYTE **match,
if (cctx->ip > cctx->imatchLimit) {
return 1;
}
#ifdef TMP_TAG_INSERT
#ifdef INSERT_BY_TAG
if (hashEveryMask == HASH_ONLY_EVERY) {
entry = HASH_getBestEntry(cctx, smallHash, checksum,
@@ -1003,13 +1016,17 @@ void LDM_outputConfiguration(void) {
printf("LDM_MEMORY_USAGE: %d\n", LDM_MEMORY_USAGE);
printf("HASH_ONLY_EVERY_LOG: %d\n", HASH_ONLY_EVERY_LOG);
printf("HASH_BUCKET_SIZE_LOG: %d\n", HASH_BUCKET_SIZE_LOG);
printf("LDM_LAG %d\n", LDM_LAG);
printf("USE_CHECKSUM %d\n", USE_CHECKSUM);
printf("LDM_LAG: %d\n", LDM_LAG);
printf("USE_CHECKSUM: %d\n", USE_CHECKSUM);
#ifdef INSERT_BY_TAG
printf("INSERT_BY_TAG: %d\n", 1);
#else
printf("INSERT_BY_TAG: %d\n", 0);
#endif
printf("HASH_CHAR_OFFSET: %d\n", HASH_CHAR_OFFSET);
printf("=====================\n");
}
// TODO: implement and test hash function
void LDM_test(const BYTE *src) {
const U32 diff = 100;

91
contrib/long_distance_matching/ldm_hashtable.h
View File

@@ -1,91 +0,0 @@
/**
* A "hash" table used in LDM compression.
*
* This is not exactly a hash table in the sense that inserted entries
* are not guaranteed to remain in the hash table.
*/
#ifndef LDM_HASHTABLE_H
#define LDM_HASHTABLE_H
#include "mem.h"
// The log size of LDM_hashEntry in bytes.
#define LDM_HASH_ENTRY_SIZE_LOG 3
typedef U32 hash_t;
typedef struct LDM_hashEntry {
U32 offset; // Represents the offset of the entry from offsetBase.
U32 checksum; // A checksum to select entries with the same hash value.
} LDM_hashEntry;
typedef struct LDM_hashTable LDM_hashTable;
/**
* Create a table that can contain size elements. This does not necessarily
* correspond to the number of hash buckets. The number of hash buckets
* is size / (1 << HASH_BUCKET_SIZE_LOG)
*
* minMatchLength is the minimum match length required in HASH_getBestEntry.
*
* maxWindowSize is the maximum distance from pIn in HASH_getBestEntry.
* The window is defined to be (pIn - offsetBase - offset).
*/
LDM_hashTable *HASH_createTable(U32 size, const BYTE *offsetBase,
U32 minMatchLength, U32 maxWindowSize);
/**
* Return the "best" entry from the table with the same hash and checksum.
*
* pIn: a pointer to the current input position.
* pEnd: a pointer to the maximum input position.
* pAnchor: a pointer to the minimum input position.
*
* This function computes the forward and backward match length from pIn
* and writes it to forwardMatchLength and backwardsMatchLength.
*
* E.g. for the two strings "aaabbbb" "aaabbbb" with pIn and the
* entry pointing at the first "b", the forward match length would be
* four (representing the "b" matches) and the backward match length would
* three (representing the "a" matches before the pointer).
*/
LDM_hashEntry *HASH_getBestEntry(const LDM_hashTable *table,
const hash_t hash,
const U32 checksum,
const BYTE *pIn,
const BYTE *pEnd,
const BYTE *pAnchor,
U64 *forwardMatchLength,
U64 *backwardsMatchLength);
/**
* Return a hash of the value.
*/
hash_t HASH_hashU32(U32 value);
/**
* Insert an LDM_hashEntry into the bucket corresponding to hash.
*
* An entry may be evicted in the process.
*/
void HASH_insert(LDM_hashTable *table, const hash_t hash,
const LDM_hashEntry entry);
/**
* Return the number of distinct hash buckets.
*/
U32 HASH_getSize(const LDM_hashTable *table);
/**
* Destroy the table.
*/
void HASH_destroyTable(LDM_hashTable *table);
/**
* Prints the percentage of the hash table occupied (where occupied is defined
* as the entry being non-zero).
*/
void HASH_outputTableOccupancy(const LDM_hashTable *hashTable);
#endif /* LDM_HASHTABLE_H */

15
contrib/long_distance_matching/main-ldm.c → contrib/long_distance_matching/main.c
View File

@@ -12,7 +12,7 @@
#include "ldm.h"
#include "zstd.h"
//#define TEST
//#define DECOMPRESS_AND_VERIFY
/* Compress file given by fname and output to oname.
* Returns 0 if successful, error code otherwise.
@@ -91,9 +91,10 @@ static int compress(const char *fname, const char *oname) {
ftruncate(fdout, compressedSize);
printf("%25s : %10lu -> %10lu - %s (%.2fx --- %.1f%%)\n", fname,
(size_t)statbuf.st_size, (size_t)compressedSize, oname,
(statbuf.st_size) / (double)compressedSize,
printf("%25s : %10lu -> %10lu - %s \n", fname,
(size_t)statbuf.st_size, (size_t)compressedSize, oname);
printf("Compression ratio: %.2fx --- %.1f%%\n",
(double)statbuf.st_size / (double)compressedSize,
(double)compressedSize / (double)(statbuf.st_size) * 100.0);
timeTaken = (double) (tv2.tv_usec - tv1.tv_usec) / 1000000 +
@@ -110,6 +111,7 @@ static int compress(const char *fname, const char *oname) {
return 0;
}
#ifdef DECOMPRESS
/* Decompress file compressed using LDM_compress.
* The input file should have the LDM_HEADER followed by payload.
* Returns 0 if succesful, and an error code otherwise.
@@ -162,7 +164,6 @@ static int decompress(const char *fname, const char *oname) {
src + LDM_HEADER_SIZE, statbuf.st_size - LDM_HEADER_SIZE,
dst, decompressedSize);
printf("Ret size out: %zu\n", outSize);
// ftruncate(fdout, decompressedSize);
close(fdin);
close(fdout);
@@ -207,6 +208,7 @@ static void verify(const char *inpFilename, const char *decFilename) {
fclose(decFp);
fclose(inpFp);
}
#endif
int main(int argc, const char *argv[]) {
const char * const exeName = argv[0];
@@ -237,6 +239,7 @@ int main(int argc, const char *argv[]) {
}
}
#ifdef DECOMPRESS_AND_VERIFY
/* Decompress */
{
struct timeval tv1, tv2;
@@ -252,6 +255,6 @@ int main(int argc, const char *argv[]) {
}
/* verify */
verify(inpFilename, decFilename);
#endif
return 0;
}