mirror of
https://github.com/lz4/lz4.git
synced 2026-10-24 01:48:57 +03:00
c0a1b631e6
lz4frame : - LZ4F_uncompressedUpdate() accepts linked blocks (dev and v1.10.0 could silently corrupt such frames) : single-threaded mode drops the LZ4 state's history when an uncompressed segment starts (the decoder's history contains the uncompressed data) ; multi-threaded mode needs nothing, jobs reload the 64 KB actually preceding them. LZ4F_compressBegin() now starts frames in compressed mode : the mode of the previous frame's end was kept, which skipped the history drop (frame corrupted with a dictionary). - Windows : worker stacks reserved instead of committed (STACK_SIZE_PARAM_IS_A_RESERVATION) ; #error when <windows.h> was already included without the Vista API (amalgamated sources) ; doc : no multi-threaded context use nor release under the loader lock (DllMain, DLL global objects). lib/Makefile, programs/Makefile : threads detection skipped by goals building nothing threaded (clean, uninstall, ...), run once per command (sub-makes inherit it) : make install probes once instead of 2-4 times, make clean never. lz4 : explicit -T# also limited to the jobs of each file (-T1000 on a 10 MB file : 3 workers, RSS 298 -> 27 MB at -9), and by the memory budget on all systems (-T99999 : 1024 liblz4 maximum, 331 threads within the 4 GB default budget) ; an explicit nb of threads reduced triggers a warning (automatic : only with -v). tests : frametest testUncompressedLinked (linked blocks, dictionary made of input, frames ending uncompressed), mixed updates and small-dst scripts and fuzzer with linked blocks ; frameMT uncompressed segments with linked blocks ; ossfuzz stream and uncompressed round-trip fuzzers with linked blocks ; test-lz4-multithread budget warnings.
156 lines
7.0 KiB
Bash
Executable File
156 lines
7.0 KiB
Bash
Executable File
#!/bin/sh
|
|
|
|
FPREFIX="tmp-tmt"
|
|
|
|
set -e
|
|
|
|
remove () {
|
|
rm -rf $FPREFIX*
|
|
}
|
|
|
|
trap remove EXIT
|
|
|
|
set -x
|
|
|
|
datagen -g13M > $FPREFIX-src # several jobs, last one incomplete
|
|
|
|
# standard block sizes : frames are cut into jobs (custom block sizes : see test-lz4-blocksize.sh)
|
|
# same output whatever the nb of threads, memory-mapped input or not, file or stdin, memory budget
|
|
for opts in "-1" "-1 -BD" "-9 -BD -BX" "-3 -B4 -BD" "-1 -B5" "--fast=3 -BD --no-frame-crc"
|
|
do
|
|
lz4 -f -T1 $opts $FPREFIX-src $FPREFIX-T1.lz4
|
|
lz4 -f -T4 $opts $FPREFIX-src $FPREFIX.lz4
|
|
cmp $FPREFIX-T1.lz4 $FPREFIX.lz4
|
|
lz4 -f -T0 $opts $FPREFIX-src $FPREFIX.lz4
|
|
cmp $FPREFIX-T1.lz4 $FPREFIX.lz4
|
|
lz4 -f -T3 --no-mmap $opts $FPREFIX-src $FPREFIX.lz4
|
|
cmp $FPREFIX-T1.lz4 $FPREFIX.lz4
|
|
lz4 -T2 $opts < $FPREFIX-src > $FPREFIX.lz4
|
|
cmp $FPREFIX-T1.lz4 $FPREFIX.lz4
|
|
lz4 -f --memory=50M $opts $FPREFIX-src $FPREFIX.lz4
|
|
cmp $FPREFIX-T1.lz4 $FPREFIX.lz4
|
|
lz4 -dc $FPREFIX-T1.lz4 | cmp - $FPREFIX-src
|
|
done
|
|
|
|
# files fitting into a single job (with independent blocks : compressed without worker threads),
|
|
# and files of different sizes (-m : the same nb of threads for all) : same output
|
|
head -c 1300000 $FPREFIX-src > $FPREFIX-s1 # 1 job with linked blocks, 2 with independent 64 KB blocks
|
|
head -c 300000 $FPREFIX-src > $FPREFIX-s2 # 1 job
|
|
for opts in "-1 -B4" "-1 -B4 -BD" "-9 -B4 -BD" "-1 -B5"
|
|
do
|
|
for f in $FPREFIX-s1 $FPREFIX-s2 $FPREFIX-src
|
|
do
|
|
lz4 -f -T1 --no-mmap $opts $f $f-ref.lz4
|
|
lz4 -f $opts $f $f.lz4
|
|
cmp $f-ref.lz4 $f.lz4
|
|
done
|
|
lz4 -f -m $opts $FPREFIX-s1 $FPREFIX-s2 $FPREFIX-src
|
|
for f in $FPREFIX-s1 $FPREFIX-s2 $FPREFIX-src
|
|
do
|
|
cmp $f-ref.lz4 $f.lz4
|
|
done
|
|
done
|
|
|
|
# memory budget : --memory=# and LZ4_MEMORY limit the nb of threads, not the output
|
|
lz4 -f --memory=abc $FPREFIX-src $FPREFIX.lz4 && exit 1 # must fail
|
|
lz4 -f --memory=0 $FPREFIX-src $FPREFIX.lz4 && exit 1 # must fail
|
|
lz4 -f --memory=18446744073709551616 $FPREFIX-src $FPREFIX.lz4 && exit 1 # must fail (overflow)
|
|
lz4 -f --memory=17179869184G $FPREFIX-src $FPREFIX.lz4 && exit 1 # must fail (overflow)
|
|
lz4 -f -1 $FPREFIX-src $FPREFIX-ref.lz4
|
|
MT=0; if lz4 -V 2>&1 | grep -q "multithread"; then MT=1; fi # otherwise, -T# and memory budget are ignored
|
|
# a requested nb of threads (-T#) reduced by the budget : warning (silenced by -q) ; automatic : only with -v
|
|
lz4 -f -T64 --memory=50M $FPREFIX-src $FPREFIX.lz4 2> $FPREFIX.err
|
|
cmp $FPREFIX-ref.lz4 $FPREFIX.lz4
|
|
if [ $MT = 1 ]; then grep "Warning : memory budget (50 MB)" $FPREFIX.err; fi
|
|
lz4 -q -f -T64 --memory=50M $FPREFIX-src $FPREFIX.lz4 2> $FPREFIX.err
|
|
if grep "memory budget" $FPREFIX.err; then exit 1; fi
|
|
lz4 -f --memory=50M $FPREFIX-src $FPREFIX.lz4 2> $FPREFIX.err
|
|
if grep -i "memory budget" $FPREFIX.err; then exit 1; fi
|
|
LZ4_MEMORY=50MB lz4 -f -T64 $FPREFIX-src $FPREFIX.lz4 2> $FPREFIX.err
|
|
if [ $MT = 1 ]; then grep "Warning : memory budget (50 MB)" $FPREFIX.err; fi
|
|
LZ4_MEMORY=2GiB lz4 -f -T64 --memory 50M $FPREFIX-src $FPREFIX.lz4 2> $FPREFIX.err # command line wins
|
|
if [ $MT = 1 ]; then grep "Warning : memory budget (50 MB)" $FPREFIX.err; fi
|
|
lz4 -f -T99999 $FPREFIX-src $FPREFIX.lz4 2> $FPREFIX.err # within liblz4 maximum and default budget
|
|
cmp $FPREFIX-ref.lz4 $FPREFIX.lz4
|
|
if [ $MT = 1 ]; then grep "Warning : requested 99999 threads too large" $FPREFIX.err; grep "Warning : memory budget (" $FPREFIX.err; fi
|
|
LZ4_MEMORY=abc lz4 -f $FPREFIX-src $FPREFIX.lz4 2>&1 | grep "LZ4_MEMORY=abc" # ignored, with a warning
|
|
cmp $FPREFIX-ref.lz4 $FPREFIX.lz4
|
|
lz4 -f --memory=1M $FPREFIX-src $FPREFIX.lz4 2> $FPREFIX.err # warning only
|
|
cmp $FPREFIX-ref.lz4 $FPREFIX.lz4
|
|
if [ $MT = 1 ]; then grep "too small" $FPREFIX.err; fi
|
|
|
|
|
|
# Source modified during its compression (see --[no-]mmap) :
|
|
# output goes into a fifo, read only once the source is modified :
|
|
# lz4 is then blocked writing its first output, after reading its first 4 MB chunk.
|
|
# -T1 : 3 job slots, so lz4 is blocked before reaching the end of the source.
|
|
case "$(uname)" in MINGW*|MSYS*|CYGWIN*) exit 0 ;; esac
|
|
command -v mkfifo > /dev/null || exit 0
|
|
MMAP=0; if lz4 -V 2>&1 | grep -q "64-bit"; then MMAP=1; fi # memory-mapped input : POSIX 64-bit systems
|
|
|
|
rewriteStart() { printf 'modified' | dd of=$FPREFIX-mod bs=8 count=1 conv=notrunc 2> /dev/null; }
|
|
appendData() { echo "appended" >> $FPREFIX-mod; }
|
|
replaceSrc() { mv $FPREFIX-mod $FPREFIX-mod.old; echo "new file" > $FPREFIX-mod; }
|
|
|
|
# compressModified OPTIONS MODIFICATION : compresses $FPREFIX-mod into $FPREFIX.out, sets rc
|
|
compressModified() {
|
|
rm -f $FPREFIX-mod* $FPREFIX.fifo $FPREFIX.out
|
|
cp $FPREFIX-src $FPREFIX-mod
|
|
touch -t 200001010000 $FPREFIX-mod # modification detected even with 1-second timestamps
|
|
mkfifo $FPREFIX.fifo
|
|
lz4 -f -T1 $1 $FPREFIX-mod $FPREFIX.fifo 2> $FPREFIX.err &
|
|
pid=$!
|
|
{ dd bs=4 count=1 2> /dev/null; $2 < /dev/null; cat; } < $FPREFIX.fifo > $FPREFIX.out
|
|
rc=0; wait $pid || rc=$?
|
|
cat $FPREFIX.err
|
|
}
|
|
|
|
# content already read is rewritten : memory-mapped input, read again, would be inconsistent (error 54) ;
|
|
# except on macOS, where a private mapping keeps the content as opened : then the output is valid
|
|
compressModified "" rewriteStart
|
|
if [ $MMAP = 1 ] && [ "$(uname)" != Darwin ]; then
|
|
test $rc = 54
|
|
grep -q "output removed" $FPREFIX.err
|
|
test ! -e $FPREFIX.fifo
|
|
else
|
|
test $rc = 0
|
|
lz4 -dc $FPREFIX.out | cmp - $FPREFIX-src
|
|
fi
|
|
# input read once : the output holds the content read
|
|
compressModified "--no-mmap" rewriteStart
|
|
test $rc = 0
|
|
grep -q "Warning" $FPREFIX.err
|
|
lz4 -dc $FPREFIX.out | cmp - $FPREFIX-src
|
|
# --rm not honored : warning also displayed when notifications are lowered (stdout)
|
|
rm -f $FPREFIX-mod $FPREFIX.fifo; cp $FPREFIX-src $FPREFIX-mod; touch -t 200001010000 $FPREFIX-mod; mkfifo $FPREFIX.fifo
|
|
lz4 -c -T1 --no-mmap --rm $FPREFIX-mod > $FPREFIX.fifo 2> $FPREFIX.err &
|
|
pid=$!
|
|
{ dd bs=4 count=1 2> /dev/null; rewriteStart < /dev/null; cat; } < $FPREFIX.fifo > $FPREFIX.out
|
|
wait $pid
|
|
grep -q "source kept" $FPREFIX.err
|
|
test -f $FPREFIX-mod
|
|
|
|
# growing file : memory-mapped input holds the file as it was when opened, verified with the frame checksum ;
|
|
# input read once also holds the appended content (unless end of file was already met : libc dependent) ; --rm keeps the source
|
|
compressModified "--rm" appendData
|
|
test $rc = 0
|
|
grep -q "source kept" $FPREFIX.err
|
|
test -f $FPREFIX-mod
|
|
if [ $MMAP = 1 ]; then lz4 -dc $FPREFIX.out | cmp - $FPREFIX-src; fi
|
|
compressModified "--rm --no-mmap" appendData
|
|
test $rc = 0
|
|
test -f $FPREFIX-mod
|
|
lz4 -dc $FPREFIX.out > $FPREFIX.dec
|
|
cmp $FPREFIX.dec $FPREFIX-mod || cmp $FPREFIX.dec $FPREFIX-src
|
|
# without content checksum, memory-mapped input can't be verified
|
|
compressModified "--no-frame-crc" appendData
|
|
if [ $MMAP = 1 ]; then test $rc = 54; else test $rc = 0; fi
|
|
|
|
# source renamed and replaced (ex: log rotation) : --rm must not remove the new file
|
|
compressModified "--rm" replaceSrc
|
|
test $rc = 0
|
|
grep -q "renamed or replaced" $FPREFIX.err
|
|
test "$(cat $FPREFIX-mod)" = "new file"
|
|
test -f $FPREFIX-mod.old
|
|
lz4 -dc $FPREFIX.out | cmp - $FPREFIX-src
|