diff --git a/contrib/pzstd/Makefile b/contrib/pzstd/Makefile index d71cf5b34..40fce267e 100644 --- a/contrib/pzstd/Makefile +++ b/contrib/pzstd/Makefile @@ -30,7 +30,7 @@ else EXT = endif -.PHONY: default all test clean +.PHONY: default all test clean test32 googletest googletest32 default: pzstd @@ -41,7 +41,6 @@ libzstd.a: $(ZSTD_FILES) $(MAKE) -C $(ZSTDDIR) libzstd @cp $(ZSTDDIR)/libzstd.a . - Pzstd.o: Pzstd.h Pzstd.cpp ErrorHolder.h utils/*.h $(CXX) $(FLAGS) -c Pzstd.cpp -o $@ @@ -54,23 +53,58 @@ Options.o: Options.h Options.cpp main.o: main.cpp *.h utils/*.h $(CXX) $(FLAGS) -c main.cpp -o $@ -pzstd: Pzstd.o SkippableFrame.o Options.o main.o libzstd.a +pzstd: Pzstd.o SkippableFrame.o Options.o main.o libzstd.a $(CXX) $(FLAGS) $^ -o $@$(EXT) -lpthread +libzstd32.a: $(ZSTD_FILES) + $(MAKE) -C $(ZSTDDIR) libzstd MOREFLAGS="-m32" + @cp $(ZSTDDIR)/libzstd.a libzstd32.a + +Pzstd32.o: Pzstd.h Pzstd.cpp ErrorHolder.h utils/*.h + $(CXX) -m32 $(FLAGS) -c Pzstd.cpp -o $@ + +SkippableFrame32.o: SkippableFrame.h SkippableFrame.cpp utils/*.h + $(CXX) -m32 $(FLAGS) -c SkippableFrame.cpp -o $@ + +Options32.o: Options.h Options.cpp + $(CXX) -m32 $(FLAGS) -c Options.cpp -o $@ + +main32.o: main.cpp *.h utils/*.h + $(CXX) -m32 $(FLAGS) -c main.cpp -o $@ + +pzstd32: Pzstd32.o SkippableFrame32.o Options32.o main32.o libzstd32.a + $(CXX) -m32 $(FLAGS) $^ -o $@$(EXT) -lpthread + googletest: + @$(RM) -rf googletest @git clone https://github.com/google/googletest @mkdir -p googletest/build @cd googletest/build && cmake .. && make +googletest32: + @$(RM) -rf googletest + @git clone https://github.com/google/googletest + @mkdir -p googletest/build + @cd googletest/build && cmake .. -DCMAKE_CXX_FLAGS=-m32 && make + test: libzstd.a Pzstd.o Options.o SkippableFrame.o $(MAKE) -C utils/test clean $(MAKE) -C utils/test test $(MAKE) -C test clean $(MAKE) -C test test +test32: + $(MAKE) clean + $(MAKE) pzstd MOREFLAGS="-m32" + $(MAKE) -C utils/test clean + $(MAKE) -C utils/test test MOREFLAGS="-m32" + $(MAKE) -C test clean + $(MAKE) -C test test MOREFLAGS="-m32" + + clean: $(MAKE) -C $(ZSTDDIR) clean $(MAKE) -C utils/test clean $(MAKE) -C test clean - @$(RM) -rf libzstd.a *.o pzstd$(EXT) + @$(RM) -rf libzstd.a *.o pzstd$(EXT) pzstd32$(EXT) @echo Cleaning completed diff --git a/contrib/pzstd/Pzstd.cpp b/contrib/pzstd/Pzstd.cpp index 978eb9968..bf42fe81e 100644 --- a/contrib/pzstd/Pzstd.cpp +++ b/contrib/pzstd/Pzstd.cpp @@ -333,10 +333,9 @@ static size_t calculateStep( size_t step = size_t{1} << (params.cParams.windowLog + 2); // If file size is known, see if a smaller step will spread work more evenly if (size != 0) { - const std::uintmax_t newStep = size / std::uintmax_t{numThreads}; - if (newStep != 0 && - newStep <= std::uintmax_t{std::numeric_limits::max()}) { - step = std::min(step, size_t{newStep}); + const std::uintmax_t newStep = size / numThreads; + if (newStep != 0 && newStep <= std::numeric_limits::max()) { + step = std::min(step, static_cast(newStep)); } } return step; diff --git a/contrib/pzstd/test/PzstdTest.cpp b/contrib/pzstd/test/PzstdTest.cpp index 4075229ab..b8e0dbd2d 100644 --- a/contrib/pzstd/test/PzstdTest.cpp +++ b/contrib/pzstd/test/PzstdTest.cpp @@ -40,8 +40,6 @@ TEST(Pzstd, SmallSizes) { for (unsigned numThreads = 1; numThreads <= 4; numThreads *= 2) { for (unsigned level = 1; level <= 8; level *= 8) { auto errorGuard = makeScopeGuard([&] { - guard.dismiss(); - std::fprintf(stderr, "file: %s\n", inputFile.c_str()); std::fprintf(stderr, "pzstd headers: %u\n", headers); std::fprintf(stderr, "# threads: %u\n", numThreads); std::fprintf(stderr, "compression level: %u\n", level); @@ -79,8 +77,6 @@ TEST(Pzstd, LargeSizes) { for (unsigned numThreads = 1; numThreads <= 16; numThreads *= 4) { for (unsigned level = 1; level <= 4; level *= 2) { auto errorGuard = makeScopeGuard([&] { - guard.dismiss(); - std::fprintf(stderr, "file: %s\n", inputFile.c_str()); std::fprintf(stderr, "pzstd headers: %u\n", headers); std::fprintf(stderr, "# threads: %u\n", numThreads); std::fprintf(stderr, "compression level: %u\n", level); @@ -98,6 +94,34 @@ TEST(Pzstd, LargeSizes) { } } +TEST(Pzstd, ExtremelyLargeSize) { + unsigned seed = std::random_device{}(); + std::fprintf(stderr, "Pzstd.ExtremelyLargeSize seed: %u\n", seed); + std::mt19937 gen(seed); + + std::string inputFile = std::tmpnam(nullptr); + auto guard = makeScopeGuard([&] { std::remove(inputFile.c_str()); }); + + { + // Write 4GB + 64 MB + constexpr size_t kLength = 1 << 26; + std::unique_ptr buf(new uint8_t[kLength]); + auto fd = std::fopen(inputFile.c_str(), "wb"); + auto closeGuard = makeScopeGuard([&] { std::fclose(fd); }); + for (size_t i = 0; i < (1 << 6) + 1; ++i) { + RDG_genBuffer(buf.get(), kLength, 0.5, 0.0, gen()); + auto written = std::fwrite(buf.get(), 1, kLength, fd); + ASSERT_EQ(written, kLength); + } + } + + Options options; + options.overwrite = true; + options.inputFiles = {inputFile}; + options.compressionLevel = 1; + ASSERT_TRUE(roundTrip(options)); +} + TEST(Pzstd, ExtremelyCompressible) { std::string inputFile = std::tmpnam(nullptr); auto guard = makeScopeGuard([&] { std::remove(inputFile.c_str()); });