From a98a6e2091a4d64df396216c2fbf695917a53e8c Mon Sep 17 00:00:00 2001 From: Nick Terrell Date: Mon, 4 Jan 2021 15:04:29 -0800 Subject: [PATCH] [test][regression] Add no source size with dictionary test * Add a test that runs without a pledgedSrcSize and with a dictionary. * Add github.tar data with uses the github dictionary while compressing github.tar, instead of each file individually. --- tests/regression/config.c | 9 ++ tests/regression/data.c | 17 ++ tests/regression/results.csv | 292 +++++++++++++++++++++++++++++++++++ 3 files changed, 318 insertions(+) diff --git a/tests/regression/config.c b/tests/regression/config.c index 232f0fc0f..ee0a71e4d 100644 --- a/tests/regression/config.c +++ b/tests/regression/config.c @@ -59,6 +59,14 @@ static config_t no_pledged_src_size = { .no_pledged_src_size = 1, }; +static config_t no_pledged_src_size_with_dict = { + .name = "no source size with dict", + .cli_args = "", + .param_values = PARAM_VALUES(level_0_param_values), + .no_pledged_src_size = 1, + .use_dictionary = 1, +}; + static param_value_t const ldm_param_values[] = { {.param = ZSTD_c_enableLongDistanceMatching, .value = 1}, }; @@ -192,6 +200,7 @@ static config_t const* g_configs[] = { #undef FAST_LEVEL &no_pledged_src_size, + &no_pledged_src_size_with_dict, &ldm, &mt, &mt_ldm, diff --git a/tests/regression/data.c b/tests/regression/data.c index c3ccf84f8..b06c6914f 100644 --- a/tests/regression/data.c +++ b/tests/regression/data.c @@ -67,10 +67,27 @@ data_t github = { }, }; +data_t github_tar = { + .name = "github.tar", + .type = data_type_file, + .data = + { + .url = REGRESSION_RELEASE("github.tar.zst"), + .xxhash64 = 0xa9b1b44b020df292LL, + }, + .dict = + { + .url = REGRESSION_RELEASE("github.dict.zst"), + .xxhash64 = 0x1eddc6f737d3cb53LL, + + }, +}; + static data_t* g_data[] = { &silesia, &silesia_tar, &github, + &github_tar, NULL, }; diff --git a/tests/regression/results.csv b/tests/regression/results.csv index 979b1d250..587b6afbe 100644 --- a/tests/regression/results.csv +++ b/tests/regression/results.csv @@ -16,6 +16,23 @@ silesia.tar, level 19, compress silesia.tar, uncompressed literals, compress simple, 4861425 silesia.tar, uncompressed literals optimal, compress simple, 4281605 silesia.tar, huffman literals, compress simple, 6186042 +github.tar, level -5, compress simple, 46856 +github.tar, level -3, compress simple, 43754 +github.tar, level -1, compress simple, 42490 +github.tar, level 0, compress simple, 38441 +github.tar, level 1, compress simple, 39265 +github.tar, level 3, compress simple, 38441 +github.tar, level 4, compress simple, 38467 +github.tar, level 5, compress simple, 39788 +github.tar, level 6, compress simple, 39603 +github.tar, level 7, compress simple, 39206 +github.tar, level 9, compress simple, 36717 +github.tar, level 13, compress simple, 35621 +github.tar, level 16, compress simple, 40255 +github.tar, level 19, compress simple, 32837 +github.tar, uncompressed literals, compress simple, 38441 +github.tar, uncompressed literals optimal, compress simple, 32837 +github.tar, huffman literals, compress simple, 42490 silesia, level -5, compress cctx, 6737607 silesia, level -3, compress cctx, 6444677 silesia, level -1, compress cctx, 6178460 @@ -170,6 +187,47 @@ github, uncompressed literals, zstdcli, github, uncompressed literals optimal, zstdcli, 159227 github, huffman literals, zstdcli, 144465 github, multithreaded with advanced params, zstdcli, 167915 +github.tar, level -5, zstdcli, 46751 +github.tar, level -5 with dict, zstdcli, 43975 +github.tar, level -3, zstdcli, 43541 +github.tar, level -3 with dict, zstdcli, 40809 +github.tar, level -1, zstdcli, 42469 +github.tar, level -1 with dict, zstdcli, 41126 +github.tar, level 0, zstdcli, 38445 +github.tar, level 0 with dict, zstdcli, 37999 +github.tar, level 1, zstdcli, 39346 +github.tar, level 1 with dict, zstdcli, 38313 +github.tar, level 3, zstdcli, 38445 +github.tar, level 3 with dict, zstdcli, 37999 +github.tar, level 4, zstdcli, 38471 +github.tar, level 4 with dict, zstdcli, 37952 +github.tar, level 5, zstdcli, 39792 +github.tar, level 5 with dict, zstdcli, 39231 +github.tar, level 6, zstdcli, 39607 +github.tar, level 6 with dict, zstdcli, 38669 +github.tar, level 7, zstdcli, 39210 +github.tar, level 7 with dict, zstdcli, 37958 +github.tar, level 9, zstdcli, 36721 +github.tar, level 9 with dict, zstdcli, 36886 +github.tar, level 13, zstdcli, 35625 +github.tar, level 13 with dict, zstdcli, 38730 +github.tar, level 16, zstdcli, 40259 +github.tar, level 16 with dict, zstdcli, 33643 +github.tar, level 19, zstdcli, 32841 +github.tar, level 19 with dict, zstdcli, 32899 +github.tar, no source size, zstdcli, 38442 +github.tar, no source size with dict, zstdcli, 213380 +github.tar, long distance mode, zstdcli, 39680 +github.tar, multithreaded, zstdcli, 38445 +github.tar, multithreaded long distance mode, zstdcli, 39680 +github.tar, small window log, zstdcli, 199432 +github.tar, small hash log, zstdcli, 129874 +github.tar, small chain log, zstdcli, 41673 +github.tar, explicit params, zstdcli, 41199 +github.tar, uncompressed literals, zstdcli, 41126 +github.tar, uncompressed literals optimal, zstdcli, 35392 +github.tar, huffman literals, zstdcli, 38804 +github.tar, multithreaded with advanced params, zstdcli, 41126 silesia, level -5, advanced one pass, 6737607 silesia, level -3, advanced one pass, 6444677 silesia, level -1, advanced one pass, 6178460 @@ -251,6 +309,7 @@ github, level 16 with dict, advanced github, level 19, advanced one pass, 134064 github, level 19 with dict, advanced one pass, 37576 github, no source size, advanced one pass, 136335 +github, no source size with dict, advanced one pass, 41148 github, long distance mode, advanced one pass, 136335 github, multithreaded, advanced one pass, 136335 github, multithreaded long distance mode, advanced one pass, 136335 @@ -262,6 +321,47 @@ github, uncompressed literals, advanced github, uncompressed literals optimal, advanced one pass, 157227 github, huffman literals, advanced one pass, 142465 github, multithreaded with advanced params, advanced one pass, 165915 +github.tar, level -5, advanced one pass, 46856 +github.tar, level -5 with dict, advanced one pass, 43971 +github.tar, level -3, advanced one pass, 43754 +github.tar, level -3 with dict, advanced one pass, 40805 +github.tar, level -1, advanced one pass, 42490 +github.tar, level -1 with dict, advanced one pass, 41122 +github.tar, level 0, advanced one pass, 38441 +github.tar, level 0 with dict, advanced one pass, 37995 +github.tar, level 1, advanced one pass, 39265 +github.tar, level 1 with dict, advanced one pass, 38309 +github.tar, level 3, advanced one pass, 38441 +github.tar, level 3 with dict, advanced one pass, 37995 +github.tar, level 4, advanced one pass, 38467 +github.tar, level 4 with dict, advanced one pass, 37948 +github.tar, level 5, advanced one pass, 39788 +github.tar, level 5 with dict, advanced one pass, 39715 +github.tar, level 6, advanced one pass, 39603 +github.tar, level 6 with dict, advanced one pass, 38800 +github.tar, level 7, advanced one pass, 39206 +github.tar, level 7 with dict, advanced one pass, 38071 +github.tar, level 9, advanced one pass, 36717 +github.tar, level 9 with dict, advanced one pass, 36898 +github.tar, level 13, advanced one pass, 35621 +github.tar, level 13 with dict, advanced one pass, 38726 +github.tar, level 16, advanced one pass, 40255 +github.tar, level 16 with dict, advanced one pass, 33639 +github.tar, level 19, advanced one pass, 32837 +github.tar, level 19 with dict, advanced one pass, 32895 +github.tar, no source size, advanced one pass, 38441 +github.tar, no source size with dict, advanced one pass, 37995 +github.tar, long distance mode, advanced one pass, 39676 +github.tar, multithreaded, advanced one pass, 38441 +github.tar, multithreaded long distance mode, advanced one pass, 39676 +github.tar, small window log, advanced one pass, 198540 +github.tar, small hash log, advanced one pass, 129870 +github.tar, small chain log, advanced one pass, 41669 +github.tar, explicit params, advanced one pass, 41199 +github.tar, uncompressed literals, advanced one pass, 41122 +github.tar, uncompressed literals optimal, advanced one pass, 35388 +github.tar, huffman literals, advanced one pass, 38777 +github.tar, multithreaded with advanced params, advanced one pass, 41122 silesia, level -5, advanced one pass small out, 6737607 silesia, level -3, advanced one pass small out, 6444677 silesia, level -1, advanced one pass small out, 6178460 @@ -343,6 +443,7 @@ github, level 16 with dict, advanced github, level 19, advanced one pass small out, 134064 github, level 19 with dict, advanced one pass small out, 37576 github, no source size, advanced one pass small out, 136335 +github, no source size with dict, advanced one pass small out, 41148 github, long distance mode, advanced one pass small out, 136335 github, multithreaded, advanced one pass small out, 136335 github, multithreaded long distance mode, advanced one pass small out, 136335 @@ -354,6 +455,47 @@ github, uncompressed literals, advanced github, uncompressed literals optimal, advanced one pass small out, 157227 github, huffman literals, advanced one pass small out, 142465 github, multithreaded with advanced params, advanced one pass small out, 165915 +github.tar, level -5, advanced one pass small out, 46856 +github.tar, level -5 with dict, advanced one pass small out, 43971 +github.tar, level -3, advanced one pass small out, 43754 +github.tar, level -3 with dict, advanced one pass small out, 40805 +github.tar, level -1, advanced one pass small out, 42490 +github.tar, level -1 with dict, advanced one pass small out, 41122 +github.tar, level 0, advanced one pass small out, 38441 +github.tar, level 0 with dict, advanced one pass small out, 37995 +github.tar, level 1, advanced one pass small out, 39265 +github.tar, level 1 with dict, advanced one pass small out, 38309 +github.tar, level 3, advanced one pass small out, 38441 +github.tar, level 3 with dict, advanced one pass small out, 37995 +github.tar, level 4, advanced one pass small out, 38467 +github.tar, level 4 with dict, advanced one pass small out, 37948 +github.tar, level 5, advanced one pass small out, 39788 +github.tar, level 5 with dict, advanced one pass small out, 39715 +github.tar, level 6, advanced one pass small out, 39603 +github.tar, level 6 with dict, advanced one pass small out, 38800 +github.tar, level 7, advanced one pass small out, 39206 +github.tar, level 7 with dict, advanced one pass small out, 38071 +github.tar, level 9, advanced one pass small out, 36717 +github.tar, level 9 with dict, advanced one pass small out, 36898 +github.tar, level 13, advanced one pass small out, 35621 +github.tar, level 13 with dict, advanced one pass small out, 38726 +github.tar, level 16, advanced one pass small out, 40255 +github.tar, level 16 with dict, advanced one pass small out, 33639 +github.tar, level 19, advanced one pass small out, 32837 +github.tar, level 19 with dict, advanced one pass small out, 32895 +github.tar, no source size, advanced one pass small out, 38441 +github.tar, no source size with dict, advanced one pass small out, 37995 +github.tar, long distance mode, advanced one pass small out, 39676 +github.tar, multithreaded, advanced one pass small out, 38441 +github.tar, multithreaded long distance mode, advanced one pass small out, 39676 +github.tar, small window log, advanced one pass small out, 198540 +github.tar, small hash log, advanced one pass small out, 129870 +github.tar, small chain log, advanced one pass small out, 41669 +github.tar, explicit params, advanced one pass small out, 41199 +github.tar, uncompressed literals, advanced one pass small out, 41122 +github.tar, uncompressed literals optimal, advanced one pass small out, 35388 +github.tar, huffman literals, advanced one pass small out, 38777 +github.tar, multithreaded with advanced params, advanced one pass small out, 41122 silesia, level -5, advanced streaming, 6882505 silesia, level -3, advanced streaming, 6568376 silesia, level -1, advanced streaming, 6183403 @@ -435,6 +577,7 @@ github, level 16 with dict, advanced github, level 19, advanced streaming, 134064 github, level 19 with dict, advanced streaming, 37576 github, no source size, advanced streaming, 136335 +github, no source size with dict, advanced streaming, 41148 github, long distance mode, advanced streaming, 136335 github, multithreaded, advanced streaming, 136335 github, multithreaded long distance mode, advanced streaming, 136335 @@ -446,6 +589,47 @@ github, uncompressed literals, advanced github, uncompressed literals optimal, advanced streaming, 157227 github, huffman literals, advanced streaming, 142465 github, multithreaded with advanced params, advanced streaming, 165915 +github.tar, level -5, advanced streaming, 46747 +github.tar, level -5 with dict, advanced streaming, 43971 +github.tar, level -3, advanced streaming, 43537 +github.tar, level -3 with dict, advanced streaming, 40805 +github.tar, level -1, advanced streaming, 42465 +github.tar, level -1 with dict, advanced streaming, 41122 +github.tar, level 0, advanced streaming, 38441 +github.tar, level 0 with dict, advanced streaming, 37995 +github.tar, level 1, advanced streaming, 39342 +github.tar, level 1 with dict, advanced streaming, 38309 +github.tar, level 3, advanced streaming, 38441 +github.tar, level 3 with dict, advanced streaming, 37995 +github.tar, level 4, advanced streaming, 38467 +github.tar, level 4 with dict, advanced streaming, 37948 +github.tar, level 5, advanced streaming, 39788 +github.tar, level 5 with dict, advanced streaming, 39715 +github.tar, level 6, advanced streaming, 39603 +github.tar, level 6 with dict, advanced streaming, 38800 +github.tar, level 7, advanced streaming, 39206 +github.tar, level 7 with dict, advanced streaming, 38071 +github.tar, level 9, advanced streaming, 36717 +github.tar, level 9 with dict, advanced streaming, 36898 +github.tar, level 13, advanced streaming, 35621 +github.tar, level 13 with dict, advanced streaming, 38726 +github.tar, level 16, advanced streaming, 40255 +github.tar, level 16 with dict, advanced streaming, 33639 +github.tar, level 19, advanced streaming, 32837 +github.tar, level 19 with dict, advanced streaming, 32895 +github.tar, no source size, advanced streaming, 38438 +github.tar, no source size with dict, advanced streaming, 214381 +github.tar, long distance mode, advanced streaming, 39676 +github.tar, multithreaded, advanced streaming, 38441 +github.tar, multithreaded long distance mode, advanced streaming, 39676 +github.tar, small window log, advanced streaming, 199558 +github.tar, small hash log, advanced streaming, 129870 +github.tar, small chain log, advanced streaming, 41669 +github.tar, explicit params, advanced streaming, 41199 +github.tar, uncompressed literals, advanced streaming, 41122 +github.tar, uncompressed literals optimal, advanced streaming, 35388 +github.tar, huffman literals, advanced streaming, 38800 +github.tar, multithreaded with advanced params, advanced streaming, 41122 silesia, level -5, old streaming, 6882505 silesia, level -3, old streaming, 6568376 silesia, level -1, old streaming, 6183403 @@ -511,9 +695,43 @@ github, level 16 with dict, old stre github, level 19, old streaming, 134064 github, level 19 with dict, old streaming, 37576 github, no source size, old streaming, 140632 +github, no source size with dict, old streaming, 40678 github, uncompressed literals, old streaming, 136335 github, uncompressed literals optimal, old streaming, 134064 github, huffman literals, old streaming, 175568 +github.tar, level -5, old streaming, 46747 +github.tar, level -5 with dict, old streaming, 43971 +github.tar, level -3, old streaming, 43537 +github.tar, level -3 with dict, old streaming, 40805 +github.tar, level -1, old streaming, 42465 +github.tar, level -1 with dict, old streaming, 41122 +github.tar, level 0, old streaming, 38441 +github.tar, level 0 with dict, old streaming, 37995 +github.tar, level 1, old streaming, 39342 +github.tar, level 1 with dict, old streaming, 38309 +github.tar, level 3, old streaming, 38441 +github.tar, level 3 with dict, old streaming, 37995 +github.tar, level 4, old streaming, 38467 +github.tar, level 4 with dict, old streaming, 37948 +github.tar, level 5, old streaming, 39788 +github.tar, level 5 with dict, old streaming, 39715 +github.tar, level 6, old streaming, 39603 +github.tar, level 6 with dict, old streaming, 38800 +github.tar, level 7, old streaming, 39206 +github.tar, level 7 with dict, old streaming, 38071 +github.tar, level 9, old streaming, 36717 +github.tar, level 9 with dict, old streaming, 36898 +github.tar, level 13, old streaming, 35621 +github.tar, level 13 with dict, old streaming, 38726 +github.tar, level 16, old streaming, 40255 +github.tar, level 16 with dict, old streaming, 33639 +github.tar, level 19, old streaming, 32837 +github.tar, level 19 with dict, old streaming, 32895 +github.tar, no source size, old streaming, 38438 +github.tar, no source size with dict, old streaming, 214384 +github.tar, uncompressed literals, old streaming, 38441 +github.tar, uncompressed literals optimal, old streaming, 32837 +github.tar, huffman literals, old streaming, 42465 silesia, level -5, old streaming advanced, 6882505 silesia, level -3, old streaming advanced, 6568376 silesia, level -1, old streaming advanced, 6183403 @@ -595,6 +813,7 @@ github, level 16 with dict, old stre github, level 19, old streaming advanced, 134064 github, level 19 with dict, old streaming advanced, 37576 github, no source size, old streaming advanced, 140632 +github, no source size with dict, old streaming advanced, 40643 github, long distance mode, old streaming advanced, 141104 github, multithreaded, old streaming advanced, 141104 github, multithreaded long distance mode, old streaming advanced, 141104 @@ -606,6 +825,47 @@ github, uncompressed literals, old stre github, uncompressed literals optimal, old streaming advanced, 134064 github, huffman literals, old streaming advanced, 181108 github, multithreaded with advanced params, old streaming advanced, 141104 +github.tar, level -5, old streaming advanced, 46747 +github.tar, level -5 with dict, old streaming advanced, 44824 +github.tar, level -3, old streaming advanced, 43537 +github.tar, level -3 with dict, old streaming advanced, 41800 +github.tar, level -1, old streaming advanced, 42465 +github.tar, level -1 with dict, old streaming advanced, 41471 +github.tar, level 0, old streaming advanced, 38441 +github.tar, level 0 with dict, old streaming advanced, 38013 +github.tar, level 1, old streaming advanced, 39342 +github.tar, level 1 with dict, old streaming advanced, 38940 +github.tar, level 3, old streaming advanced, 38441 +github.tar, level 3 with dict, old streaming advanced, 38013 +github.tar, level 4, old streaming advanced, 38467 +github.tar, level 4 with dict, old streaming advanced, 38063 +github.tar, level 5, old streaming advanced, 39788 +github.tar, level 5 with dict, old streaming advanced, 39310 +github.tar, level 6, old streaming advanced, 39603 +github.tar, level 6 with dict, old streaming advanced, 39279 +github.tar, level 7, old streaming advanced, 39206 +github.tar, level 7 with dict, old streaming advanced, 38728 +github.tar, level 9, old streaming advanced, 36717 +github.tar, level 9 with dict, old streaming advanced, 36504 +github.tar, level 13, old streaming advanced, 35621 +github.tar, level 13 with dict, old streaming advanced, 36035 +github.tar, level 16, old streaming advanced, 40255 +github.tar, level 16 with dict, old streaming advanced, 38736 +github.tar, level 19, old streaming advanced, 32837 +github.tar, level 19 with dict, old streaming advanced, 32876 +github.tar, no source size, old streaming advanced, 38438 +github.tar, no source size with dict, old streaming advanced, 214384 +github.tar, long distance mode, old streaming advanced, 38441 +github.tar, multithreaded, old streaming advanced, 38441 +github.tar, multithreaded long distance mode, old streaming advanced, 38441 +github.tar, small window log, old streaming advanced, 199561 +github.tar, small hash log, old streaming advanced, 129870 +github.tar, small chain log, old streaming advanced, 41669 +github.tar, explicit params, old streaming advanced, 41199 +github.tar, uncompressed literals, old streaming advanced, 38441 +github.tar, uncompressed literals optimal, old streaming advanced, 32837 +github.tar, huffman literals, old streaming advanced, 42465 +github.tar, multithreaded with advanced params, old streaming advanced, 38441 github, level -5 with dict, old streaming cdcit, 46718 github, level -3 with dict, old streaming cdcit, 45395 github, level -1 with dict, old streaming cdcit, 43170 @@ -620,6 +880,22 @@ github, level 9 with dict, old stre github, level 13 with dict, old streaming cdcit, 39743 github, level 16 with dict, old streaming cdcit, 37577 github, level 19 with dict, old streaming cdcit, 37576 +github, no source size with dict, old streaming cdcit, 40678 +github.tar, level -5 with dict, old streaming cdcit, 45018 +github.tar, level -3 with dict, old streaming cdcit, 41886 +github.tar, level -1 with dict, old streaming cdcit, 41636 +github.tar, level 0 with dict, old streaming cdcit, 37956 +github.tar, level 1 with dict, old streaming cdcit, 38766 +github.tar, level 3 with dict, old streaming cdcit, 37956 +github.tar, level 4 with dict, old streaming cdcit, 37927 +github.tar, level 5 with dict, old streaming cdcit, 39209 +github.tar, level 6 with dict, old streaming cdcit, 38983 +github.tar, level 7 with dict, old streaming cdcit, 38584 +github.tar, level 9 with dict, old streaming cdcit, 36363 +github.tar, level 13 with dict, old streaming cdcit, 36372 +github.tar, level 16 with dict, old streaming cdcit, 39353 +github.tar, level 19 with dict, old streaming cdcit, 32676 +github.tar, no source size with dict, old streaming cdcit, 214384 github, level -5 with dict, old streaming advanced cdict, 49562 github, level -3 with dict, old streaming advanced cdict, 44956 github, level -1 with dict, old streaming advanced cdict, 42383 @@ -634,3 +910,19 @@ github, level 9 with dict, old stre github, level 13 with dict, old streaming advanced cdict, 39731 github, level 16 with dict, old streaming advanced cdict, 40789 github, level 19 with dict, old streaming advanced cdict, 37576 +github, no source size with dict, old streaming advanced cdict, 40643 +github.tar, level -5 with dict, old streaming advanced cdict, 44307 +github.tar, level -3 with dict, old streaming advanced cdict, 41359 +github.tar, level -1 with dict, old streaming advanced cdict, 41322 +github.tar, level 0 with dict, old streaming advanced cdict, 38013 +github.tar, level 1 with dict, old streaming advanced cdict, 39002 +github.tar, level 3 with dict, old streaming advanced cdict, 38013 +github.tar, level 4 with dict, old streaming advanced cdict, 38063 +github.tar, level 5 with dict, old streaming advanced cdict, 39310 +github.tar, level 6 with dict, old streaming advanced cdict, 39279 +github.tar, level 7 with dict, old streaming advanced cdict, 38728 +github.tar, level 9 with dict, old streaming advanced cdict, 36504 +github.tar, level 13 with dict, old streaming advanced cdict, 36035 +github.tar, level 16 with dict, old streaming advanced cdict, 38736 +github.tar, level 19 with dict, old streaming advanced cdict, 32876 +github.tar, no source size with dict, old streaming advanced cdict, 214384