lib/zstd
1
0
Fork 0
mirror of https://github.com/facebook/zstd.git synced 2025-08-01 09:47:01 +03:00
Code Activity

test-zstd-versions.py: create and test dictionaries for v0.5.0

Browse Source
This commit is contained in:
inikep
2016-06-17 14:43:24 +02:00
parent 2ef16501ca
commit 7e3597bf38
1 changed files with 30 additions and 16 deletions
Download Patch File Download Diff File Expand all files Collapse all files

44
tests/test-zstd-versions.py
View File

@ -9,7 +9,7 @@ import os
import shutil import shutil
import sys import sys
import subprocess import subprocess
from subprocess import Popen, PIPE, DEVNULL from subprocess import Popen, PIPE
repo_url = 'https://github.com/Cyan4973/zstd.git' repo_url = 'https://github.com/Cyan4973/zstd.git'
tmp_dir_name = 'tests/versionsTest' tmp_dir_name = 'tests/versionsTest'
@ -66,7 +66,11 @@ def create_dict(tag, dict_source_path):
if not os.path.isfile(dict_name): if not os.path.isfile(dict_name):
cFiles = glob.glob(dict_source_path + "/*.c") cFiles = glob.glob(dict_source_path + "/*.c")
hFiles = glob.glob(dict_source_path + "/*.h") hFiles = glob.glob(dict_source_path + "/*.h")
if execute('./zstd.' + tag + ' -f --train ' + ' '.join(cFiles) + ' ' + ' '.join(hFiles) + ' -o ' + dict_name, print_output=False, param_shell=True) == 0: if tag == 'v0.5.0':
result = execute('./dictBuilder.' + tag + ' ' + ' '.join(cFiles) + ' ' + ' '.join(hFiles) + ' -o ' + dict_name, print_output=False, param_shell=True)
else:
result = execute('./zstd.' + tag + ' ' + ' '.join(cFiles) + ' ' + ' '.join(hFiles) + ' -o ' + dict_name, print_output=False, param_shell=True)
if result == 0:
print(dict_name + ' created') print(dict_name + ' created')
else: else:
print('ERROR: creating of ' + dict_name + ' failed') print('ERROR: creating of ' + dict_name + ' failed')
@ -76,17 +80,24 @@ def create_dict(tag, dict_source_path):
def dict_compress_sample(tag, sample): def dict_compress_sample(tag, sample):
dict_name = 'dict.' + tag dict_name = 'dict.' + tag
subprocess.call(['./zstd.' + tag, '-D', dict_name, '-f', sample, '-o', sample + '_01_64_' + tag + '_dictio.zst'], stderr=DEVNULL) DEVNULL = open(os.devnull, 'wb')
subprocess.call(['./zstd.' + tag, '-D', dict_name, '-5f', sample, '-o', sample + '_05_64_' + tag + '_dictio.zst'], stderr=DEVNULL) if subprocess.call(['./zstd.' + tag, '-D', dict_name, '-f', sample], stderr=DEVNULL) == 0:
subprocess.call(['./zstd.' + tag, '-D', dict_name, '-9f', sample, '-o', sample + '_09_64_' + tag + '_dictio.zst'], stderr=DEVNULL) os.rename(sample + '.zst', sample + '_01_64_' + tag + '_dictio.zst')
subprocess.call(['./zstd.' + tag, '-D', dict_name, '-15f', sample, '-o', sample + '_15_64_' + tag + '_dictio.zst'], stderr=DEVNULL) if subprocess.call(['./zstd.' + tag, '-D', dict_name, '-5f', sample], stderr=DEVNULL) == 0:
subprocess.call(['./zstd.' + tag, '-D', dict_name, '-18f', sample, '-o', sample + '_18_64_' + tag + '_dictio.zst'], stderr=DEVNULL) os.rename(sample + '.zst', sample + '_05_64_' + tag + '_dictio.zst')
if subprocess.call(['./zstd.' + tag, '-D', dict_name, '-9f', sample], stderr=DEVNULL) == 0:
os.rename(sample + '.zst', sample + '_09_64_' + tag + '_dictio.zst')
if subprocess.call(['./zstd.' + tag, '-D', dict_name, '-15f', sample], stderr=DEVNULL) == 0:
os.rename(sample + '.zst', sample + '_15_64_' + tag + '_dictio.zst')
if subprocess.call(['./zstd.' + tag, '-D', dict_name, '-18f', sample], stderr=DEVNULL) == 0:
os.rename(sample + '.zst', sample + '_18_64_' + tag + '_dictio.zst')
# zstdFiles = glob.glob("*.zst*") # zstdFiles = glob.glob("*.zst*")
# print(zstdFiles) # print(zstdFiles)
print(tag + " : dict compression completed") print(tag + " : dict compression completed")
def compress_sample(tag, sample): def compress_sample(tag, sample):
DEVNULL = open(os.devnull, 'wb')
if subprocess.call(['./zstd.' + tag, '-f', sample], stderr=DEVNULL) == 0: if subprocess.call(['./zstd.' + tag, '-f', sample], stderr=DEVNULL) == 0:
os.rename(sample + '.zst', sample + '_01_64_' + tag + '_nodict.zst') os.rename(sample + '.zst', sample + '_01_64_' + tag + '_nodict.zst')
if subprocess.call(['./zstd.' + tag, '-5f', sample], stderr=DEVNULL) == 0: if subprocess.call(['./zstd.' + tag, '-5f', sample], stderr=DEVNULL) == 0:
@ -122,14 +133,14 @@ def remove_duplicates():
print('duplicated : {} == {}'.format(ref_zst, compared_zst)) print('duplicated : {} == {}'.format(ref_zst, compared_zst))
def decompress_zst(tag, zstd_up_to_v05=False): def decompress_zst(tag):
dec_error = 0 dec_error = 0
list_zst = sorted(glob.glob('*_nodict.zst')) list_zst = sorted(glob.glob('*_nodict.zst'))
for file_zst in list_zst: for file_zst in list_zst:
print(file_zst, end=' ') print(file_zst, end=' ')
print(tag, end=' ') print(tag, end=' ')
file_dec = file_zst + '_d64_' + tag + '.dec' file_dec = file_zst + '_d64_' + tag + '.dec'
if zstd_up_to_v05: if tag <= 'v0.5.0':
params = ['./zstd.' + tag, '-df', file_zst, file_dec] params = ['./zstd.' + tag, '-df', file_zst, file_dec]
else: else:
params = ['./zstd.' + tag, '-df', file_zst, '-o', file_dec] params = ['./zstd.' + tag, '-df', file_zst, '-o', file_dec]
@ -144,7 +155,7 @@ def decompress_zst(tag, zstd_up_to_v05=False):
return dec_error return dec_error
def decompress_dict(tag, zstd_up_to_v05=False): def decompress_dict(tag):
dec_error = 0 dec_error = 0
list_zst = sorted(glob.glob('*_dictio.zst')) list_zst = sorted(glob.glob('*_dictio.zst'))
for file_zst in list_zst: for file_zst in list_zst:
@ -153,10 +164,12 @@ def decompress_dict(tag, zstd_up_to_v05=False):
dict_tag = head dict_tag = head
else: else:
dict_tag = dict_tag[dict_tag.rfind('v'):] dict_tag = dict_tag[dict_tag.rfind('v'):]
if tag == 'v0.6.0' and dict_tag < 'v0.6.0':
continue
dict_name = 'dict.' + dict_tag dict_name = 'dict.' + dict_tag
print(file_zst + ' ' + tag + ' dict=' + dict_tag, end=' ') print(file_zst + ' ' + tag + ' dict=' + dict_tag, end=' ')
file_dec = file_zst + '_d64_' + tag + '.dec' file_dec = file_zst + '_d64_' + tag + '.dec'
if zstd_up_to_v05: if tag <= 'v0.5.0':
params = ['./zstd.' + tag, '-D', dict_name, '-df', file_zst, file_dec] params = ['./zstd.' + tag, '-D', dict_name, '-df', file_zst, file_dec]
else: else:
params = ['./zstd.' + tag, '-D', dict_name, '-df', file_zst, '-o', file_dec] params = ['./zstd.' + tag, '-D', dict_name, '-df', file_zst, '-o', file_dec]
@ -203,6 +216,10 @@ if __name__ == '__main__':
os.makedirs(r_dir, exist_ok=True) os.makedirs(r_dir, exist_ok=True)
os.chdir(clone_dir) os.chdir(clone_dir)
git(['--work-tree=' + r_dir, 'checkout', tag, '--', '.'], False) git(['--work-tree=' + r_dir, 'checkout', tag, '--', '.'], False)
if tag == 'v0.5.0':
os.chdir(r_dir + '/dictBuilder') # /path/to/zstd/tests/versionsTest/v0.5.0/dictBuilder
make(['clean', 'dictBuilder'], False)
shutil.copy2('dictBuilder', '{}/dictBuilder.{}'.format(tmp_dir, tag))
os.chdir(r_dir + '/programs') # /path/to/zstd/tests/versionsTest/<TAG>/programs os.chdir(r_dir + '/programs') # /path/to/zstd/tests/versionsTest/<TAG>/programs
make(['clean', 'zstd'], False) make(['clean', 'zstd'], False)
else: else:
@ -228,17 +245,14 @@ if __name__ == '__main__':
error_code = 0 error_code = 0
for tag in tags: for tag in tags:
print(tag) print(tag)
if tag >= 'v0.5.1': if tag >= 'v0.5.0':
create_dict(tag, dict_source_path) create_dict(tag, dict_source_path)
dict_compress_sample(tag, test_dat) dict_compress_sample(tag, test_dat)
remove_duplicates() remove_duplicates()
error_code += decompress_dict(tag) error_code += decompress_dict(tag)
compress_sample(tag, test_dat) compress_sample(tag, test_dat)
remove_duplicates() remove_duplicates()
if tag >= 'v0.5.1':
error_code += decompress_zst(tag) error_code += decompress_zst(tag)
else:
error_code += decompress_zst(tag, zstd_up_to_v05=True)
print('') print('')
print('Enumerate different compressed files') print('Enumerate different compressed files')