lib/zstd
1
0
Fork 0
mirror of https://github.com/facebook/zstd.git synced 2025-08-08 17:22:10 +03:00
Code Activity

Test and tidy

Browse Source 
Made the Python more Python-like. Added notes and general tidy. Tested exclusions and building with various options. Tested all scripts.
This commit is contained in:
Carl Woffenden
2022-01-19 11:32:53 +01:00
parent 8f1e51f99f
commit 7d90f0b520
7 changed files with 133 additions and 118 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
build/single_file_libs/build_library_test.sh
View File

@@ -69,7 +69,7 @@ fi
echo "Single file library creation script: PASSED" echo "Single file library creation script: PASSED"
# Copy the header to here (for the tests) # Copy the header to here (for the tests)
cp "$ZSTD_SRC_ROOT/zstd.h" zstd.h cp "$ZSTD_SRC_ROOT/zstd.h" examples/zstd.h
# Compile the generated output # Compile the generated output
cc -Wall -Wextra -Werror -Wshadow -pthread -I. -Os -g0 -o $OUT_FILE zstd.c examples/roundtrip.c cc -Wall -Wextra -Werror -Wshadow -pthread -I. -Os -g0 -o $OUT_FILE zstd.c examples/roundtrip.c

232
build/single_file_libs/combine.py
View File

@@ -2,6 +2,18 @@
# Tool to bundle multiple C/C++ source files, inlining any includes. # Tool to bundle multiple C/C++ source files, inlining any includes.
# #
# Note: there are two types of exclusion options: the '-x' flag, which besides
# excluding a file also adds an #error directive in place of the #include, and
# the '-k' flag, which keeps the #include and doesn't inline the file. The
# intended use cases are: '-x' for files that would normally be #if'd out, so
# features that 100% won't be used in the amalgamated file, for which every
# occurrence adds the error, and '-k' for headers that we wish to manually
# include, such as a project's public API, for which occurrences after the first
# are removed.
#
# Todo: the error handling could be better, which currently throws and halts
# (which is functional just not very friendly).
#
# Author: Carl Woffenden, Numfum GmbH (this script is released under a CC0 license/Public Domain) # Author: Carl Woffenden, Numfum GmbH (this script is released under a CC0 license/Public Domain)
import argparse, re, sys import argparse, re, sys
@@ -31,18 +43,18 @@ found: Set[Path] = set()
# Compiled regex Patern to handle the following type of file includes: # Compiled regex Patern to handle the following type of file includes:
# #
# #include "file" # #include "file"
# #include "file" # #include "file"
# # include "file" # # include "file"
# #include "file" # #include "file"
# #include "file" // comment # #include "file" // comment
# #include "file" // comment with quote " # #include "file" // comment with quote "
# #
# And all combinations of, as well as ignoring the following: # And all combinations of, as well as ignoring the following:
# #
# #include <file> # #include <file>
# //#include "file" # //#include "file"
# /*#include "file"*/ # /*#include "file"*/
# #
# We don't try to catch errors since the compiler will do this (and the code is # We don't try to catch errors since the compiler will do this (and the code is
# expected to be valid before processing) and we don't care what follows the # expected to be valid before processing) and we don't care what follows the
@@ -54,27 +66,27 @@ include_regex: Pattern = re.compile(r'^\s*#\s*include\s*"(.+?)"')
# Simple tests to prove include_regex's cases. # Simple tests to prove include_regex's cases.
# #
def test_match_include() -> bool: def test_match_include() -> bool:
if (include_regex.match('#include "file"') and if (include_regex.match('#include "file"') and
include_regex.match(' #include "file"') and include_regex.match(' #include "file"') and
include_regex.match('# include "file"') and include_regex.match('# include "file"') and
include_regex.match('#include "file"') and include_regex.match('#include "file"') and
include_regex.match('#include "file" // comment')): include_regex.match('#include "file" // comment')):
if (not include_regex.match('#include <file>') and if (not include_regex.match('#include <file>') and
not include_regex.match('//#include "file"') and not include_regex.match('//#include "file"') and
not include_regex.match('/*#include "file"*/')): not include_regex.match('/*#include "file"*/')):
found = include_regex.match('#include "file" // "') found = include_regex.match('#include "file" // "')
if (found and found.group(1) == 'file'): if (found and found.group(1) == 'file'):
print('#include match valid') print('#include match valid')
return True return True
return False return False
# Compiled regex Patern to handle "#pragma once" in various formats: # Compiled regex Patern to handle "#pragma once" in various formats:
# #
# #pragma once # #pragma once
# #pragma once # #pragma once
# # pragma once # # pragma once
# #pragma once # #pragma once
# #pragma once // comment # #pragma once // comment
# #
# Ignoring commented versions, same as include_regex. # Ignoring commented versions, same as include_regex.
# #
@@ -83,103 +95,105 @@ pragma_regex: Pattern = re.compile(r'^\s*#\s*pragma\s*once\s*')
# Simple tests to prove pragma_regex's cases. # Simple tests to prove pragma_regex's cases.
# #
def text_match_pragma() -> bool: def text_match_pragma() -> bool:
if (pragma_regex.match('#pragma once') and if (pragma_regex.match('#pragma once') and
pragma_regex.match(' #pragma once') and pragma_regex.match(' #pragma once') and
pragma_regex.match('# pragma once') and pragma_regex.match('# pragma once') and
pragma_regex.match('#pragma once') and pragma_regex.match('#pragma once') and
pragma_regex.match('#pragma once // comment')): pragma_regex.match('#pragma once // comment')):
if (not pragma_regex.match('//#pragma once') and if (not pragma_regex.match('//#pragma once') and
not pragma_regex.match('/*#pragma once*/')): not pragma_regex.match('/*#pragma once*/')):
print('#pragma once match valid') print('#pragma once match valid')
return True return True
return False return False
# Finds 'file'. First the currently processing file's 'parent' path is looked at # Finds 'file'. First the currently processing file's 'parent' path is looked at
# for a match, followed by the list of 'root' paths, returning a valid Path in # for a match, followed by the list of 'root' paths, returning a valid Path in
# canonical form. If no match is found None is returned. # canonical form. If no match is found None is returned.
# #
def resolve_include(file: str, parent: Optional[Path] = None) -> Optional[Path]: def resolve_include(file: str, parent: Optional[Path] = None) -> Optional[Path]:
if (parent): if (parent):
found = parent.joinpath(file).resolve(); found = parent.joinpath(file).resolve();
else: else:
found = Path(file) found = Path(file)
if (found.is_file()): if (found.is_file()):
return found return found
for root in roots: for root in roots:
found = root.joinpath(file).resolve() found = root.joinpath(file).resolve()
if (found.is_file()): if (found.is_file()):
return found return found
return None return None
# Helper to resolve lists of files. 'file_list' is passed in from the arguments # Helper to resolve lists of files. 'file_list' is passed in from the arguments
# and each entry resolved to its canonical path (like any include entry, either # and each entry resolved to its canonical path (like any include entry, either
# from the list of root paths or the owning file's 'parent', which in this case # from the list of root paths or the owning file's 'parent', which in this case
# is case is the input file). The results are stored in 'resolved'. # is case is the input file). The results are stored in 'resolved'.
# #
def resolve_files(file_list: Optional[List[str]], resolved: Set[Path], parent: Optional[Path] = None) -> None: def resolve_excluded_files(file_list: Optional[List[str]], resolved: Set[Path], parent: Optional[Path] = None) -> None:
if (file_list): if (file_list):
for filename in file_list: for filename in file_list:
found = resolve_include(filename, parent) found = resolve_include(filename, parent)
if (found): if (found):
resolved.add(found) resolved.add(found)
else: else:
error_line(f'Warning: excluded file not found: {filename}') error_line(f'Warning: excluded file not found: {filename}')
# Writes 'line' to the open 'destn' (or stdout). # Writes 'line' to the open 'destn' (or stdout).
# #
def write_line(line: str) -> None: def write_line(line: str) -> None:
print(line, file=destn) print(line, file=destn)
# Logs 'line' to stderr. This is also used for general notifications that we # Logs 'line' to stderr. This is also used for general notifications that we
# don't want to go to stdout (so the source can be piped). # don't want to go to stdout (so the source can be piped).
# #
def error_line(line: Any) -> None: def error_line(line: Any) -> None:
print(line, file=sys.stderr) print(line, file=sys.stderr)
# Inline the contents of 'file' (with any of its includes also inlined, etc.). # Inline the contents of 'file' (with any of its includes also inlined, etc.).
# #
def add_file(file: Path) -> None: def add_file(file: Path, file_name: str = None) -> None:
if (file.is_file()): if (file.is_file()):
error_line(f'Processing: {file}') if (not file_name):
with file.open('r') as opened: file_name = file.name
for line in opened: error_line(f'Processing: {file_name}')
line = line.rstrip('\n') with file.open('r') as opened:
match_include = include_regex.match(line); for line in opened:
if (match_include): line = line.rstrip('\n')
# We have a quoted include directive so grab the file match_include = include_regex.match(line);
inc_name = match_include.group(1) if (match_include):
resolved = resolve_include(inc_name, file.parent) # We have a quoted include directive so grab the file
if (resolved): inc_name = match_include.group(1)
if (resolved in excludes): resolved = resolve_include(inc_name, file.parent)
# The file was excluded so error if the source attempts to use it if (resolved):
write_line(f'#error Using excluded file: {inc_name}') if (resolved in excludes):
error_line(f'Excluding: {inc_name}') # The file was excluded so error if the compiler uses it
else: write_line(f'#error Using excluded file: {inc_name}')
if (resolved not in found): error_line(f'Excluding: {inc_name}')
# The file was not previously encountered else:
found.add(resolved) if (resolved not in found):
if (resolved in keeps): # The file was not previously encountered
# But the include was flagged to keep as included found.add(resolved)
write_line(f'/**** *NOT* inlining {inc_name} ****/') if (resolved in keeps):
write_line(line) # But the include was flagged to keep as included
error_line('Not Inlining: {inc_name}') write_line(f'/**** *NOT* inlining {inc_name} ****/')
else: write_line(line)
# The file was neither excluded nor seen before so inline it error_line(f'Not inlining: {inc_name}')
write_line(f'/**** start inlining {inc_name} ****/') else:
add_file(resolved) # The file was neither excluded nor seen before so inline it
write_line(f'/**** ended inlining {inc_name} ****/') write_line(f'/**** start inlining {inc_name} ****/')
else: add_file(resolved, inc_name)
write_line(f'/**** skipping file: {inc_name} ****/') write_line(f'/**** ended inlining {inc_name} ****/')
else: else:
# The include file didn't resolve to a file write_line(f'/**** skipping file: {inc_name} ****/')
write_line(f'#error Unable to find: {inc_name}') else:
error_line(f'Error: Unable to find: {inc_name}') # The include file didn't resolve to a file
else: write_line(f'#error Unable to find: {inc_name}')
# Skip any 'pragma once' directives, otherwise write the source line error_line(f'Error: Unable to find: {inc_name}')
if (keep_pragma or not pragma_regex.match(line)): else:
write_line(line) # Skip any 'pragma once' directives, otherwise write the source line
else: if (keep_pragma or not pragma_regex.match(line)):
error_line(f'Error: Invalid file: {file}') write_line(line)
else:
error_line(f'Error: Invalid file: {file}')
# Start here # Start here
parser = argparse.ArgumentParser(description='Amalgamate Tool', epilog=f'example: {sys.argv[0]} -r ../my/path -r ../other/path -o out.c in.c') parser = argparse.ArgumentParser(description='Amalgamate Tool', epilog=f'example: {sys.argv[0]} -r ../my/path -r ../other/path -o out.c in.c')
@@ -197,19 +211,19 @@ found.add(args.input)
# Resolve all of the root paths upfront (we'll halt here on invalid roots) # Resolve all of the root paths upfront (we'll halt here on invalid roots)
if (args.root): if (args.root):
for path in args.root: for path in args.root:
roots.add(path.resolve(strict=True)) roots.add(path.resolve(strict=True))
# The remaining params: so resolve the excluded files and #pragma once directive # The remaining params: so resolve the excluded files and #pragma once directive
resolve_files(args.exclude, excludes, args.input.parent) resolve_excluded_files(args.exclude, excludes, args.input.parent)
resolve_files(args.keep, keeps, args.input.parent) resolve_excluded_files(args.keep, keeps, args.input.parent)
keep_pragma = args.pragma; keep_pragma = args.pragma;
# Then recursively process the input file # Then recursively process the input file
try: try:
if (args.output): if (args.output):
destn = args.output destn = args.output
add_file(args.input) add_file(args.input)
finally: finally:
if (not destn): if (not destn):
destn.close() destn.close()

1
build/single_file_libs/combine.sh
View File

@@ -200,6 +200,7 @@ if [ -n "$1" ]; then
printf "" > "$DESTN" printf "" > "$DESTN"
fi fi
test_deps test_deps
log_line "Processing using the slower shell script; this might take a while"
add_file "$1" add_file "$1"
else else
echo "Input file not found: \"$1\"" echo "Input file not found: \"$1\""

6
build/single_file_libs/create_single_file_decoder.sh
View File

@@ -4,12 +4,12 @@
ZSTD_SRC_ROOT="../../lib" ZSTD_SRC_ROOT="../../lib"
# Amalgamate the sources # Amalgamate the sources
echo "Amalgamating files... this can take a while" echo "Amalgamating files..."
# Using the faster Python script if we have 3.8 or higher # Using the faster Python script if we have 3.8 or higher
if python3 -c 'import sys; assert sys.version_info >= (3,8)' 2>/dev/null; then if python3 -c 'import sys; assert sys.version_info >= (3,8)' 2>/dev/null; then
./combine.py -r "$ZSTD_SRC_ROOT" -o zstddeclib.c zstddeclib-in.c ./combine.py -r "$ZSTD_SRC_ROOT" -x legacy/zstd_legacy.h -o zstddeclib.c zstddeclib-in.c
else else
./combine.sh -r "$ZSTD_SRC_ROOT" -o zstddeclib.c zstddeclib-in.c ./combine.sh -r "$ZSTD_SRC_ROOT" -x legacy/zstd_legacy.h -o zstddeclib.c zstddeclib-in.c
fi fi
# Did combining work? # Did combining work?
if [ $? -ne 0 ]; then if [ $? -ne 0 ]; then

6
build/single_file_libs/create_single_file_library.sh
View File

@@ -4,12 +4,12 @@
ZSTD_SRC_ROOT="../../lib" ZSTD_SRC_ROOT="../../lib"
# Amalgamate the sources # Amalgamate the sources
echo "Amalgamating files... this can take a while" echo "Amalgamating files..."
# Using the faster Python script if we have 3.8 or higher # Using the faster Python script if we have 3.8 or higher
if python3 -c 'import sys; assert sys.version_info >= (3,8)' 2>/dev/null; then if python3 -c 'import sys; assert sys.version_info >= (3,8)' 2>/dev/null; then
./combine.py -r "$ZSTD_SRC_ROOT" -o zstd.c zstd-in.c ./combine.py -r "$ZSTD_SRC_ROOT" -x legacy/zstd_legacy.h -o zstd.c zstd-in.c
else else
./combine.sh -r "$ZSTD_SRC_ROOT" -o zstd.c zstd-in.c ./combine.sh -r "$ZSTD_SRC_ROOT" -x legacy/zstd_legacy.h -o zstd.c zstd-in.c
fi fi
# Did combining work? # Did combining work?
if [ $? -ne 0 ]; then if [ $? -ne 0 ]; then

2
build/single_file_libs/zstd-in.c
View File

@@ -4,7 +4,7 @@
* *
* Generate using: * Generate using:
* \code * \code
* combine.sh -r ../../lib -o zstd.c zstd-in.c * combine.sh -r ../../lib -x legacy/zstd_legacy.h -o zstd.c zstd-in.c
* \endcode * \endcode
*/ */
/* /*

2
build/single_file_libs/zstddeclib-in.c
View File

@@ -4,7 +4,7 @@
* *
* Generate using: * Generate using:
* \code * \code
* combine.sh -r ../../lib -o zstddeclib.c zstddeclib-in.c * combine.sh -r ../../lib -x legacy/zstd_legacy.h -o zstddeclib.c zstddeclib-in.c
* \endcode * \endcode
*/ */
/* /*