diff --git a/Makefile b/Makefile index c80b20f39..93d5e0528 100644 --- a/Makefile +++ b/Makefile @@ -35,7 +35,6 @@ PRGDIR = programs ZSTDDIR = lib -DICTDIR = dictBuilder # Define nul output ifneq (,$(filter Windows%,$(OS))) @@ -51,7 +50,6 @@ default: zstdprogram all: $(MAKE) -C $(ZSTDDIR) $@ $(MAKE) -C $(PRGDIR) $@ - $(MAKE) -C $(DICTDIR) $@ zstdprogram: $(MAKE) -C $(PRGDIR) @@ -59,7 +57,6 @@ zstdprogram: clean: @$(MAKE) -C $(ZSTDDIR) $@ > $(VOID) @$(MAKE) -C $(PRGDIR) $@ > $(VOID) - @$(MAKE) -C $(DICTDIR) $@ > $(VOID) @echo Cleaning completed @@ -80,7 +77,6 @@ travis-install: test: $(MAKE) -C $(PRGDIR) $@ - $(MAKE) -C $(DICTDIR) $@ cmaketest: cd contrib/cmake ; cmake . ; $(MAKE) diff --git a/dictBuilder/COPYING b/dictBuilder/COPYING deleted file mode 100644 index d159169d1..000000000 --- a/dictBuilder/COPYING +++ /dev/null @@ -1,339 +0,0 @@ - GNU GENERAL PUBLIC LICENSE - Version 2, June 1991 - - Copyright (C) 1989, 1991 Free Software Foundation, Inc., - 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA - Everyone is permitted to copy and distribute verbatim copies - of this license document, but changing it is not allowed. - - Preamble - - The licenses for most software are designed to take away your -freedom to share and change it. By contrast, the GNU General Public -License is intended to guarantee your freedom to share and change free -software--to make sure the software is free for all its users. This -General Public License applies to most of the Free Software -Foundation's software and to any other program whose authors commit to -using it. (Some other Free Software Foundation software is covered by -the GNU Lesser General Public License instead.) You can apply it to -your programs, too. - - When we speak of free software, we are referring to freedom, not -price. Our General Public Licenses are designed to make sure that you -have the freedom to distribute copies of free software (and charge for -this service if you wish), that you receive source code or can get it -if you want it, that you can change the software or use pieces of it -in new free programs; and that you know you can do these things. - - To protect your rights, we need to make restrictions that forbid -anyone to deny you these rights or to ask you to surrender the rights. -These restrictions translate to certain responsibilities for you if you -distribute copies of the software, or if you modify it. - - For example, if you distribute copies of such a program, whether -gratis or for a fee, you must give the recipients all the rights that -you have. You must make sure that they, too, receive or can get the -source code. And you must show them these terms so they know their -rights. - - We protect your rights with two steps: (1) copyright the software, and -(2) offer you this license which gives you legal permission to copy, -distribute and/or modify the software. - - Also, for each author's protection and ours, we want to make certain -that everyone understands that there is no warranty for this free -software. If the software is modified by someone else and passed on, we -want its recipients to know that what they have is not the original, so -that any problems introduced by others will not reflect on the original -authors' reputations. - - Finally, any free program is threatened constantly by software -patents. We wish to avoid the danger that redistributors of a free -program will individually obtain patent licenses, in effect making the -program proprietary. To prevent this, we have made it clear that any -patent must be licensed for everyone's free use or not licensed at all. - - The precise terms and conditions for copying, distribution and -modification follow. - - GNU GENERAL PUBLIC LICENSE - TERMS AND CONDITIONS FOR COPYING, DISTRIBUTION AND MODIFICATION - - 0. This License applies to any program or other work which contains -a notice placed by the copyright holder saying it may be distributed -under the terms of this General Public License. The "Program", below, -refers to any such program or work, and a "work based on the Program" -means either the Program or any derivative work under copyright law: -that is to say, a work containing the Program or a portion of it, -either verbatim or with modifications and/or translated into another -language. (Hereinafter, translation is included without limitation in -the term "modification".) Each licensee is addressed as "you". - -Activities other than copying, distribution and modification are not -covered by this License; they are outside its scope. The act of -running the Program is not restricted, and the output from the Program -is covered only if its contents constitute a work based on the -Program (independent of having been made by running the Program). -Whether that is true depends on what the Program does. - - 1. You may copy and distribute verbatim copies of the Program's -source code as you receive it, in any medium, provided that you -conspicuously and appropriately publish on each copy an appropriate -copyright notice and disclaimer of warranty; keep intact all the -notices that refer to this License and to the absence of any warranty; -and give any other recipients of the Program a copy of this License -along with the Program. - -You may charge a fee for the physical act of transferring a copy, and -you may at your option offer warranty protection in exchange for a fee. - - 2. You may modify your copy or copies of the Program or any portion -of it, thus forming a work based on the Program, and copy and -distribute such modifications or work under the terms of Section 1 -above, provided that you also meet all of these conditions: - - a) You must cause the modified files to carry prominent notices - stating that you changed the files and the date of any change. - - b) You must cause any work that you distribute or publish, that in - whole or in part contains or is derived from the Program or any - part thereof, to be licensed as a whole at no charge to all third - parties under the terms of this License. - - c) If the modified program normally reads commands interactively - when run, you must cause it, when started running for such - interactive use in the most ordinary way, to print or display an - announcement including an appropriate copyright notice and a - notice that there is no warranty (or else, saying that you provide - a warranty) and that users may redistribute the program under - these conditions, and telling the user how to view a copy of this - License. (Exception: if the Program itself is interactive but - does not normally print such an announcement, your work based on - the Program is not required to print an announcement.) - -These requirements apply to the modified work as a whole. If -identifiable sections of that work are not derived from the Program, -and can be reasonably considered independent and separate works in -themselves, then this License, and its terms, do not apply to those -sections when you distribute them as separate works. But when you -distribute the same sections as part of a whole which is a work based -on the Program, the distribution of the whole must be on the terms of -this License, whose permissions for other licensees extend to the -entire whole, and thus to each and every part regardless of who wrote it. - -Thus, it is not the intent of this section to claim rights or contest -your rights to work written entirely by you; rather, the intent is to -exercise the right to control the distribution of derivative or -collective works based on the Program. - -In addition, mere aggregation of another work not based on the Program -with the Program (or with a work based on the Program) on a volume of -a storage or distribution medium does not bring the other work under -the scope of this License. - - 3. You may copy and distribute the Program (or a work based on it, -under Section 2) in object code or executable form under the terms of -Sections 1 and 2 above provided that you also do one of the following: - - a) Accompany it with the complete corresponding machine-readable - source code, which must be distributed under the terms of Sections - 1 and 2 above on a medium customarily used for software interchange; or, - - b) Accompany it with a written offer, valid for at least three - years, to give any third party, for a charge no more than your - cost of physically performing source distribution, a complete - machine-readable copy of the corresponding source code, to be - distributed under the terms of Sections 1 and 2 above on a medium - customarily used for software interchange; or, - - c) Accompany it with the information you received as to the offer - to distribute corresponding source code. (This alternative is - allowed only for noncommercial distribution and only if you - received the program in object code or executable form with such - an offer, in accord with Subsection b above.) - -The source code for a work means the preferred form of the work for -making modifications to it. For an executable work, complete source -code means all the source code for all modules it contains, plus any -associated interface definition files, plus the scripts used to -control compilation and installation of the executable. However, as a -special exception, the source code distributed need not include -anything that is normally distributed (in either source or binary -form) with the major components (compiler, kernel, and so on) of the -operating system on which the executable runs, unless that component -itself accompanies the executable. - -If distribution of executable or object code is made by offering -access to copy from a designated place, then offering equivalent -access to copy the source code from the same place counts as -distribution of the source code, even though third parties are not -compelled to copy the source along with the object code. - - 4. You may not copy, modify, sublicense, or distribute the Program -except as expressly provided under this License. Any attempt -otherwise to copy, modify, sublicense or distribute the Program is -void, and will automatically terminate your rights under this License. -However, parties who have received copies, or rights, from you under -this License will not have their licenses terminated so long as such -parties remain in full compliance. - - 5. You are not required to accept this License, since you have not -signed it. However, nothing else grants you permission to modify or -distribute the Program or its derivative works. These actions are -prohibited by law if you do not accept this License. Therefore, by -modifying or distributing the Program (or any work based on the -Program), you indicate your acceptance of this License to do so, and -all its terms and conditions for copying, distributing or modifying -the Program or works based on it. - - 6. Each time you redistribute the Program (or any work based on the -Program), the recipient automatically receives a license from the -original licensor to copy, distribute or modify the Program subject to -these terms and conditions. You may not impose any further -restrictions on the recipients' exercise of the rights granted herein. -You are not responsible for enforcing compliance by third parties to -this License. - - 7. If, as a consequence of a court judgment or allegation of patent -infringement or for any other reason (not limited to patent issues), -conditions are imposed on you (whether by court order, agreement or -otherwise) that contradict the conditions of this License, they do not -excuse you from the conditions of this License. If you cannot -distribute so as to satisfy simultaneously your obligations under this -License and any other pertinent obligations, then as a consequence you -may not distribute the Program at all. For example, if a patent -license would not permit royalty-free redistribution of the Program by -all those who receive copies directly or indirectly through you, then -the only way you could satisfy both it and this License would be to -refrain entirely from distribution of the Program. - -If any portion of this section is held invalid or unenforceable under -any particular circumstance, the balance of the section is intended to -apply and the section as a whole is intended to apply in other -circumstances. - -It is not the purpose of this section to induce you to infringe any -patents or other property right claims or to contest validity of any -such claims; this section has the sole purpose of protecting the -integrity of the free software distribution system, which is -implemented by public license practices. Many people have made -generous contributions to the wide range of software distributed -through that system in reliance on consistent application of that -system; it is up to the author/donor to decide if he or she is willing -to distribute software through any other system and a licensee cannot -impose that choice. - -This section is intended to make thoroughly clear what is believed to -be a consequence of the rest of this License. - - 8. If the distribution and/or use of the Program is restricted in -certain countries either by patents or by copyrighted interfaces, the -original copyright holder who places the Program under this License -may add an explicit geographical distribution limitation excluding -those countries, so that distribution is permitted only in or among -countries not thus excluded. In such case, this License incorporates -the limitation as if written in the body of this License. - - 9. The Free Software Foundation may publish revised and/or new versions -of the General Public License from time to time. Such new versions will -be similar in spirit to the present version, but may differ in detail to -address new problems or concerns. - -Each version is given a distinguishing version number. If the Program -specifies a version number of this License which applies to it and "any -later version", you have the option of following the terms and conditions -either of that version or of any later version published by the Free -Software Foundation. If the Program does not specify a version number of -this License, you may choose any version ever published by the Free Software -Foundation. - - 10. If you wish to incorporate parts of the Program into other free -programs whose distribution conditions are different, write to the author -to ask for permission. For software which is copyrighted by the Free -Software Foundation, write to the Free Software Foundation; we sometimes -make exceptions for this. Our decision will be guided by the two goals -of preserving the free status of all derivatives of our free software and -of promoting the sharing and reuse of software generally. - - NO WARRANTY - - 11. BECAUSE THE PROGRAM IS LICENSED FREE OF CHARGE, THERE IS NO WARRANTY -FOR THE PROGRAM, TO THE EXTENT PERMITTED BY APPLICABLE LAW. EXCEPT WHEN -OTHERWISE STATED IN WRITING THE COPYRIGHT HOLDERS AND/OR OTHER PARTIES -PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY OF ANY KIND, EITHER EXPRESSED -OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF -MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE. THE ENTIRE RISK AS -TO THE QUALITY AND PERFORMANCE OF THE PROGRAM IS WITH YOU. SHOULD THE -PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF ALL NECESSARY SERVICING, -REPAIR OR CORRECTION. - - 12. IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING -WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MAY MODIFY AND/OR -REDISTRIBUTE THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, -INCLUDING ANY GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING -OUT OF THE USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED -TO LOSS OF DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY -YOU OR THIRD PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER -PROGRAMS), EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE -POSSIBILITY OF SUCH DAMAGES. - - END OF TERMS AND CONDITIONS - - How to Apply These Terms to Your New Programs - - If you develop a new program, and you want it to be of the greatest -possible use to the public, the best way to achieve this is to make it -free software which everyone can redistribute and change under these terms. - - To do so, attach the following notices to the program. It is safest -to attach them to the start of each source file to most effectively -convey the exclusion of warranty; and each file should have at least -the "copyright" line and a pointer to where the full notice is found. - - - Copyright (C) - - This program is free software; you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation; either version 2 of the License, or - (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU General Public License for more details. - - You should have received a copy of the GNU General Public License along - with this program; if not, write to the Free Software Foundation, Inc., - 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. - -Also add information on how to contact you by electronic and paper mail. - -If the program is interactive, make it output a short notice like this -when it starts in an interactive mode: - - Gnomovision version 69, Copyright (C) year name of author - Gnomovision comes with ABSOLUTELY NO WARRANTY; for details type `show w'. - This is free software, and you are welcome to redistribute it - under certain conditions; type `show c' for details. - -The hypothetical commands `show w' and `show c' should show the appropriate -parts of the General Public License. Of course, the commands you use may -be called something other than `show w' and `show c'; they could even be -mouse-clicks or menu items--whatever suits your program. - -You should also get your employer (if you work as a programmer) or your -school, if any, to sign a "copyright disclaimer" for the program, if -necessary. Here is a sample; alter the names: - - Yoyodyne, Inc., hereby disclaims all copyright interest in the program - `Gnomovision' (which makes passes at compilers) written by James Hacker. - - , 1 April 1989 - Ty Coon, President of Vice - -This General Public License does not permit incorporating your program into -proprietary programs. If your program is a subroutine library, you may -consider it more useful to permit linking proprietary applications with the -library. If this is what you want to do, use the GNU Lesser General -Public License instead of this License. diff --git a/dictBuilder/Makefile b/dictBuilder/Makefile deleted file mode 100644 index 949995253..000000000 --- a/dictBuilder/Makefile +++ /dev/null @@ -1,68 +0,0 @@ -# ########################################################################## -# Dict Builder - Makefile -# Copyright (C) Yann Collet 2016 -# -# GPL v2 License -# -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation; either version 2 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. -# -# You can contact the author at : -# - ZSTD homepage : http://www.zstd.net/ -# ########################################################################## - -CPPFLAGS= -I../lib -CFLAGS ?= -O3 -CFLAGS += -std=c99 -Wall -Wextra -Wshadow -Wcast-qual -Wcast-align -Wundef -Wstrict-prototypes -Wstrict-aliasing=1 -FLAGS = $(CPPFLAGS) $(CFLAGS) $(LDFLAGS) $(MOREFLAGS) - -ZSTDDIR = ../lib - - -# Define *.exe as extension for Windows systems -ifneq (,$(filter Windows%,$(OS))) -EXT =.exe -VOID = nul -else -EXT = -VOID = /dev/null -endif - - -.PHONY: default all test - -default: dictBuilder - -all: dictBuilder - -dictBuilder: dibio.c dibcli.c $(ZSTDDIR)/dictBuilder.c $(ZSTDDIR)/huff0.c $(ZSTDDIR)/fse.c $(ZSTDDIR)/zstd_decompress.c $(ZSTDDIR)/zstd_compress.c $(ZSTDDIR)/divsufsort.c - $(CC) $(FLAGS) $^ -o $@$(EXT) - -clean: - @rm -f core *.o tmp* result* *.gcda \ - dictBuilder$(EXT) - @echo Cleaning completed - -test: dictBuilder - ./dictBuilder * - @rm dictionary - -clangtest: CC = clang -clangtest: CFLAGS += -Werror -clangtest: clean dictBuilder - -gpptest: CC = g++ -gpptest: CFLAGS=-O3 -Wall -Wextra -Wshadow -Wcast-align -Wcast-qual -Wundef -Werror -gpptest: clean dictBuilder - diff --git a/dictBuilder/dibcli.c b/dictBuilder/dibcli.c deleted file mode 100644 index a8009ae63..000000000 --- a/dictBuilder/dibcli.c +++ /dev/null @@ -1,263 +0,0 @@ -/* - dibcli - Command Line Interface (cli) for Dictionary Builder - Copyright (C) Yann Collet 2016 - - GPL v2 License - - This program is free software; you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation; either version 2 of the License, or - (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU General Public License for more details. - - You should have received a copy of the GNU General Public License along - with this program; if not, write to the Free Software Foundation, Inc., - 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. - - You can contact the author at : - - zstd source repository : https://github.com/Cyan4973/zstd -*/ - -/* ************************************** -* Compiler Specifics -****************************************/ -/* Disable some Visual warning messages */ -#ifdef _MSC_VER -# pragma warning(disable : 4127) /* disable: C4127: conditional expression is constant */ -#endif - - -/*-************************************ -* Includes -**************************************/ -#include /* exit, calloc, free */ -#include /* strcmp, strlen */ -#include /* fprintf, getchar */ -#include "dibio.h" -#include "zstd.h" /* version numbers */ - - -/*-************************************ -* Constants -**************************************/ -#define PROGRAM_DESCRIPTION "Dictionary builder" -#ifndef PROGRAM_VERSION -# define QUOTE(str) #str -# define EXP_Q(str) QUOTE(str) -# define PROGRAM_VERSION "v" EXP_Q(ZSTD_VERSION_MAJOR) "." EXP_Q(ZSTD_VERSION_MINOR) "." EXP_Q(ZSTD_VERSION_RELEASE) -#endif -#define AUTHOR "Yann Collet" -#define WELCOME_MESSAGE "*** %s %s %i-bits, by %s ***\n", PROGRAM_DESCRIPTION, PROGRAM_VERSION, (int)(sizeof(void*)*8), AUTHOR - -#define KB *(1 <<10) -#define MB *(1 <<20) -#define GB *(1U<<30) - -static const unsigned compressionLevelDefault = 5; -static const unsigned selectionLevelDefault = 9; /* determined experimentally */ -static const unsigned maxDictSizeDefault = 110 KB; -static const char* dictFileNameDefault = "dictionary"; - - -/*-************************************ -* Display Macros -**************************************/ -#define DISPLAY(...) fprintf(g_displayOut, __VA_ARGS__) -#define DISPLAYLEVEL(l, ...) if (g_displayLevel>=l) { DISPLAY(__VA_ARGS__); } -static FILE* g_displayOut; -static unsigned g_displayLevel = 2; // 0 : no display // 1: errors // 2 : + result + interaction + warnings ; // 3 : + progression; // 4 : + information - - -/*-************************************ -* Exceptions -**************************************/ -#define DEBUG 0 -#define DEBUGOUTPUT(...) if (DEBUG) DISPLAY(__VA_ARGS__); -#define EXM_THROW(error, ...) \ -{ \ - DEBUGOUTPUT("Error defined at %s, line %i : \n", __FILE__, __LINE__); \ - DISPLAYLEVEL(1, "Error %i : ", error); \ - DISPLAYLEVEL(1, __VA_ARGS__); \ - DISPLAYLEVEL(1, "\n"); \ - exit(error); \ -} - - -/*-************************************ -* Command Line -**************************************/ -static int usage(const char* programName) -{ - DISPLAY( "Usage :\n"); - DISPLAY( " %s [arg] [filenames]\n", programName); - DISPLAY( "\n"); - DISPLAY( "Arguments :\n"); - DISPLAY( " -o : name of dictionary file (default: %s) \n", dictFileNameDefault); - DISPLAY( "--maxdict : limit dictionary to specified size (default : %u) \n", maxDictSizeDefault); - DISPLAY( " -h/-H : display help/long help and exit\n"); - return 0; -} - -static int usage_advanced(const char* programName) -{ - DISPLAY(WELCOME_MESSAGE); - usage(programName); - DISPLAY( "\n"); - DISPLAY( "Advanced arguments :\n"); - DISPLAY( " -V : display Version number and exit\n"); - DISPLAY( "--fast : fast sampling mode\n"); - DISPLAY( " -L# : target compression level (default: %u)\n", compressionLevelDefault); - DISPLAY( " -S# : dictionary selectivity level (default: %u)\n", selectionLevelDefault); - DISPLAY( " -v : verbose mode\n"); - DISPLAY( " -q : suppress notifications; specify twice to suppress errors too\n"); - return 0; -} - -static int badusage(const char* programName) -{ - DISPLAYLEVEL(1, "Incorrect parameters\n"); - if (g_displayLevel >= 1) usage(programName); - return 1; -} - - -static void waitEnter(void) -{ - int unused; - DISPLAY("Press enter to continue...\n"); - unused = getchar(); - (void)unused; -} - - -int main(int argCount, const char** argv) -{ - int i, - main_pause=0, - operationResult=0, - nextArgumentIsMaxDict=0, - nextArgumentIsDictFileName=0; - unsigned cLevel = compressionLevelDefault; - unsigned maxDictSize = maxDictSizeDefault; - unsigned selectionLevel = selectionLevelDefault; - const char** filenameTable = (const char**)malloc(argCount * sizeof(const char*)); /* argCount >= 1 */ - unsigned filenameIdx = 0; - const char* programName = argv[0]; - const char* dictFileName = dictFileNameDefault; - - /* init */ - g_displayOut = stderr; /* unfortunately, cannot be set at declaration */ - if (filenameTable==NULL) EXM_THROW(1, "not enough memory\n"); - /* Pick out program name from path. Don't rely on stdlib because of conflicting behavior */ - for (i = (int)strlen(programName); i > 0; i--) { if ((programName[i] == '/') || (programName[i] == '\\')) { i++; break; } } - programName += i; - - /* command switches */ - for(i=1; i='0') && (*argument<='9')) - maxDictSize = maxDictSize * 10 + (*argument - '0'), argument++; - if (*argument=='k' || *argument=='K') - maxDictSize <<= 10; - continue; - } - - /* long commands (--long-word) */ - if (!strcmp(argument, "--version")) { g_displayOut=stdout; DISPLAY(WELCOME_MESSAGE); return 0; } - if (!strcmp(argument, "--help")) { g_displayOut=stdout; return usage_advanced(programName); } - if (!strcmp(argument, "--verbose")) { g_displayLevel++; if (g_displayLevel<3) g_displayLevel=3; continue; } - if (!strcmp(argument, "--quiet")) { g_displayLevel--; continue; } - if (!strcmp(argument, "--maxdict")) { nextArgumentIsMaxDict=1; continue; } - if (!strcmp(argument, "--fast")) { selectionLevel=1; cLevel=1; continue; } - - /* Decode commands (note : aggregated commands are allowed) */ - if (argument[0]=='-') { - argument++; - - while (argument[0]!=0) { - switch(argument[0]) - { - /* Display help */ - case 'V': g_displayOut=stdout; DISPLAY(WELCOME_MESSAGE); return 0; /* Version Only */ - case 'H': - case 'h': g_displayOut=stdout; return usage_advanced(programName); - - /* Selection level */ - case 'S': argument++; - selectionLevel = 0; - while ((*argument >= '0') && (*argument <= '9')) - selectionLevel *= 10, selectionLevel += *argument++ - '0'; - break; - - /* Selection level */ - case 'L': argument++; - cLevel = 0; - while ((*argument >= '0') && (*argument <= '9')) - cLevel *= 10, cLevel += *argument++ - '0'; - break; - - /* Verbose mode */ - case 'v': g_displayLevel++; if (g_displayLevel<3) g_displayLevel=3; argument++; break; - - /* Quiet mode */ - case 'q': g_displayLevel--; argument++; break; - - /* dictionary name */ - case 'o': nextArgumentIsDictFileName=1; argument++; break; - - /* Pause at the end (hidden option) */ - case 'p': main_pause=1; argument++; break; - - /* unknown command */ - default : return badusage(programName); - } } - continue; - } - - /* add filename to list */ - filenameTable[filenameIdx++] = argument; - } - - /* Welcome message (if verbose) */ - DISPLAYLEVEL(3, WELCOME_MESSAGE); - - /* check nb files */ - if (filenameIdx==0) return badusage(programName); - if (filenameIdx < 100) - { - DISPLAYLEVEL(2, "Warning : set contains only %u files ... \n", filenameIdx); - DISPLAYLEVEL(3, "!! For better results, consider providing > 1.000 samples !!\n"); - DISPLAYLEVEL(3, "!! Each sample should preferably be stored as a separate file !!\n"); - } - - /* building ... */ - { - ZDICT_params_t param; - param.selectivityLevel = selectionLevel; - param.compressionLevel = cLevel; - DiB_setNotificationLevel(g_displayLevel); - operationResult = DiB_trainFromFiles(dictFileName, maxDictSize, - filenameTable, filenameIdx, - param); - } - - if (main_pause) waitEnter(); - free((void*)filenameTable); - return operationResult; -} diff --git a/dictBuilder/dibio.c b/dictBuilder/dibio.c deleted file mode 100644 index a471d5450..000000000 --- a/dictBuilder/dibio.c +++ /dev/null @@ -1,275 +0,0 @@ -/* - dibio - I/O API for dictionary builder - Copyright (C) Yann Collet 2016 - - GPL v2 License - - This program is free software; you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation; either version 2 of the License, or - (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU General Public License for more details. - - You should have received a copy of the GNU General Public License along - with this program; if not, write to the Free Software Foundation, Inc., - 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. - - You can contact the author at : - - zstd homepage : http://www.zstd.net/ -*/ - -/*-************************************** -* Compiler Options -****************************************/ -/* Disable some Visual warning messages */ -#ifdef _MSC_VER -# define _CRT_SECURE_NO_WARNINGS /* fopen */ -# pragma warning(disable : 4127) /* disable: C4127: conditional expression is constant */ -#endif - -/* Unix Large Files support (>4GB) */ -#define _FILE_OFFSET_BITS 64 -#if (defined(__sun__) && (!defined(__LP64__))) /* Sun Solaris 32-bits requires specific definitions */ -# define _LARGEFILE_SOURCE -#elif ! defined(__LP64__) /* No point defining Large file for 64 bit */ -# define _LARGEFILE64_SOURCE -#endif - - -/*-************************************* -* Includes -***************************************/ -#include /* malloc, free */ -#include /* memset */ -#include /* fprintf, fopen, ftello64 */ -#include /* stat64 */ -#include /* stat64 */ -#include /* clock */ - -#include "mem.h" /* read */ -#include "error_private.h" -#include "dictBuilder_static.h" - -/*-************************************* -* Compiler specifics -***************************************/ -#if !defined(S_ISREG) -# define S_ISREG(x) (((x) & S_IFMT) == S_IFREG) -#endif - - -/*-************************************* -* Constants -***************************************/ -#define KB *(1 <<10) -#define MB *(1 <<20) -#define GB *(1U<<30) - -#define DICTLISTSIZE 10000 -#define MEMMULT 11 -static const size_t maxMemory = (sizeof(size_t) == 4) ? (2 GB - 64 MB) : ((size_t)(512 MB) << sizeof(size_t)); - -#define NOISELENGTH 32 -#define PRIME1 2654435761U -#define PRIME2 2246822519U - - -/*-************************************* -* Console display -***************************************/ -#define DISPLAY(...) fprintf(stderr, __VA_ARGS__) -#define DISPLAYLEVEL(l, ...) if (g_displayLevel>=l) { DISPLAY(__VA_ARGS__); } -static unsigned g_displayLevel = 0; /* 0 : no display; 1: errors; 2: default; 4: full information */ -void DiB_setNotificationLevel(unsigned l) { g_displayLevel=l; ZDICT_setNotificationLevel(l); } - -void DiB_printHex(U32 dlevel, const void* ptr, size_t length) -{ - const BYTE* const b = (const BYTE*)ptr; - size_t u; - for (u=0; u126) c = '.'; /* non-printable char */ - DISPLAYLEVEL(dlevel, "%c", c); - } -} - - -/*-************************************* -* Exceptions -***************************************/ -#ifndef DEBUG -# define DEBUG 0 -#endif -#define DEBUGOUTPUT(...) if (DEBUG) DISPLAY(__VA_ARGS__); -#define EXM_THROW(error, ...) \ -{ \ - DEBUGOUTPUT("Error defined at %s, line %i : \n", __FILE__, __LINE__); \ - DISPLAYLEVEL(1, "Error %i : ", error); \ - DISPLAYLEVEL(1, __VA_ARGS__); \ - DISPLAYLEVEL(1, "\n"); \ - exit(error); \ -} - - -/* ******************************************************** -* Helper functions -**********************************************************/ -unsigned DiB_isError(size_t errorCode) { return ERR_isError(errorCode); } - -const char* DiB_getErrorName(size_t errorCode) { return ERR_getErrorName(errorCode); } - - -/* ******************************************************** -* File related operations -**********************************************************/ -static unsigned long long DiB_getFileSize(const char* infilename) -{ - int r; -#if defined(_MSC_VER) - struct _stat64 statbuf; - r = _stat64(infilename, &statbuf); -#else - struct stat statbuf; - r = stat(infilename, &statbuf); -#endif - if (r || !S_ISREG(statbuf.st_mode)) return 0; /* No good... */ - return (unsigned long long)statbuf.st_size; -} - - -static unsigned long long DiB_getTotalFileSize(const char** fileNamesTable, unsigned nbFiles) -{ - unsigned long long total = 0; - unsigned n; - for (n=0; n bufferSize-pos) fileSize = 0; /* stop there, not enough memory to load all files */ - readSize = fread(buff+pos, 1, (size_t)fileSize, f); - if (readSize != (size_t)fileSize) EXM_THROW(11, "could not read %s", fileNamesTable[n]); - pos += readSize; - fileSizes[n] = (size_t)fileSize; - fclose(f); - } -} - - -/*-******************************************************** -* Dictionary training functions -**********************************************************/ -static size_t DiB_findMaxMem(unsigned long long requiredMem) -{ - size_t step = 8 MB; - void* testmem = NULL; - - requiredMem = (((requiredMem >> 23) + 1) << 23); - requiredMem += 2 * step; - if (requiredMem > maxMemory) requiredMem = maxMemory; - - while (!testmem) { - requiredMem -= step; - testmem = malloc((size_t)requiredMem); - } - - free(testmem); - return (size_t)(requiredMem - step); -} - - -static void DiB_fillNoise(void* buffer, size_t length) -{ - unsigned acc = PRIME1; - size_t p=0;; - - for (p=0; p> 21); - } -} - - -static void DiB_saveDict(const char* dictFileName, - const void* buff, size_t buffSize) -{ - FILE* f; - size_t n; - - f = fopen(dictFileName, "wb"); - if (f==NULL) EXM_THROW(3, "cannot open %s ", dictFileName); - - n = fwrite(buff, 1, buffSize, f); - if (n!=buffSize) EXM_THROW(4, "%s : write error", dictFileName) - - n = (size_t)fclose(f); - if (n!=0) EXM_THROW(5, "%s : flush error", dictFileName) -} - - -int DiB_trainFromFiles(const char* dictFileName, unsigned maxDictSize, - const char** fileNamesTable, unsigned nbFiles, - ZDICT_params_t params) -{ - void* srcBuffer; - size_t benchedSize; - size_t* fileSizes = (size_t*)malloc(nbFiles * sizeof(size_t)); - unsigned long long totalSizeToLoad = DiB_getTotalFileSize(fileNamesTable, nbFiles); - void* dictBuffer = malloc(maxDictSize); - size_t dictSize; - int result = 0; - - /* init */ - benchedSize = DiB_findMaxMem(totalSizeToLoad * MEMMULT) / MEMMULT; - if ((unsigned long long)benchedSize > totalSizeToLoad) benchedSize = (size_t)totalSizeToLoad; - if (benchedSize < totalSizeToLoad) - DISPLAYLEVEL(1, "Not enough memory; training on %u MB only...\n", (unsigned)(benchedSize >> 20)); - - /* Memory allocation & restrictions */ - srcBuffer = malloc(benchedSize+NOISELENGTH); /* + noise */ - if ((!fileSizes) || (!srcBuffer) || (!dictBuffer)) EXM_THROW(12, "not enough memory for DiB_trainFiles"); /* should not happen */ - - /* Load input buffer */ - DiB_loadFiles(srcBuffer, benchedSize, fileSizes, fileNamesTable, nbFiles); - DiB_fillNoise((char*)srcBuffer + benchedSize, NOISELENGTH); /* guard band, for end of buffer condition */ - - /* call buffer version */ - dictSize = ZDICT_trainFromBuffer_unsafe(dictBuffer, maxDictSize, - srcBuffer, fileSizes, nbFiles, - params); - if (ZDICT_isError(dictSize)) { - DISPLAYLEVEL(1, "dictionary training failed : %s", ZDICT_getErrorName(dictSize)); /* should not happen */ - result = 1; - goto _cleanup; - } - - /* save dict */ - DISPLAYLEVEL(2, "Save dictionary of size %u into file %s \n", (U32)dictSize, dictFileName); - DiB_saveDict(dictFileName, dictBuffer, dictSize); - - /* clean up */ -_cleanup: - free(srcBuffer); - free(dictBuffer); - free(fileSizes); - return result; -} diff --git a/dictBuilder/dibio.h b/dictBuilder/dibio.h deleted file mode 100644 index c9aee1fcd..000000000 --- a/dictBuilder/dibio.h +++ /dev/null @@ -1,64 +0,0 @@ -/* - dibio.h - I/O API for dictionary builder - Copyright (C) Yann Collet 2016 - - GPL v2 License - - This program is free software; you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation; either version 2 of the License, or - (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU General Public License for more details. - - You should have received a copy of the GNU General Public License along - with this program; if not, write to the Free Software Foundation, Inc., - 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. - - You can contact the author at : - - zstd source repository : https://github.com/Cyan4973/zstd - - ztsd public forum : https://groups.google.com/forum/#!forum/lz4c -*/ - -/* This library is designed for a single-threaded console application. -* It exit() and printf() into stderr when it encounters an error condition. */ - -#ifndef DIBIO_H_003 -#define DIBIO_H_003 - - -/*-************************************* -* Dependencies -***************************************/ -#include "dictBuilder_static.h" /* ZDICT_params_t */ - - -/*-************************************* -* Public functions -***************************************/ -/*! DiB_trainFromFiles() : - Train a dictionary from a set of files provided by `fileNamesTable`. - Resulting dictionary is written into file `dictFileName`. - `parameters` is optional and can be provided with values set to 0, meaning "default". - @return : 0 == ok. Any other : error. -*/ -int DiB_trainFromFiles(const char* dictFileName, unsigned maxDictSize, - const char** fileNamesTable, unsigned nbFiles, - ZDICT_params_t parameters); - - -/*-************************************* -* Helper functions -***************************************/ -/*! DiB_setNotificationLevel - Set amount of notification to be displayed on the console. - default initial value : 0 = no console notification. - Note : not thread-safe (use a global constant) -*/ -void DiB_setNotificationLevel(unsigned l); - - -#endif diff --git a/dictBuilder/dictionary b/dictBuilder/dictionary deleted file mode 100644 index d6bf575c6..000000000 Binary files a/dictBuilder/dictionary and /dev/null differ diff --git a/lib/Makefile b/lib/Makefile index 5ce8c18b7..2b724b7b7 100644 --- a/lib/Makefile +++ b/lib/Makefile @@ -51,7 +51,7 @@ FLAGS = $(CPPFLAGS) $(CFLAGS) $(LDFLAGS) $(MOREFLAGS) LIBDIR ?= $(PREFIX)/lib INCLUDEDIR=$(PREFIX)/include -ZSTD_FILES := zstd_compress.c zstd_decompress.c fse.c huff0.c dictBuilder.c divsufsort.c +ZSTD_FILES := zstd_compress.c zstd_decompress.c fse.c huff0.c zdict.c divsufsort.c ZSTD_LEGACY:= legacy/zstd_v01.c legacy/zstd_v02.c legacy/zstd_v03.c legacy/zstd_v04.c ifeq ($(ZSTD_LEGACY_SUPPORT), 0) @@ -119,6 +119,7 @@ install: libzstd libzstd.pc @install -m 644 libzstd.a $(DESTDIR)$(LIBDIR)/libzstd.a @install -m 644 zstd.h $(DESTDIR)$(INCLUDEDIR)/zstd.h @install -m 644 zstd.h $(DESTDIR)$(INCLUDEDIR)/zstd_buffered.h + @install -m 644 zstd.h $(DESTDIR)$(INCLUDEDIR)/zdict.h @echo zstd static and shared library installed uninstall: @@ -128,6 +129,8 @@ uninstall: @[ -x $(DESTDIR)$(LIBDIR)/libzstd.$(SHARED_EXT_VER) ] && rm -f $(DESTDIR)$(LIBDIR)/libzstd.$(SHARED_EXT_VER) @[ -f $(DESTDIR)$(LIBDIR)/libzstd.a ] && rm -f $(DESTDIR)$(LIBDIR)/libzstd.a @[ -f $(DESTDIR)$(INCLUDEDIR)/zstd.h ] && rm -f $(DESTDIR)$(INCLUDEDIR)/zstd.h + @[ -f $(DESTDIR)$(INCLUDEDIR)/zstd.h ] && rm -f $(DESTDIR)$(INCLUDEDIR)/zstd_buffered.h + @[ -f $(DESTDIR)$(INCLUDEDIR)/zstd.h ] && rm -f $(DESTDIR)$(INCLUDEDIR)/zdict.h @echo zstd libraries successfully uninstalled endif diff --git a/lib/dictBuilder.c b/lib/zdict.c similarity index 99% rename from lib/dictBuilder.c rename to lib/zdict.c index 50b67d58d..ec2ffbb70 100644 --- a/lib/dictBuilder.c +++ b/lib/zdict.c @@ -61,11 +61,11 @@ #include "mem.h" /* read */ #include "error_private.h" -#include "divsufsort.h" -#include "dictBuilder_static.h" #include "fse.h" #include "huff0_static.h" #include "zstd_internal.h" +#include "divsufsort.h" +#include "zdict_static.h" /*-************************************* diff --git a/lib/dictBuilder.h b/lib/zdict.h similarity index 100% rename from lib/dictBuilder.h rename to lib/zdict.h diff --git a/lib/dictBuilder_static.h b/lib/zdict_static.h similarity index 99% rename from lib/dictBuilder_static.h rename to lib/zdict_static.h index 0bddb30df..e1ef73136 100644 --- a/lib/dictBuilder_static.h +++ b/lib/zdict_static.h @@ -44,7 +44,7 @@ extern "C" { /*-************************************* * Dependencies ***************************************/ -#include "dictBuilder.h" +#include "zdict.h" /*-************************************* diff --git a/programs/Makefile b/programs/Makefile index b107b72d6..e96a0dd20 100644 --- a/programs/Makefile +++ b/programs/Makefile @@ -84,11 +84,11 @@ default: zstd all: zstd zstd32 fullbench fullbench32 fuzzer fuzzer32 zbufftest zbufftest32 paramgrill datagen -zstd : $(ZSTD_FILES) $(ZSTD_FILES_LEGACY) $(ZSTDDIR)/zstd_buffered.c $(ZSTDDIR)/dictBuilder.c $(ZSTDDIR)/divsufsort.c \ +zstd : $(ZSTD_FILES) $(ZSTD_FILES_LEGACY) $(ZSTDDIR)/zstd_buffered.c $(ZSTDDIR)/zdict.c $(ZSTDDIR)/divsufsort.c \ zstdcli.c fileio.c bench.c xxhash.c datagen.c dibio.c $(CC) $(FLAGS) -DZSTD_LEGACY_SUPPORT=$(ZSTD_LEGACY_SUPPORT) $^ -o $@$(EXT) -zstd32: $(ZSTD_FILES) $(ZSTD_FILES_LEGACY) $(ZSTDDIR)/zstd_buffered.c $(ZSTDDIR)/dictBuilder.c $(ZSTDDIR)/divsufsort.c \ +zstd32: $(ZSTD_FILES) $(ZSTD_FILES_LEGACY) $(ZSTDDIR)/zstd_buffered.c $(ZSTDDIR)/zdict.c $(ZSTDDIR)/divsufsort.c \ zstdcli.c fileio.c bench.c xxhash.c datagen.c dibio.c $(CC) -m32 $(FLAGS) -DZSTD_LEGACY_SUPPORT=$(ZSTD_LEGACY_SUPPORT) $^ -o $@$(EXT) diff --git a/programs/dibio.c b/programs/dibio.c index a471d5450..7b6611536 100644 --- a/programs/dibio.c +++ b/programs/dibio.c @@ -52,7 +52,8 @@ #include "mem.h" /* read */ #include "error_private.h" -#include "dictBuilder_static.h" +#include "zdict_static.h" + /*-************************************* * Compiler specifics @@ -86,17 +87,6 @@ static const size_t maxMemory = (sizeof(size_t) == 4) ? (2 GB - 64 MB) : ((size_ static unsigned g_displayLevel = 0; /* 0 : no display; 1: errors; 2: default; 4: full information */ void DiB_setNotificationLevel(unsigned l) { g_displayLevel=l; ZDICT_setNotificationLevel(l); } -void DiB_printHex(U32 dlevel, const void* ptr, size_t length) -{ - const BYTE* const b = (const BYTE*)ptr; - size_t u; - for (u=0; u126) c = '.'; /* non-printable char */ - DISPLAYLEVEL(dlevel, "%c", c); - } -} - /*-************************************* * Exceptions diff --git a/programs/dibio.h b/programs/dibio.h index 9b8deb54b..4d24cc16a 100644 --- a/programs/dibio.h +++ b/programs/dibio.h @@ -32,7 +32,7 @@ /*-************************************* * Dependencies ***************************************/ -#include "dictBuilder_static.h" /* ZDICT_params_t */ +#include "zdict_static.h" /* ZDICT_params_t */ /*-*************************************