database/mariadb
1
0
Fork 0
mirror of https://github.com/MariaDB/server.git synced 2025-08-08 11:22:35 +03:00
Code Activity

- MDEV-11295: developing handling files contained in ZIP file.

Browse Source 
Enable using multiple zip files
  modified:   storage/connect/filamzip.cpp
  modified:   storage/connect/ha_connect.cc
  modified:   storage/connect/ha_connect.h
  modified:   storage/connect/mycat.h
  modified:   storage/connect/tabdos.cpp
  modified:   storage/connect/tabdos.h
  modified:   storage/connect/tabfmt.cpp
  modified:   storage/connect/tabjson.cpp
This commit is contained in:
Olivier Bertrand
2016-12-14 14:20:23 +01:00
parent d44723e621
commit 952306502e
10 changed files with 63 additions and 96 deletions
Download Patch File Download Diff File Expand all files Collapse all files

19
storage/connect/filamzip.cpp
View File

@@ -48,11 +48,11 @@
ZIPFAM::ZIPFAM(PDOSDEF tdp) : MAPFAM(tdp)
{
zipfile = NULL;
zfn = tdp->Zipfn;
target = tdp->Fn;
//zfn = tdp->Fn;
target = tdp->Entry;
//*fn = 0;
entryopen = false;
multiple = tdp->Multiple;
multiple = (target && !(strchr(target, '*') || strchr(target, '?'))) ? 0 : 1;
// Init the case mapping table.
#if defined(__WIN__)
@@ -65,7 +65,7 @@ ZIPFAM::ZIPFAM(PDOSDEF tdp) : MAPFAM(tdp)
ZIPFAM::ZIPFAM(PZIPFAM txfp) : MAPFAM(txfp)
{
zipfile = txfp->zipfile;
zfn = txfp->zfn;
//zfn = txfp->zfn;
target = txfp->target;
//strcpy(fn, txfp->fn);
finfo = txfp->finfo;
@@ -129,7 +129,7 @@ int ZIPFAM::GetFileLength(PGLOBAL g)
bool ZIPFAM::open(PGLOBAL g, const char *filename)
{
if (!zipfile && !(zipfile = unzOpen64(filename)))
sprintf(g->Message, "Zipfile open error");
sprintf(g->Message, "Zipfile open error on %s", filename);
return (zipfile == NULL);
} // end of open
@@ -205,7 +205,7 @@ bool ZIPFAM::OpenTableFile(PGLOBAL g)
/*********************************************************************/
if (mode == MODE_READ) {
// We used the file name relative to recorded datapath
PlugSetPath(filename, zfn, Tdbp->GetPath());
PlugSetPath(filename, To_File, Tdbp->GetPath());
bool b = open(g, filename);
@@ -218,7 +218,7 @@ bool ZIPFAM::OpenTableFile(PGLOBAL g)
if (rc == UNZ_END_OF_LIST_OF_FILE) {
sprintf(g->Message, "Target file %s not in %s", target, filename);
return true;
return false;
} else if (rc != UNZ_OK) {
sprintf(g->Message, "unzLocateFile rc=%d", rc);
return true;
@@ -229,7 +229,7 @@ bool ZIPFAM::OpenTableFile(PGLOBAL g)
return true;
else if (rc == RC_NF) {
sprintf(g->Message, "No match of %s in %s", target, filename);
return true;
return false;
} // endif rc
} // endif multiple
@@ -258,7 +258,8 @@ bool ZIPFAM::OpenTableFile(PGLOBAL g)
} // endif fp
To_Fb = fp; // Useful when closing
} // endif b
} else
return true;
} else {
strcpy(g->Message, "Only READ mode supported for ZIP files");

2
storage/connect/filamzip.h
View File

@@ -51,7 +51,7 @@ protected:
// Members
unzFile zipfile; // The ZIP container file
PSZ zfn; // The ZIP file name
//PSZ zfn; // The ZIP file name
PSZ target; // The target file name
unz_file_info finfo; // The current file info
//char fn[FILENAME_MAX]; // The current file name

10
storage/connect/ha_connect.cc
View File

@@ -171,9 +171,9 @@
#define JSONMAX 10 // JSON Default max grp size
extern "C" {
char version[]= "Version 1.04.0009 December 09, 2016";
char version[]= "Version 1.05.0001 December 13, 2016";
#if defined(__WIN__)
char compver[]= "Version 1.04.0009 " __DATE__ " " __TIME__;
char compver[]= "Version 1.05.0001 " __DATE__ " " __TIME__;
char slash= '\\';
#else // !__WIN__
char slash= '/';
@@ -512,12 +512,12 @@ ha_create_table_option connect_table_option_list[]=
HA_TOPTION_NUMBER("QUOTED", quoted, (ulonglong) -1, 0, 3, 1),
HA_TOPTION_NUMBER("ENDING", ending, (ulonglong) -1, 0, INT_MAX32, 1),
HA_TOPTION_NUMBER("COMPRESS", compressed, 0, 0, 2, 1),
//HA_TOPTION_BOOL("COMPRESS", compressed, 0),
HA_TOPTION_BOOL("MAPPED", mapped, 0),
HA_TOPTION_BOOL("HUGE", huge, 0),
HA_TOPTION_BOOL("SPLIT", split, 0),
HA_TOPTION_BOOL("READONLY", readonly, 0),
HA_TOPTION_BOOL("SEPINDEX", sepindex, 0),
HA_TOPTION_BOOL("ZIPPED", zipped, 0),
HA_TOPTION_END
};
@@ -532,7 +532,6 @@ ha_create_table_option connect_field_option_list[]=
{
HA_FOPTION_NUMBER("FLAG", offset, (ulonglong) -1, 0, INT_MAX32, 1),
HA_FOPTION_NUMBER("MAX_DIST", freq, 0, 0, INT_MAX32, 1), // BLK_INDX
//HA_FOPTION_NUMBER("DISTRIB", opt, 0, 0, 2, 1), // used for BLK_INDX
HA_FOPTION_NUMBER("FIELD_LENGTH", fldlen, 0, 0, INT_MAX32, 1),
HA_FOPTION_STRING("DATE_FORMAT", dateformat),
HA_FOPTION_STRING("FIELD_FORMAT", fieldformat),
@@ -678,7 +677,6 @@ static int connect_init_func(void *p)
connect_hton= (handlerton *)p;
connect_hton->state= SHOW_OPTION_YES;
connect_hton->create= connect_create_handler;
//connect_hton->flags= HTON_TEMPORARY_NOT_SUPPORTED | HTON_NO_PARTITION;
connect_hton->flags= HTON_TEMPORARY_NOT_SUPPORTED;
connect_hton->table_options= connect_table_option_list;
connect_hton->field_options= connect_field_option_list;
@@ -1135,6 +1133,8 @@ bool GetBooleanTableOption(PGLOBAL g, PTOS options, char *opname, bool bdef)
opval= options->sepindex;
else if (!stricmp(opname, "Header"))
opval= (options->header != 0); // Is Boolean for some table types
else if (!stricmp(opname, "Zipped"))
opval = options->zipped;
else if (options->oplist)
if ((pv= GetListOption(g, opname, options->oplist)))
opval= (!*pv || *pv == 'y' || *pv == 'Y' || atoi(pv) != 0);

37
storage/connect/ha_connect.h
View File

@@ -83,42 +83,9 @@ extern handlerton *connect_hton;
These can be specified in the CREATE TABLE:
CREATE TABLE ( ... ) {...here...}
*/
#if 0 // moved to mycat.h
typedef struct ha_table_option_struct TOS, *PTOS;
struct ha_table_option_struct {
const char *type;
const char *filename;
const char *optname;
const char *tabname;
const char *tablist;
const char *dbname;
const char *separator;
//const char *connect;
const char *qchar;
const char *module;
const char *subtype;
const char *catfunc;
const char *srcdef;
const char *colist;
const char *oplist;
const char *data_charset;
ulonglong lrecl;
ulonglong elements;
//ulonglong estimate;
ulonglong multiple;
ulonglong header;
ulonglong quoted;
ulonglong ending;
ulonglong compressed;
bool mapped;
bool huge;
bool split;
bool readonly;
bool sepindex;
};
#endif // 0
------ Was moved to mycat.h ------
*/
/**
structure for CREATE TABLE options (field options)

1
storage/connect/mycat.h
View File

@@ -62,6 +62,7 @@ struct ha_table_option_struct {
bool split;
bool readonly;
bool sepindex;
bool zipped;
};
// Possible value for catalog functions

19
storage/connect/tabdos.cpp
View File

@@ -96,10 +96,11 @@ DOSDEF::DOSDEF(void)
Pseudo = 3;
Fn = NULL;
Ofn = NULL;
Zipfn = NULL;
Entry = NULL;
To_Indx = NULL;
Recfm = RECFM_VAR;
Mapped = false;
Zipped = false;
Padded = false;
Huge = false;
Accept = false;
@@ -131,20 +132,11 @@ bool DOSDEF::DefineAM(PGLOBAL g, LPCSTR am, int)
: (am && !stricmp(am, "DBF")) ? "D" : "V";
if (*dfm != 'D')
Zipfn = GetStringCatInfo(g, "Zipfile", NULL);
Zipped = GetBoolCatInfo("Zipped", false);
if (Zipfn && Multiple) {
// Prevent Fn to default to table name
Desc = GetStringCatInfo(g, "Filename", NULL);
Fn = GetStringCatInfo(g, "Filename", "<%>");
if (!strcmp(Fn, "<%>"))
Fn = NULL;
} else
Desc = Fn = GetStringCatInfo(g, "Filename", NULL);
Ofn = GetStringCatInfo(g, "Optname", Fn);
Entry = GetStringCatInfo(g, "Entry", NULL);
GetCharCatInfo("Recfm", (PSZ)dfm, buf, sizeof(buf));
Recfm = (toupper(*buf) == 'F') ? RECFM_FIX :
(toupper(*buf) == 'B') ? RECFM_BIN :
@@ -350,7 +342,7 @@ PTDB DOSDEF::GetTable(PGLOBAL g, MODE mode)
/* Allocate table and file processing class of the proper type. */
/* Column blocks will be allocated only when needed. */
/*********************************************************************/
if (Zipfn) {
if (Zipped) {
#if defined(ZIP_SUPPORT)
if (Recfm == RECFM_VAR)
txfp = new(g) ZIPFAM(this);
@@ -358,7 +350,6 @@ PTDB DOSDEF::GetTable(PGLOBAL g, MODE mode)
txfp = new(g) ZPXFAM(this);
tdbp = new(g) TDBDOS(this, txfp);
return tdbp;
#else // !ZIP_SUPPORT
strcpy(g->Message, "ZIP not supported");
return NULL;

5
storage/connect/tabdos.h
View File

@@ -59,7 +59,7 @@ class DllExport DOSDEF : public TABDEF { /* Logical table description */
// Methods
virtual int Indexable(void)
{return (!Multiple && !Zipfn && Compressed != 1) ? 1 : 0;}
{return (!Multiple && !Zipped && Compressed != 1) ? 1 : 0;}
virtual bool DeleteIndexFile(PGLOBAL g, PIXDEF pxdf);
virtual bool DefineAM(PGLOBAL g, LPCSTR am, int poff);
virtual PTDB GetTable(PGLOBAL g, MODE mode);
@@ -73,10 +73,11 @@ class DllExport DOSDEF : public TABDEF { /* Logical table description */
// Members
PSZ Fn; /* Path/Name of corresponding file */
PSZ Ofn; /* Base Path/Name of matching index files*/
PSZ Zipfn; /* Zip container name */
PSZ Entry; /* Zip entry name or pattern */
PIXDEF To_Indx; /* To index definitions blocks */
RECFM Recfm; /* 0:VAR, 1:FIX, 2:BIN, 3:VCT, 6:DBF */
bool Mapped; /* 0: disk file, 1: memory mapped file */
bool Zipped; /* true for zipped table file */
bool Padded; /* true for padded table file */
bool Huge; /* true for files larger than 2GB */
bool Accept; /* true if wrong lines are accepted */

26
storage/connect/tabfmt.cpp
View File

@@ -108,6 +108,11 @@ PQRYRES CSVColumns(PGLOBAL g, char *dp, PTOS topt, bool info)
goto skipit;
} // endif info
if (GetIntegerTableOption(g, topt, "Multiple", 0)) {
strcpy(g->Message, "Cannot find column definition for multiple table");
return NULL;
} // endif Multiple
// num_max = atoi(p+1); // Max num of record to test
imax = hmax = nerr = 0;
@@ -123,18 +128,16 @@ PQRYRES CSVColumns(PGLOBAL g, char *dp, PTOS topt, bool info)
/*********************************************************************/
tdp = new(g) CSVDEF;
#if defined(ZIP_SUPPORT)
tdp->Zipfn = GetStringTableOption(g, topt, "Zipfile", NULL);
tdp->Multiple = GetIntegerTableOption(g, topt, "Multiple", 0);
tdp->Entry = GetStringTableOption(g, topt, "Entry", NULL);
tdp->Zipped = GetBooleanTableOption(g, topt, "Zipped", false);
#endif // ZIP_SUPPORT
tdp->Fn = GetStringTableOption(g, topt, "Filename", NULL);
fn = tdp->Fn = GetStringTableOption(g, topt, "Filename", NULL);
if (!tdp->Fn && !tdp->Zipfn && !tdp->Multiple) {
if (!tdp->Fn) {
strcpy(g->Message, MSG(MISSING_FNAME));
return NULL;
} // endif Fn
fn = (tdp->Fn) ? tdp->Fn : "unnamed";
if (!(tdp->Lrecl = GetIntegerTableOption(g, topt, "Lrecl", 0)))
tdp->Lrecl = 4096;
@@ -174,7 +177,7 @@ PQRYRES CSVColumns(PGLOBAL g, char *dp, PTOS topt, bool info)
htrc("File %s Sep=%c Qot=%c Header=%d maxerr=%d\n",
SVP(tdp->Fn), tdp->Sep, tdp->Qot, tdp->Header, tdp->Maxerr);
if (tdp->Zipfn)
if (tdp->Zipped)
tdbp = new(g) TDBCSV(tdp, new(g) ZIPFAM(tdp));
else
tdbp = new(g) TDBCSV(tdp, new(g) DOSFAM(tdp));
@@ -493,16 +496,9 @@ PTDB CSVDEF::GetTable(PGLOBAL g, MODE mode)
/*******************************************************************/
/* Allocate a file processing class of the proper type. */
/*******************************************************************/
if (Zipfn) {
if (Zipped) {
#if defined(ZIP_SUPPORT)
txfp = new(g) ZIPFAM(this);
if (!Fmtd)
tdbp = new(g) TDBCSV(this, txfp);
else
tdbp = new(g) TDBFMT(this, txfp);
return tdbp;
#else // !ZIP_SUPPORT
strcpy(g->Message, "ZIP not supported");
return NULL;

21
storage/connect/tabjson.cpp
View File

@@ -94,6 +94,11 @@ PQRYRES JSONColumns(PGLOBAL g, char *db, PTOS topt, bool info)
goto skipit;
} // endif info
if (GetIntegerTableOption(g, topt, "Multiple", 0)) {
strcpy(g->Message, "Cannot find column definition for multiple table");
return NULL;
} // endif Multiple
/*********************************************************************/
/* Open the input file. */
/*********************************************************************/
@@ -102,12 +107,12 @@ PQRYRES JSONColumns(PGLOBAL g, char *db, PTOS topt, bool info)
tdp = new(g) JSONDEF;
#if defined(ZIP_SUPPORT)
tdp->Zipfn = GetStringTableOption(g, topt, "Zipfile", NULL);
tdp->Multiple = GetIntegerTableOption(g, topt, "Multiple", 0);
tdp->Entry = GetStringTableOption(g, topt, "Entry", NULL);
tdp->Zipped = GetBooleanTableOption(g, topt, "Zipped", false);
#endif // ZIP_SUPPORT
tdp->Fn = GetStringTableOption(g, topt, "Filename", NULL);
if (!tdp->Fn && !tdp->Zipfn && !tdp->Multiple) {
if (!tdp->Fn) {
strcpy(g->Message, MSG(MISSING_FNAME));
return NULL;
} // endif Fn
@@ -122,7 +127,7 @@ PQRYRES JSONColumns(PGLOBAL g, char *db, PTOS topt, bool info)
tdp->Fn, tdp->Objname, tdp->Pretty, lvl);
if (tdp->Pretty == 2) {
if (tdp->Zipfn)
if (tdp->Zipped)
tjsp = new(g) TDBJSON(tdp, new(g) ZIPFAM(tdp));
else
tjsp = new(g) TDBJSON(tdp, new(g) MAPFAM(tdp));
@@ -139,7 +144,7 @@ PQRYRES JSONColumns(PGLOBAL g, char *db, PTOS topt, bool info)
tdp->Ending = GetIntegerTableOption(g, topt, "Ending", CRLF);
if (tdp->Zipfn)
if (tdp->Zipped)
tjnp = new(g) TDBJSN(tdp, new(g) ZIPFAM(tdp));
else
tjnp = new(g) TDBJSN(tdp, new(g) DOSFAM(tdp));
@@ -424,7 +429,7 @@ PTDB JSONDEF::GetTable(PGLOBAL g, MODE m)
!(tmp == TMP_FORCE &&
(m == MODE_UPDATE || m == MODE_DELETE));
if (Zipfn) {
if (Zipped) {
#if defined(ZIP_SUPPORT)
txfp = new(g) ZIPFAM(this);
#else // !ZIP_SUPPORT
@@ -462,7 +467,7 @@ PTDB JSONDEF::GetTable(PGLOBAL g, MODE m)
((TDBJSN*)tdbp)->G = g;
#endif
} else {
if (Zipfn)
if (Zipped)
txfp = new(g) ZIPFAM(this);
else
txfp = new(g) MAPFAM(this);
@@ -471,7 +476,7 @@ PTDB JSONDEF::GetTable(PGLOBAL g, MODE m)
((TDBJSON*)tdbp)->G = g;
} // endif Pretty
if (Multiple && !Zipfn)
if (Multiple)
tdbp = new(g) TDBMUL(tdbp);
return tdbp;

5
storage/connect/tabxml.cpp
View File

@@ -136,6 +136,11 @@ PQRYRES XMLColumns(PGLOBAL g, char *db, char *tab, PTOS topt, bool info)
goto skipit;
} // endif info
if (GetIntegerTableOption(g, topt, "Multiple", 0)) {
strcpy(g->Message, "Cannot find column definition for multiple table");
return NULL;
} // endif Multiple
/*********************************************************************/
/* Open the input file. */
/*********************************************************************/