database/mariadb
1
0
Fork 0
mirror of https://github.com/MariaDB/server.git synced 2025-08-08 11:22:35 +03:00
Code Activity

- Fix a bug causing CONNECT to loop when expanding a JSON column

Browse Source 
when the expanded column value is null or void array.
- Adding the FullArray option to JSON tables.
- Skipping expanded JSON lines when the expanded column value is null.
  modified:   storage/connect/connect.cc
  modified:   storage/connect/tabdos.h
  modified:   storage/connect/tabjson.cpp
  modified:   storage/connect/tabjson.h
This commit is contained in:
Olivier Bertrand
2018-02-12 16:26:12 +01:00
parent efe80675f5
commit 273233119c
4 changed files with 444 additions and 345 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
storage/connect/connect.cc
View File

@@ -404,6 +404,7 @@ RCODE EvalColumns(PGLOBAL g, PTDB tdbp, bool reset, bool mrr)
rc = RC_FX;
} catch (const char *msg) {
strcpy(g->Message, msg);
rc = RC_NF;
} // end catch
return rc;

1
storage/connect/tabdos.h
View File

@@ -29,6 +29,7 @@ class DllExport DOSDEF : public TABDEF { /* Logical table description */
friend class TXTFAM;
friend class DBFBASE;
friend class UNZIPUTL;
friend class JSONCOL;
public:
// Constructor
DOSDEF(void);

469
storage/connect/tabjson.cpp
View File

@@ -54,16 +54,16 @@
USETEMP UseTemp(void);
char *GetJsonNull(void);
typedef struct _jncol {
struct _jncol *Next;
char *Name;
char *Fmt;
int Type;
int Len;
int Scale;
bool Cbn;
bool Found;
} JCOL, *PJCL;
//typedef struct _jncol {
// struct _jncol *Next;
// char *Name;
// char *Fmt;
// int Type;
// int Len;
// int Scale;
// bool Cbn;
// bool Found;
//} JCOL, *PJCL;
/***********************************************************************/
/* JSONColumns: construct the result blocks containing the description */
@@ -76,26 +76,13 @@ PQRYRES JSONColumns(PGLOBAL g, PCSZ db, PCSZ dsn, PTOS topt, bool info)
static XFLD fldtyp[] = {FLD_NAME, FLD_TYPE, FLD_TYPENAME, FLD_PREC,
FLD_LENGTH, FLD_SCALE, FLD_NULL, FLD_FORMAT};
static unsigned int length[] = {0, 6, 8, 10, 10, 6, 6, 0};
char *p, colname[65], fmt[129];
int i, j, lvl, n = 0;
int i, n = 0;
int ncol = sizeof(buftyp) / sizeof(int);
bool mgo = (GetTypeID(topt->type) == TAB_MONGO);
PCSZ sep, level;
PVAL valp;
JCOL jcol;
PJCL jcp, fjcp = NULL, pjcp = NULL;
PJPR *jrp, jpp;
PJSON jsp;
PJVAL jvp;
PJOB row;
PJDEF tdp;
TDBJSN *tjnp = NULL;
PJTDB tjsp = NULL;
PJCL jcp;
JSONDISC *pjdc = NULL;
PQRYRES qrp;
PCOLRES crp;
jcol.Name = jcol.Fmt = NULL;
if (info) {
length[0] = 128;
length[7] = 256;
@@ -107,10 +94,87 @@ PQRYRES JSONColumns(PGLOBAL g, PCSZ db, PCSZ dsn, PTOS topt, bool info)
return NULL;
} // endif Multiple
pjdc = new(g) JSONDISC(g, (int*)length);
if (!(n = pjdc->GetColumns(g, db, dsn, topt)))
return NULL;
skipit:
if (trace(1))
htrc("JSONColumns: n=%d len=%d\n", n, length[0]);
/*********************************************************************/
/* Open the input file. */
/* Allocate the structures used to refer to the result set. */
/*********************************************************************/
level = GetStringTableOption(g, topt, "Level", NULL);
qrp = PlgAllocResult(g, ncol, n, IDS_COLUMNS + 3,
buftyp, fldtyp, length, false, false);
crp = qrp->Colresp->Next->Next->Next->Next->Next->Next;
crp->Name = "Nullable";
crp->Next->Name = "Jpath";
if (info || !qrp)
return qrp;
qrp->Nblin = n;
/*********************************************************************/
/* Now get the results into blocks. */
/*********************************************************************/
for (i = 0, jcp = pjdc->fjcp; jcp; i++, jcp = jcp->Next) {
if (jcp->Type == TYPE_UNKNOWN)
jcp->Type = TYPE_STRING; // Void column
crp = qrp->Colresp; // Column Name
crp->Kdata->SetValue(jcp->Name, i);
crp = crp->Next; // Data Type
crp->Kdata->SetValue(jcp->Type, i);
crp = crp->Next; // Type Name
crp->Kdata->SetValue(GetTypeName(jcp->Type), i);
crp = crp->Next; // Precision
crp->Kdata->SetValue(jcp->Len, i);
crp = crp->Next; // Length
crp->Kdata->SetValue(jcp->Len, i);
crp = crp->Next; // Scale (precision)
crp->Kdata->SetValue(jcp->Scale, i);
crp = crp->Next; // Nullable
crp->Kdata->SetValue(jcp->Cbn ? 1 : 0, i);
crp = crp->Next; // Field format
if (crp->Kdata)
crp->Kdata->SetValue(jcp->Fmt, i);
} // endfor i
/*********************************************************************/
/* Return the result pointer. */
/*********************************************************************/
return qrp;
} // end of JSONColumns
/* -------------------------- Class JSONDISC ------------------------- */
/***********************************************************************/
/* Class used to get the columns of a JSON table. */
/***********************************************************************/
JSONDISC::JSONDISC(PGLOBAL g, int *lg)
{
length = lg;
jcp = fjcp = pjcp = NULL;
tjnp = NULL;
jpp = NULL;
tjsp = NULL;
jsp = NULL;
row = NULL;
sep = NULL;
i = n = bf = ncol = lvl = 0;
all = false;
} // end of JSONDISC constructor
int JSONDISC::GetColumns(PGLOBAL g, PCSZ db, PCSZ dsn, PTOS topt)
{
bool mgo = (GetTypeID(topt->type) == TAB_MONGO);
PCSZ level = GetStringTableOption(g, topt, "Level", NULL);
if (level) {
lvl = atoi(level);
@@ -120,6 +184,9 @@ PQRYRES JSONColumns(PGLOBAL g, PCSZ db, PCSZ dsn, PTOS topt, bool info)
sep = GetStringTableOption(g, topt, "Separator", ".");
/*********************************************************************/
/* Open the input file. */
/*********************************************************************/
tdp = new(g) JSONDEF;
#if defined(ZIP_SUPPORT)
tdp->Entry = GetStringTableOption(g, topt, "Entry", NULL);
@@ -139,7 +206,7 @@ PQRYRES JSONColumns(PGLOBAL g, PCSZ db, PCSZ dsn, PTOS topt, bool info)
if (!tdp->Fn && !tdp->Uri) {
strcpy(g->Message, MSG(MISSING_FNAME));
return NULL;
return 0;
} // endif Fn
if (trace(1))
@@ -160,7 +227,7 @@ PQRYRES JSONColumns(PGLOBAL g, PCSZ db, PCSZ dsn, PTOS topt, bool info)
tdp->Pretty = 0;
#else // !MONGO_SUPPORT
sprintf(g->Message, MSG(NO_FEAT_SUPPORT), "MONGO");
return NULL;
return 0;
#endif // !MONGO_SUPPORT
} // endif Uri
@@ -170,7 +237,7 @@ PQRYRES JSONColumns(PGLOBAL g, PCSZ db, PCSZ dsn, PTOS topt, bool info)
tjsp = new(g) TDBJSON(tdp, new(g) UNZFAM(tdp));
#else // !ZIP_SUPPORT
sprintf(g->Message, MSG(NO_FEAT_SUPPORT), "ZIP");
return NULL;
return 0;
#endif // !ZIP_SUPPORT
} else
tjsp = new(g) TDBJSON(tdp, new(g) MAPFAM(tdp));
@@ -255,140 +322,28 @@ PQRYRES JSONColumns(PGLOBAL g, PCSZ db, PCSZ dsn, PTOS topt, bool info)
goto err;
} // endif row
all = GetBooleanTableOption(g, topt, "Fullarray", false);
jcol.Name = jcol.Fmt = NULL;
jcol.Next = NULL;
jcol.Found = true;
colname[64] = 0;
fmt[128] = 0;
colname[0] = 0;
if (!tdp->Uri) {
*fmt = '$';
fmt[0] = '$';
fmt[1] = '.';
p = fmt + 2;
} else
p = fmt;
jrp = (PJPR*)PlugSubAlloc(g, NULL, sizeof(PJPR) * MY_MAX(lvl, 0));
bf = 2;
} // endif Uri
/*********************************************************************/
/* Analyse the JSON tree and define columns. */
/*********************************************************************/
for (i = 1; ; i++) {
for (jpp = row->GetFirst(); jpp; jpp = jpp->GetNext()) {
for (j = 0; j < lvl; j++)
jrp[j] = NULL;
more:
strncpy(colname, jpp->GetKey(), 64);
*p = 0;
j = 0;
jvp = jpp->GetVal();
fmt[bf] = 0;
retry:
if ((valp = jvp ? jvp->GetValue() : NULL)) {
jcol.Type = valp->GetType();
jcol.Len = valp->GetValLen();
jcol.Scale = valp->GetValPrec();
jcol.Cbn = valp->IsNull();
} else if (!jvp || jvp->IsNull()) {
jcol.Type = TYPE_UNKNOWN;
jcol.Len = jcol.Scale = 0;
jcol.Cbn = true;
} else if (j < lvl) {
if (!*p)
strcat(fmt, colname);
jsp = jvp->GetJson();
switch (jsp->GetType()) {
case TYPE_JOB:
if (!jrp[j])
jrp[j] = jsp->GetFirst();
if (*jrp[j]->GetKey() != '$') {
strncat(strncat(fmt, sep, 128), jrp[j]->GetKey(), 128);
strncat(strncat(colname, "_", 64), jrp[j]->GetKey(), 64);
} // endif Key
jvp = jrp[j]->GetVal();
j++;
break;
case TYPE_JAR:
if (!tdp->Xcol || stricmp(tdp->Xcol, colname)) {
if (tdp->Uri)
strncat(strncat(fmt, sep, 128), "0", 128);
else
strncat(fmt, "[0]", 128);
} else
strncat(fmt, (tdp->Uri ? sep : "[]"), 128);
jvp = jsp->GetValue(0);
break;
default:
sprintf(g->Message, "Logical error after %s", fmt);
if (Find(g, jpp->GetVal(), MY_MIN(lvl, 0)))
goto err;
} // endswitch jsp
goto retry;
} else if (lvl >= 0) {
jcol.Type = TYPE_STRING;
jcol.Len = 256;
jcol.Scale = 0;
jcol.Cbn = true;
} else
continue;
// Check whether this column was already found
for (jcp = fjcp; jcp; jcp = jcp->Next)
if (!strcmp(colname, jcp->Name))
break;
if (jcp) {
if (jcp->Type != jcol.Type) {
if (jcp->Type == TYPE_UNKNOWN)
jcp->Type = jcol.Type;
else if (jcol.Type != TYPE_UNKNOWN)
jcp->Type = TYPE_STRING;
} // endif Type
if (*p && (!jcp->Fmt || strlen(jcp->Fmt) < strlen(fmt))) {
jcp->Fmt = PlugDup(g, fmt);
length[7] = MY_MAX(length[7], strlen(fmt));
} // endif fmt
jcp->Len = MY_MAX(jcp->Len, jcol.Len);
jcp->Scale = MY_MAX(jcp->Scale, jcol.Scale);
jcp->Cbn |= jcol.Cbn;
jcp->Found = true;
} else if (jcol.Type != TYPE_UNKNOWN || tdp->Accept) {
// New column
jcp = (PJCL)PlugSubAlloc(g, NULL, sizeof(JCOL));
*jcp = jcol;
jcp->Cbn |= (i > 1);
jcp->Name = PlugDup(g, colname);
length[0] = MY_MAX(length[0], strlen(colname));
if (*p) {
jcp->Fmt = PlugDup(g, fmt);
length[7] = MY_MAX(length[7], strlen(fmt));
} else
jcp->Fmt = NULL;
if (pjcp) {
jcp->Next = pjcp->Next;
pjcp->Next = jcp;
} else
fjcp = jcp;
n++;
} // endif jcp
pjcp = jcp;
for (j = lvl - 1; j >= 0; j--)
if (jrp[j] && (jrp[j] = jrp[j]->GetNext()))
goto more;
} // endfor jpp
@@ -416,69 +371,160 @@ PQRYRES JSONColumns(PGLOBAL g, PCSZ db, PCSZ dsn, PTOS topt, bool info)
if (!(row = (jsp) ? jsp->GetObject() : NULL))
break;
} // endor i
} // endfor i
if (tdp->Pretty != 2)
tjnp->CloseDB(g);
skipit:
if (trace(1))
htrc("JSONColumns: n=%d len=%d\n", n, length[0]);
/*********************************************************************/
/* Allocate the structures used to refer to the result set. */
/*********************************************************************/
qrp = PlgAllocResult(g, ncol, n, IDS_COLUMNS + 3,
buftyp, fldtyp, length, false, false);
crp = qrp->Colresp->Next->Next->Next->Next->Next->Next;
crp->Name = "Nullable";
crp->Next->Name = "Jpath";
if (info || !qrp)
return qrp;
qrp->Nblin = n;
/*********************************************************************/
/* Now get the results into blocks. */
/*********************************************************************/
for (i = 0, jcp = fjcp; jcp; i++, jcp = jcp->Next) {
if (jcp->Type == TYPE_UNKNOWN)
jcp->Type = TYPE_STRING; // Void column
crp = qrp->Colresp; // Column Name
crp->Kdata->SetValue(jcp->Name, i);
crp = crp->Next; // Data Type
crp->Kdata->SetValue(jcp->Type, i);
crp = crp->Next; // Type Name
crp->Kdata->SetValue(GetTypeName(jcp->Type), i);
crp = crp->Next; // Precision
crp->Kdata->SetValue(jcp->Len, i);
crp = crp->Next; // Length
crp->Kdata->SetValue(jcp->Len, i);
crp = crp->Next; // Scale (precision)
crp->Kdata->SetValue(jcp->Scale, i);
crp = crp->Next; // Nullable
crp->Kdata->SetValue(jcp->Cbn ? 1 : 0, i);
crp = crp->Next; // Field format
if (crp->Kdata)
crp->Kdata->SetValue(jcp->Fmt, i);
} // endfor i
/*********************************************************************/
/* Return the result pointer. */
/*********************************************************************/
return qrp;
return n;
err:
if (tdp->Pretty != 2)
tjnp->CloseDB(g);
return NULL;
} // end of JSONColumns
return 0;
} // end of GetColumns
bool JSONDISC::Find(PGLOBAL g, PJVAL jvp, int j)
{
char *p, *pc = colname + strlen(colname);
int ars;
PJOB job;
PJAR jar;
if ((valp = jvp ? jvp->GetValue() : NULL)) {
jcol.Type = valp->GetType();
jcol.Len = valp->GetValLen();
jcol.Scale = valp->GetValPrec();
jcol.Cbn = valp->IsNull();
} else if (!jvp || jvp->IsNull()) {
jcol.Type = TYPE_UNKNOWN;
jcol.Len = jcol.Scale = 0;
jcol.Cbn = true;
} else if (j < lvl) {
if (!fmt[bf])
strcat(fmt, colname);
p = fmt + strlen(fmt);
jsp = jvp->GetJson();
switch (jsp->GetType()) {
case TYPE_JOB:
job = (PJOB)jsp;
for (PJPR jrp = job->GetFirst(); jrp; jrp = jrp->GetNext()) {
if (*jrp->GetKey() != '$') {
strncat(strncat(fmt, sep, 128), jrp->GetKey(), 128);
strncat(strncat(colname, "_", 64), jrp->GetKey(), 64);
} // endif Key
if (Find(g, jrp->GetVal(), j + 1))
return true;
*p = *pc = 0;
} // endfor jrp
return false;
case TYPE_JAR:
jar = (PJAR)jsp;
if (all || (tdp->Xcol && !stricmp(tdp->Xcol, colname)))
ars = jar->GetSize(false);
else
ars = MY_MIN(jar->GetSize(false), 1);
for (int k = 0; k < ars; k++) {
if (!tdp->Xcol || stricmp(tdp->Xcol, colname)) {
sprintf(buf, "%d", k);
if (tdp->Uri)
strncat(strncat(fmt, sep, 128), buf, 128);
else
strncat(strncat(strncat(fmt, "[", 128), buf, 128), "]", 128);
if (all)
strncat(strncat(colname, "_", 64), buf, 64);
} else
strncat(fmt, (tdp->Uri ? sep : "[*]"), 128);
if (Find(g, jar->GetValue(k), j))
return true;
*p = *pc = 0;
} // endfor k
return false;
default:
sprintf(g->Message, "Logical error after %s", fmt);
return true;
} // endswitch Type
} else if (lvl >= 0) {
jcol.Type = TYPE_STRING;
jcol.Len = 256;
jcol.Scale = 0;
jcol.Cbn = true;
} else
return false;
AddColumn(g);
return false;
} // end of Find
void JSONDISC::AddColumn(PGLOBAL g)
{
bool b = fmt[bf] != 0; // True if formatted
// Check whether this column was already found
for (jcp = fjcp; jcp; jcp = jcp->Next)
if (!strcmp(colname, jcp->Name))
break;
if (jcp) {
if (jcp->Type != jcol.Type) {
if (jcp->Type == TYPE_UNKNOWN)
jcp->Type = jcol.Type;
else if (jcol.Type != TYPE_UNKNOWN)
jcp->Type = TYPE_STRING;
} // endif Type
if (b && (!jcp->Fmt || strlen(jcp->Fmt) < strlen(fmt))) {
jcp->Fmt = PlugDup(g, fmt);
length[7] = MY_MAX(length[7], strlen(fmt));
} // endif fmt
jcp->Len = MY_MAX(jcp->Len, jcol.Len);
jcp->Scale = MY_MAX(jcp->Scale, jcol.Scale);
jcp->Cbn |= jcol.Cbn;
jcp->Found = true;
} else if (jcol.Type != TYPE_UNKNOWN || tdp->Accept) {
// New column
jcp = (PJCL)PlugSubAlloc(g, NULL, sizeof(JCOL));
*jcp = jcol;
jcp->Cbn |= (i > 1);
jcp->Name = PlugDup(g, colname);
length[0] = MY_MAX(length[0], strlen(colname));
if (b) {
jcp->Fmt = PlugDup(g, fmt);
length[7] = MY_MAX(length[7], strlen(fmt));
} else
jcp->Fmt = NULL;
if (pjcp) {
jcp->Next = pjcp->Next;
pjcp->Next = jcp;
} else
fjcp = jcp;
n++;
} // endif jcp
pjcp = jcp;
} // end of AddColumn
/* -------------------------- Class JSONDEF -------------------------- */
@@ -513,6 +559,7 @@ bool JSONDEF::DefineAM(PGLOBAL g, LPCSTR, int poff)
Limit = GetIntCatInfo("Limit", 10);
Base = GetIntCatInfo("Base", 0) ? 1 : 0;
Sep = *GetStringCatInfo(g, "Separator", ".");
Accept = GetBoolCatInfo("Accept", false);
if (Uri = GetStringCatInfo(g, "Connect", NULL)) {
#if defined(JAVA_SUPPORT) || defined(CMGO_SUPPORT)
@@ -1471,6 +1518,9 @@ void JSONCOL::ReadColumn(PGLOBAL g)
if (!Tjp->SameRow || Xnod >= Tjp->SameRow)
Value->SetValue_pval(GetColumnValue(g, Tjp->Row, 0));
if (Xpd && Value->IsNull() && !((PJDEF)Tjp->To_Def)->Accept)
throw("Null expandable JSON value");
// Set null when applicable
if (!Nullable)
Value->SetNull(false);
@@ -1546,11 +1596,16 @@ PVAL JSONCOL::GetColumnValue(PGLOBAL g, PJSON row, int i)
/***********************************************************************/
PVAL JSONCOL::ExpandArray(PGLOBAL g, PJAR arp, int n)
{
int ars;
int ars = MY_MIN(Tjp->Limit, arp->size());
PJVAL jvp;
JVALUE jval;
ars = MY_MIN(Tjp->Limit, arp->size());
if (!ars) {
Value->Reset();
Value->SetNull(true);
Tjp->NextSame = 0;
return Value;
} // endif ars
if (!(jvp = arp->GetValue((Nodes[n].Rx = Nodes[n].Nx)))) {
strcpy(g->Message, "Logical error expanding array");

44
storage/connect/tabjson.h
View File

@@ -15,6 +15,7 @@ enum JMODE {MODE_OBJECT, MODE_ARRAY, MODE_VALUE};
typedef class JSONDEF *PJDEF;
typedef class TDBJSON *PJTDB;
typedef class JSONCOL *PJCOL;
class TDBJSN;
/***********************************************************************/
/* The JSON tree node. Can be an Object or an Array. */
@@ -29,6 +30,47 @@ typedef struct _jnode {
int Nx; // Next to read row number
} JNODE, *PJNODE;
typedef struct _jncol {
struct _jncol *Next;
char *Name;
char *Fmt;
int Type;
int Len;
int Scale;
bool Cbn;
bool Found;
} JCOL, *PJCL;
/***********************************************************************/
/* Class used to get the columns of a mongo collection. */
/***********************************************************************/
class JSONDISC : public BLOCK {
public:
// Constructor
JSONDISC(PGLOBAL g, int *lg);
// Functions
int GetColumns(PGLOBAL g, PCSZ db, PCSZ dsn, PTOS topt);
bool Find(PGLOBAL g, PJVAL jvp, int j);
void AddColumn(PGLOBAL g);
// Members
JCOL jcol;
PJCL jcp, fjcp, pjcp;
PVAL valp;
PJDEF tdp;
TDBJSN *tjnp;
PJTDB tjsp;
PJPR jpp;
PJSON jsp;
PJOB row;
PCSZ sep;
char colname[65], fmt[129], buf[16];
int *length;
int i, n, bf, ncol, lvl;
bool all;
}; // end of JSONDISC
/***********************************************************************/
/* JSON table. */
/***********************************************************************/
@@ -36,13 +78,13 @@ class DllExport JSONDEF : public DOSDEF { /* Table description */
friend class TDBJSON;
friend class TDBJSN;
friend class TDBJCL;
friend class JSONDISC;
#if defined(CMGO_SUPPORT)
friend class CMGFAM;
#endif // CMGO_SUPPORT
#if defined(JAVA_SUPPORT)
friend class JMGFAM;
#endif // JAVA_SUPPORT
friend PQRYRES JSONColumns(PGLOBAL, PCSZ, PCSZ, PTOS, bool);
public:
// Constructor
JSONDEF(void);