From 671d9b6c611be423246b562878da2a5788cc327d Mon Sep 17 00:00:00 2001 From: Olivier Bertrand Date: Thu, 16 Jul 2020 16:30:54 +0200 Subject: [PATCH 01/31] - Fix MDEV-22571 and MDEV-22572. Allow multiple ZIP table and enable using special column in them. modified: storage/connect/tabzip.cpp modified: storage/connect/tabzip.h - Fix some compiler errors modified: storage/connect/tabcmg.cpp --- storage/connect/connect.cc | 1 - storage/connect/filamdbf.cpp | 195 +++++++++++++++++------ storage/connect/filamdbf.h | 2 +- storage/connect/filamzip.cpp | 282 ++++++++++++++++++++++++++++++++-- storage/connect/filamzip.h | 38 ++++- storage/connect/ha_connect.cc | 19 +-- storage/connect/mongo.cpp | 1 - storage/connect/mongo.h | 1 - storage/connect/plgxml.cpp | 2 +- storage/connect/tabcmg.cpp | 2 + storage/connect/tabdos.cpp | 36 +++-- storage/connect/tabdos.h | 1 + storage/connect/tabfix.h | 18 ++- storage/connect/tabjson.cpp | 1 + storage/connect/tabjson.h | 1 - storage/connect/tabzip.cpp | 13 +- storage/connect/tabzip.h | 2 + 17 files changed, 509 insertions(+), 106 deletions(-) diff --git a/storage/connect/connect.cc b/storage/connect/connect.cc index dfc619cf4af..2a0f2ed037f 100644 --- a/storage/connect/connect.cc +++ b/storage/connect/connect.cc @@ -355,7 +355,6 @@ bool CntOpenTable(PGLOBAL g, PTDB tdbp, MODE mode, char *c1, char *c2, } // endif mode rcop = false; - } catch (int n) { if (trace(1)) htrc("Exception %d: %s\n", n, g->Message); diff --git a/storage/connect/filamdbf.cpp b/storage/connect/filamdbf.cpp index c8bab2b53a4..542159fd172 100644 --- a/storage/connect/filamdbf.cpp +++ b/storage/connect/filamdbf.cpp @@ -49,6 +49,7 @@ #include "global.h" #include "plgdbsem.h" #include "filamdbf.h" +#include "filamzip.h" #include "tabdos.h" #include "valblk.h" #define NO_FUNC @@ -139,7 +140,7 @@ static int dbfhead(PGLOBAL g, FILE *file, PCSZ fn, DBFHEADER *buf) if (fread(buf, HEADLEN, 1, file) != 1) { strcpy(g->Message, MSG(NO_READ_32)); return RC_NF; - } // endif fread + } // endif fread // Check first byte to be sure of .dbf type if ((buf->Version & 0x03) != DBFTYPE) { @@ -149,7 +150,7 @@ static int dbfhead(PGLOBAL g, FILE *file, PCSZ fn, DBFHEADER *buf) if ((buf->Version & 0x30) == 0x30) { strcpy(g->Message, MSG(FOXPRO_FILE)); dbc = 264; // FoxPro database container - } // endif Version + } // endif Version } else strcpy(g->Message, MSG(DBASE_FILE)); @@ -158,12 +159,12 @@ static int dbfhead(PGLOBAL g, FILE *file, PCSZ fn, DBFHEADER *buf) if (fseek(file, buf->Headlen() - dbc, SEEK_SET) != 0) { sprintf(g->Message, MSG(BAD_HEADER), fn); return RC_FX; - } // endif fseek + } // endif fseek if (fread(&endmark, 2, 1, file) != 1) { strcpy(g->Message, MSG(BAD_HEAD_END)); return RC_FX; - } // endif fread + } // endif fread // Some files have just 1D others have 1D00 following fields if (endmark[0] != EOH && endmark[1] != EOH) { @@ -172,7 +173,7 @@ static int dbfhead(PGLOBAL g, FILE *file, PCSZ fn, DBFHEADER *buf) if (rc == RC_OK) return RC_FX; - } // endif endmark + } // endif endmark // Calculate here the number of fields while we have the dbc info buf->SetFields((buf->Headlen() - dbc - 1) / 32); @@ -180,13 +181,58 @@ static int dbfhead(PGLOBAL g, FILE *file, PCSZ fn, DBFHEADER *buf) return rc; } // end of dbfhead +/****************************************************************************/ +/* dbfields: Analyze a DBF header and set the table fields number. */ +/* Parameters: */ +/* PGLOBAL g -- pointer to the CONNECT Global structure */ +/* DBFHEADER *hdrp -- pointer to _dbfheader structure */ +/* Returns: */ +/* RC_OK, RC_INFO, or RC_FX if error. */ +/****************************************************************************/ +static int dbfields(PGLOBAL g, DBFHEADER* hdrp) +{ + char* endmark; + int dbc = 2, rc = RC_OK; + + *g->Message = '\0'; + + // Check first byte to be sure of .dbf type + if ((hdrp->Version & 0x03) != DBFTYPE) { + strcpy(g->Message, MSG(NOT_A_DBF_FILE)); + rc = RC_INFO; + + if ((hdrp->Version & 0x30) == 0x30) { + strcpy(g->Message, MSG(FOXPRO_FILE)); + dbc = 264; // FoxPro database container + } // endif Version + + } else + strcpy(g->Message, MSG(DBASE_FILE)); + + // Check last byte(s) of header + endmark = (char*)hdrp + hdrp->Headlen() - dbc; + + // Some headers just have 1D others have 1D00 following fields + if (endmark[0] != EOH && endmark[1] != EOH) { + sprintf(g->Message, MSG(NO_0DH_HEAD), dbc); + + if (rc == RC_OK) + return RC_FX; + + } // endif endmark + + // Calculate here the number of fields while we have the dbc info + hdrp->SetFields((hdrp->Headlen() - dbc - 1) / 32); + return rc; +} // end of dbfields + /* -------------------------- Function DBFColumns ------------------------- */ /****************************************************************************/ /* DBFColumns: constructs the result blocks containing the description */ /* of all the columns of a DBF file that will be retrieved by #GetData. */ /****************************************************************************/ -PQRYRES DBFColumns(PGLOBAL g, PCSZ dp, PCSZ fn, bool info) +PQRYRES DBFColumns(PGLOBAL g, PCSZ dp, PCSZ fn, PTOS topt, bool info) { int buftyp[] = {TYPE_STRING, TYPE_SHORT, TYPE_STRING, TYPE_INT, TYPE_INT, TYPE_SHORT}; @@ -196,10 +242,12 @@ PQRYRES DBFColumns(PGLOBAL g, PCSZ dp, PCSZ fn, bool info) char buf[2], filename[_MAX_PATH]; int ncol = sizeof(buftyp) / sizeof(int); int rc, type, len, field, fields; - bool bad; - DBFHEADER mainhead; - DESCRIPTOR thisfield; - FILE *infile = NULL; + bool bad, mul; + PCSZ target, pwd; + DBFHEADER mainhead, *hp; + DESCRIPTOR thisfield, *tfp; + FILE *infile = NULL; + UNZIPUTL *zutp = NULL; PQRYRES qrp; PCOLRES crp; @@ -217,21 +265,55 @@ PQRYRES DBFColumns(PGLOBAL g, PCSZ dp, PCSZ fn, bool info) /************************************************************************/ PlugSetPath(filename, fn, dp); - if (!(infile= global_fopen(g, MSGID_CANNOT_OPEN, filename, "rb"))) - return NULL; + if (topt->zipped) { + target = GetStringTableOption(g, topt, "Entry", NULL); + mul = (target && *target) ? strchr(target, '*') || strchr(target, '?') + : false; + mul = GetBooleanTableOption(g, topt, "Mulentries", mul); - /************************************************************************/ - /* Get the first 32 bytes of the header. */ - /************************************************************************/ - if ((rc = dbfhead(g, infile, filename, &mainhead)) == RC_FX) { - fclose(infile); - return NULL; - } // endif dbfhead + if (mul) { + strcpy(g->Message, "Cannot find column definition for multiple entries"); + return NULL; + } // endif Multiple - /************************************************************************/ - /* Allocate the structures used to refer to the result set. */ - /************************************************************************/ - fields = mainhead.Fields(); + pwd = GetStringTableOption(g, topt, "Password", NULL); + zutp = new(g) UNZIPUTL(target, pwd, mul); + + if (!zutp->OpenTable(g, MODE_READ, filename)) + hp = (DBFHEADER*)zutp->memory; + else + return NULL; + + /**********************************************************************/ + /* Set the table fields number. */ + /**********************************************************************/ + if ((rc = dbfields(g, hp)) == RC_FX) { + zutp->close(); + return NULL; + } // endif dbfields + + tfp = (DESCRIPTOR*)hp; + } else { + if (!(infile = global_fopen(g, MSGID_CANNOT_OPEN, filename, "rb"))) + return NULL; + else + hp = &mainhead; + + /**********************************************************************/ + /* Get the first 32 bytes of the header. */ + /**********************************************************************/ + if ((rc = dbfhead(g, infile, filename, hp)) == RC_FX) { + fclose(infile); + return NULL; + } // endif dbfhead + + tfp = &thisfield; + } // endif zipped + + /************************************************************************/ + /* Get the number of the table fields. */ + /************************************************************************/ + fields = hp->Fields(); } else fields = 0; @@ -241,19 +323,21 @@ PQRYRES DBFColumns(PGLOBAL g, PCSZ dp, PCSZ fn, bool info) if (info || !qrp) { if (infile) fclose(infile); + else if (zutp) + zutp->close(); return qrp; - } // endif info + } // endif info if (trace(1)) { htrc("Structure of %s\n", filename); htrc("headlen=%hd reclen=%hd degree=%d\n", - mainhead.Headlen(), mainhead.Reclen(), fields); - htrc("flags(iem)=%d,%d,%d cp=%d\n", mainhead.Incompleteflag, - mainhead.Encryptflag, mainhead.Mdxflag, mainhead.Language); + hp->Headlen(), hp->Reclen(), fields); + htrc("flags(iem)=%d,%d,%d cp=%d\n", hp->Incompleteflag, + hp->Encryptflag, hp->Mdxflag, hp->Language); htrc("%hd records, last changed %02d/%02d/%d\n", - mainhead.Records(), mainhead.Filedate[1], mainhead.Filedate[2], - mainhead.Filedate[0] + (mainhead.Filedate[0] <= 30) ? 2000 : 1900); + hp->Records(), hp->Filedate[1], hp->Filedate[2], + hp->Filedate[0] + (hp->Filedate[0] <= 30) ? 2000 : 1900); htrc("Field Type Offset Len Dec Set Mdx\n"); } // endif trace @@ -265,21 +349,24 @@ PQRYRES DBFColumns(PGLOBAL g, PCSZ dp, PCSZ fn, bool info) for (field = 0; field < fields; field++) { bad = FALSE; - if (fread(&thisfield, HEADLEN, 1, infile) != 1) { + if (topt->zipped) { + tfp = (DESCRIPTOR*)((char*)tfp + HEADLEN); + } else if (fread(tfp, HEADLEN, 1, infile) != 1) { sprintf(g->Message, MSG(ERR_READING_REC), field+1, fn); goto err; - } else - len = thisfield.Length; + } // endif fread + + len = tfp->Length; if (trace(1)) htrc("%-11s %c %6ld %3d %2d %3d %3d\n", - thisfield.Name, thisfield.Type, thisfield.Offset, len, - thisfield.Decimals, thisfield.Setfield, thisfield.Mdxfield); + tfp->Name, tfp->Type, tfp->Offset, len, + tfp->Decimals, tfp->Setfield, tfp->Mdxfield); /************************************************************************/ /* Now get the results into blocks. */ /************************************************************************/ - switch (thisfield.Type) { + switch (tfp->Type) { case 'C': // Characters case 'L': // Logical 'T' or 'F' or space type = TYPE_STRING; @@ -294,7 +381,7 @@ PQRYRES DBFColumns(PGLOBAL g, PCSZ dp, PCSZ fn, bool info) // type = TYPE_INT; // break; case 'N': - type = (thisfield.Decimals) ? TYPE_DOUBLE + type = (tfp->Decimals) ? TYPE_DOUBLE : (len > 10) ? TYPE_BIGINT : TYPE_INT; break; case 'F': // Float @@ -306,8 +393,8 @@ PQRYRES DBFColumns(PGLOBAL g, PCSZ dp, PCSZ fn, bool info) break; default: if (!info) { - sprintf(g->Message, MSG(BAD_DBF_TYPE), thisfield.Type - , thisfield.Name); + sprintf(g->Message, MSG(BAD_DBF_TYPE), tfp->Type + , tfp->Name); goto err; } // endif info @@ -316,27 +403,31 @@ PQRYRES DBFColumns(PGLOBAL g, PCSZ dp, PCSZ fn, bool info) } // endswitch Type crp = qrp->Colresp; // Column Name - crp->Kdata->SetValue(thisfield.Name, field); + crp->Kdata->SetValue(tfp->Name, field); crp = crp->Next; // Data Type crp->Kdata->SetValue((int)type, field); crp = crp->Next; // Type Name if (bad) { - buf[0] = thisfield.Type; + buf[0] = tfp->Type; crp->Kdata->SetValue(buf, field); } else crp->Kdata->SetValue(GetTypeName(type), field); crp = crp->Next; // Precision - crp->Kdata->SetValue((int)thisfield.Length, field); + crp->Kdata->SetValue((int)tfp->Length, field); crp = crp->Next; // Length - crp->Kdata->SetValue((int)thisfield.Length, field); + crp->Kdata->SetValue((int)tfp->Length, field); crp = crp->Next; // Scale (precision) - crp->Kdata->SetValue((int)thisfield.Decimals, field); + crp->Kdata->SetValue((int)tfp->Decimals, field); } // endfor field qrp->Nblin = field; - fclose(infile); + + if (infile) + fclose(infile); + else if (zutp) + zutp->close(); #if 0 if (info) { @@ -347,9 +438,9 @@ PQRYRES DBFColumns(PGLOBAL g, PCSZ dp, PCSZ fn, bool info) sprintf(buf, "Ver=%02x ncol=%hu nlin=%u lrecl=%hu headlen=%hu date=%02d/%02d/%02d", - mainhead.Version, fields, mainhead.Records, mainhead.Reclen, - mainhead.Headlen, mainhead.Filedate[0], mainhead.Filedate[1], - mainhead.Filedate[2]); + hp->Version, fields, hp->Records, hp->Reclen, + hp->Headlen, hp->Filedate[0], hp->Filedate[1], + hp->Filedate[2]); strcat(g->Message, buf); } // endif info @@ -360,9 +451,13 @@ PQRYRES DBFColumns(PGLOBAL g, PCSZ dp, PCSZ fn, bool info) /**************************************************************************/ return qrp; - err: - fclose(infile); - return NULL; +err: + if (infile) + fclose(infile); + else if (zutp) + zutp->close(); + + return NULL; } // end of DBFColumns /* ---------------------------- Class DBFBASE ----------------------------- */ diff --git a/storage/connect/filamdbf.h b/storage/connect/filamdbf.h index 640fc349b4c..dfe5cb5cfc4 100644 --- a/storage/connect/filamdbf.h +++ b/storage/connect/filamdbf.h @@ -19,7 +19,7 @@ typedef class DBMFAM *PDBMFAM; /****************************************************************************/ /* Functions used externally. */ /****************************************************************************/ -PQRYRES DBFColumns(PGLOBAL g, PCSZ dp, PCSZ fn, bool info); +PQRYRES DBFColumns(PGLOBAL g, PCSZ dp, PCSZ fn, PTOS tiop, bool info); /****************************************************************************/ /* This is the base class for dBASE file access methods. */ diff --git a/storage/connect/filamzip.cpp b/storage/connect/filamzip.cpp index e76dc496246..eeb23e1f053 100644 --- a/storage/connect/filamzip.cpp +++ b/storage/connect/filamzip.cpp @@ -1,11 +1,11 @@ /*********** File AM Zip C++ Program Source Code File (.CPP) ***********/ /* PROGRAM NAME: FILAMZIP */ /* ------------- */ -/* Version 1.3 */ +/* Version 1.4 */ /* */ /* COPYRIGHT: */ /* ---------- */ -/* (C) Copyright to the author Olivier BERTRAND 2016-2017 */ +/* (C) Copyright to the author Olivier BERTRAND 2016-2020 */ /* */ /* WHAT THIS PROGRAM DOES: */ /* ----------------------- */ @@ -45,6 +45,62 @@ #define WRITEBUFFERSIZE (16384) +/****************************************************************************/ +/* Definitions used for DBF tables. */ +/****************************************************************************/ +#define HEADLEN 32 /* sizeof ( mainhead or thisfield ) */ +//efine MEMOLEN 10 /* length of memo field in .dbf */ +#define DBFTYPE 3 /* value of bits 0 and 1 if .dbf */ +#define EOH 0x0D /* end-of-header marker in .dbf file */ + +/****************************************************************************/ +/* First 32 bytes of a DBF table. */ +/* Note: some reserved fields are used here to store info (Fields) */ +/****************************************************************************/ +typedef struct _dbfheader { + uchar Version; /* Version information flags */ + char Filedate[3]; /* date, YYMMDD, binary. YY=year-1900 */ +private: + /* The following four members are stored in little-endian format on disk */ + char m_RecordsBuf[4]; /* records in the file */ + char m_HeadlenBuf[2]; /* bytes in the header */ + char m_ReclenBuf[2]; /* bytes in a record */ + char m_FieldsBuf[2]; /* Reserved but used to store fields */ +public: + char Incompleteflag; /* 01 if incomplete, else 00 */ + char Encryptflag; /* 01 if encrypted, else 00 */ + char Reserved2[12]; /* for LAN use */ + char Mdxflag; /* 01 if production .mdx, else 00 */ + char Language; /* Codepage */ + char Reserved3[2]; + + uint Records(void) const { return uint4korr(m_RecordsBuf); } + ushort Headlen(void) const { return uint2korr(m_HeadlenBuf); } + ushort Reclen(void) const { return uint2korr(m_ReclenBuf); } + ushort Fields(void) const { return uint2korr(m_FieldsBuf); } + + void SetHeadlen(ushort num) { int2store(m_HeadlenBuf, num); } + void SetReclen(ushort num) { int2store(m_ReclenBuf, num); } + void SetFields(ushort num) { int2store(m_FieldsBuf, num); } +} DBFHEADER; + +/****************************************************************************/ +/* Column field descriptor of a .dbf file. */ +/****************************************************************************/ +typedef struct _descriptor { + char Name[11]; /* field name, in capitals, null filled*/ + char Type; /* field type, C, D, F, L, M or N */ + uint Offset; /* used in memvars, not in files. */ + uchar Length; /* field length */ + uchar Decimals; /* number of decimal places */ + short Reserved4; + char Workarea; /* ??? */ + char Reserved5[2]; + char Setfield; /* ??? */ + char Reserved6[7]; + char Mdxfield; /* 01 if tag field in production .mdx */ +} DESCRIPTOR; + bool ZipLoadFile(PGLOBAL g, PCSZ zfn, PCSZ fn, PCSZ entry, bool append, bool mul); /***********************************************************************/ @@ -214,10 +270,21 @@ bool ZipLoadFile(PGLOBAL g, PCSZ zfn, PCSZ fn, PCSZ entry, bool append, bool mul buf = (char*)PlugSubAlloc(g, NULL, WRITEBUFFERSIZE); - if (mul) - err = ZipFiles(g, zutp, fn, buf); - else - err = ZipFile(g, zutp, fn, entry, buf); + if (!mul) { + PCSZ entp; + + if (!entry) { // entry defaults to the file name + char* p = strrchr((char*)fn, '/'); +#if defined(__WIN__) + if (!p) p = strrchr((char*)fn, '\\'); +#endif // __WIN__ + entp = (p) ? p + 1 : entry; + } else + entp = entry; + + err = ZipFile(g, zutp, fn, entp, buf); + } else + err = ZipFiles(g, zutp, fn, buf); zutp->close(); return err; @@ -232,6 +299,7 @@ ZIPUTIL::ZIPUTIL(PCSZ tgt) { zipfile = NULL; target = tgt; + pwd = NULL; fp = NULL; entryopen = false; } // end of ZIPUTIL standard constructor @@ -241,6 +309,7 @@ ZIPUTIL::ZIPUTIL(ZIPUTIL *zutp) { zipfile = zutp->zipfile; target = zutp->target; + pwd = zutp->pwd; fp = zutp->fp; entryopen = zutp->entryopen; } // end of UNZIPUTL copy constructor @@ -385,11 +454,11 @@ void ZIPUTIL::closeEntry() /***********************************************************************/ /* Constructors. */ /***********************************************************************/ -UNZIPUTL::UNZIPUTL(PCSZ tgt, bool mul) +UNZIPUTL::UNZIPUTL(PCSZ tgt, PCSZ pw, bool mul) { zipfile = NULL; target = tgt; - pwd = NULL; + pwd = pw; fp = NULL; memory = NULL; size = 0; @@ -959,7 +1028,7 @@ int UZXFAM::Cardinality(PGLOBAL g) } // end of Cardinality /***********************************************************************/ -/* OpenTableFile: Open a DOS/UNIX table file from a ZIP file. */ +/* OpenTableFile: Open a FIX/UNIX table file from a ZIP file. */ /***********************************************************************/ bool UZXFAM::OpenTableFile(PGLOBAL g) { @@ -1015,6 +1084,197 @@ int UZXFAM::GetNext(PGLOBAL g) return RC_OK; } // end of GetNext +/* -------------------------- class UZDFAM --------------------------- */ + +/***********************************************************************/ +/* Constructors. */ +/***********************************************************************/ +UZDFAM::UZDFAM(PDOSDEF tdp) : DBMFAM(tdp) +{ + zutp = NULL; + tdfp = tdp; + //target = tdp->GetEntry(); + //mul = tdp->GetMul(); + //Lrecl = tdp->GetLrecl(); +} // end of UZXFAM standard constructor + +UZDFAM::UZDFAM(PUZDFAM txfp) : DBMFAM(txfp) +{ + zutp = txfp->zutp; + tdfp = txfp->tdfp; + //target = txfp->target; + //mul = txfp->mul; + //Lrecl = txfp->Lrecl; +} // end of UZXFAM copy constructor + +#if 0 +/****************************************************************************/ +/* dbfhead: Routine to analyze a DBF header. */ +/* Parameters: */ +/* PGLOBAL g -- pointer to the CONNECT Global structure */ +/* DBFHEADER *hdrp -- pointer to _dbfheader structure */ +/* Returns: */ +/* RC_OK, RC_NF, RC_INFO, or RC_FX if error. */ +/* Side effects: */ +/* Set the fields number in the header. */ +/****************************************************************************/ +int UZDFAM::dbfhead(PGLOBAL g, void* buf) +{ + char *endmark; + int dbc = 2, rc = RC_OK; + DBFHEADER* hdrp = (DBFHEADER*)buf; + + *g->Message = '\0'; + + // Check first byte to be sure of .dbf type + if ((hdrp->Version & 0x03) != DBFTYPE) { + strcpy(g->Message, MSG(NOT_A_DBF_FILE)); + rc = RC_INFO; + + if ((hdrp->Version & 0x30) == 0x30) { + strcpy(g->Message, MSG(FOXPRO_FILE)); + dbc = 264; // FoxPro database container + } // endif Version + + } else + strcpy(g->Message, MSG(DBASE_FILE)); + + // Check last byte(s) of header + endmark = (char*)hdrp + hdrp->Headlen() - dbc; + + // Some headers just have 1D others have 1D00 following fields + if (endmark[0] != EOH && endmark[1] != EOH) { + sprintf(g->Message, MSG(NO_0DH_HEAD), dbc); + + if (rc == RC_OK) + return RC_FX; + + } // endif endmark + + // Calculate here the number of fields while we have the dbc info + hdrp->SetFields((hdrp->Headlen() - dbc - 1) / 32); + return rc; +} // end of dbfhead + +/****************************************************************************/ +/* ScanHeader: scan the DBF file header for number of records, record size,*/ +/* and header length. Set Records, check that Reclen is equal to lrecl and */ +/* return the header length or 0 in case of error. */ +/****************************************************************************/ +int UZDFAM::ScanHeader(PGLOBAL g, int* rln) +{ + int rc; + DBFHEADER header; + + /************************************************************************/ + /* Get the first 32 bytes of the header. */ + /************************************************************************/ + rc = dbfhead(g, &header); + + if (rc == RC_FX) + return -1; + + *rln = (int)header.Reclen(); + Records = (int)header.Records(); + return (int)header.Headlen(); +} // end of ScanHeader +#endif // 0 + +/***********************************************************************/ +/* ZIP GetFileLength: returns file size in number of bytes. */ +/***********************************************************************/ +int UZDFAM::GetFileLength(PGLOBAL g) +{ + int len; + + if (!zutp && OpenTableFile(g)) + return 0; + + if (zutp->entryopen) + len = zutp->size; + else + len = 0; + + return len; +} // end of GetFileLength + +/***********************************************************************/ +/* ZIP Cardinality: return the number of rows if possible. */ +/***********************************************************************/ +int UZDFAM::Cardinality(PGLOBAL g) +{ + if (!g) + return 1; + + int card = -1; + int len = GetFileLength(g); + + card = Records; + + // Set number of blocks for later use + Block = (card > 0) ? (card + Nrec - 1) / Nrec : 0; + return card; +} // end of Cardinality + +/***********************************************************************/ +/* OpenTableFile: Open a DBF table file from a ZIP file. */ +/***********************************************************************/ +bool UZDFAM::OpenTableFile(PGLOBAL g) +{ + // May have been already opened in GetFileLength + if (!zutp || !zutp->zipfile) { + char filename[_MAX_PATH]; + MODE mode = Tdbp->GetMode(); + + /*********************************************************************/ + /* Allocate the ZIP utility class. */ + /*********************************************************************/ + if (!zutp) + zutp = new(g)UNZIPUTL(tdfp); + + // We used the file name relative to recorded datapath + PlugSetPath(filename, To_File, Tdbp->GetPath()); + + if (!zutp->OpenTable(g, mode, filename)) { + // The pseudo "buffer" is here the entire real buffer + Memory = zutp->memory; + Top = Memory + zutp->size; + To_Fb = zutp->fp; // Useful when closing + return AllocateBuffer(g); + } else + return true; + + } else + Reset(); + + return false; +} // end of OpenTableFile + +/***********************************************************************/ +/* GetNext: go to next entry. */ +/***********************************************************************/ +int UZDFAM::GetNext(PGLOBAL g) +{ + int rc = zutp->nextEntry(g); + + if (rc != RC_OK) + return rc; + + int len = zutp->size; + +#if 0 + if (len % Lrecl) { + sprintf(g->Message, MSG(NOT_FIXED_LEN), zutp->fn, len, Lrecl); + return RC_FX; + } // endif size +#endif // 0 + + Memory = zutp->memory; + Top = Memory + len; + Rewind(); + return RC_OK; +} // end of GetNext + /* -------------------------- class ZIPFAM --------------------------- */ /***********************************************************************/ @@ -1045,7 +1305,7 @@ bool ZIPFAM::OpenTableFile(PGLOBAL g) strcpy(g->Message, "No insert into existing zip file"); return true; } else if (append && len > 0) { - UNZIPUTL *zutp = new(g) UNZIPUTL(target, false); + UNZIPUTL *zutp = new(g) UNZIPUTL(target, NULL, false); if (!zutp->IsInsertOk(g, filename)) { strcpy(g->Message, "No insert into existing entry"); @@ -1129,7 +1389,7 @@ bool ZPXFAM::OpenTableFile(PGLOBAL g) strcpy(g->Message, "No insert into existing zip file"); return true; } else if (append && len > 0) { - UNZIPUTL *zutp = new(g) UNZIPUTL(target, false); + UNZIPUTL *zutp = new(g) UNZIPUTL(target, NULL, false); if (!zutp->IsInsertOk(g, filename)) { strcpy(g->Message, "No insert into existing entry"); diff --git a/storage/connect/filamzip.h b/storage/connect/filamzip.h index be17d954728..7ff1fb0a543 100644 --- a/storage/connect/filamzip.h +++ b/storage/connect/filamzip.h @@ -1,7 +1,7 @@ /************** filamzip H Declares Source Code File (.H) **************/ -/* Name: filamzip.h Version 1.2 */ +/* Name: filamzip.h Version 1.3 */ /* */ -/* (C) Copyright to the author Olivier BERTRAND 2016-2017 */ +/* (C) Copyright to the author Olivier BERTRAND 2016-2020 */ /* */ /* This file contains the ZIP file access method classes declares. */ /***********************************************************************/ @@ -11,6 +11,7 @@ #include "block.h" #include "filamap.h" #include "filamfix.h" +#include "filamdbf.h" #include "zip.h" #include "unzip.h" @@ -18,6 +19,7 @@ typedef class UNZFAM *PUNZFAM; typedef class UZXFAM *PUZXFAM; +typedef class UZDFAM* PUZDFAM; typedef class ZIPFAM *PZIPFAM; typedef class ZPXFAM *PZPXFAM; @@ -53,7 +55,7 @@ class DllExport ZIPUTIL : public BLOCK { class DllExport UNZIPUTL : public BLOCK { public: // Constructor - UNZIPUTL(PCSZ tgt, bool mul); + UNZIPUTL(PCSZ tgt, PCSZ pw, bool mul); UNZIPUTL(PDOSDEF tdp); // Implementation @@ -143,6 +145,36 @@ class DllExport UZXFAM : public MPXFAM { PDOSDEF tdfp; }; // end of UZXFAM +/***********************************************************************/ +/* This is the fixed unzip file access method. */ +/***********************************************************************/ +class DllExport UZDFAM : public DBMFAM { + //friend class UNZFAM; +public: + // Constructors + UZDFAM(PDOSDEF tdp); + UZDFAM(PUZDFAM txfp); + + // Implementation + virtual AMT GetAmType(void) { return TYPE_AM_ZIP; } + virtual PTXF Duplicate(PGLOBAL g) { return (PTXF) new(g)UZDFAM(this); } + + // Methods + virtual int GetFileLength(PGLOBAL g); + virtual int Cardinality(PGLOBAL g); + virtual bool OpenTableFile(PGLOBAL g); + virtual int GetNext(PGLOBAL g); + //virtual int ReadBuffer(PGLOBAL g); + +protected: + int dbfhead(PGLOBAL g, void* buf); + int ScanHeader(PGLOBAL g, int* rln); + + // Members + UNZIPUTL* zutp; + PDOSDEF tdfp; +}; // end of UZDFAM + /***********************************************************************/ /* This is the zip file access method. */ /***********************************************************************/ diff --git a/storage/connect/ha_connect.cc b/storage/connect/ha_connect.cc index a111082e786..bb56d6cd655 100644 --- a/storage/connect/ha_connect.cc +++ b/storage/connect/ha_connect.cc @@ -4507,12 +4507,12 @@ bool ha_connect::check_privileges(THD *thd, PTOS options, char *dbn, bool quick) case TAB_DIR: case TAB_ZIP: case TAB_OEM: - if (table && table->pos_in_table_list) // if SELECT - { - //Switch_to_definer_security_ctx backup_ctx(thd, table->pos_in_table_list); + if (table && table->pos_in_table_list) { // if SELECT +#if MYSQL_VERSION_ID > 100200 + Switch_to_definer_security_ctx backup_ctx(thd, table->pos_in_table_list); +#endif // VERSION_ID > 100200 return check_global_access(thd, FILE_ACL); - } - else + } else return check_global_access(thd, FILE_ACL); case TAB_ODBC: case TAB_JDBC: @@ -4528,7 +4528,7 @@ bool ha_connect::check_privileges(THD *thd, PTOS options, char *dbn, bool quick) case TAB_VIR: // This is temporary until a solution is found return false; - } // endswitch type + } // endswitch type my_printf_error(ER_UNKNOWN_ERROR, "check_privileges failed", MYF(0)); return true; @@ -5882,7 +5882,7 @@ static int connect_assisted_discovery(handlerton *, THD* thd, } else switch (ttp) { case TAB_DBF: - qrp= DBFColumns(g, dpath, fn, fnc == FNC_COL); + qrp= DBFColumns(g, dpath, fn, topt, fnc == FNC_COL); break; #if defined(ODBC_SUPPORT) case TAB_ODBC: @@ -6733,11 +6733,6 @@ int ha_connect::create(const char *name, TABLE *table_arg, PCSZ m= GetListOption(g, "Mulentries", options->oplist, "NO"); bool mul= *m == '1' || *m == 'Y' || *m == 'y' || !stricmp(m, "ON"); - if (!entry && !mul) { - my_message(ER_UNKNOWN_ERROR, "Missing entry name", MYF(0)); - DBUG_RETURN(HA_ERR_INTERNAL_ERROR); - } // endif entry - strcat(strcat(strcpy(dbpath, "./"), table->s->db.str), "/"); PlugSetPath(zbuf, options->filename, dbpath); PlugSetPath(buf, fn, dbpath); diff --git a/storage/connect/mongo.cpp b/storage/connect/mongo.cpp index 53e2bf377c4..bd3d3b893c1 100644 --- a/storage/connect/mongo.cpp +++ b/storage/connect/mongo.cpp @@ -380,7 +380,6 @@ MGODEF::MGODEF(void) Uri = NULL; Colist = NULL; Filter = NULL; - Level = 0; Base = 0; Version = 0; Pipe = false; diff --git a/storage/connect/mongo.h b/storage/connect/mongo.h index 97c391a217f..dcefac372c0 100644 --- a/storage/connect/mongo.h +++ b/storage/connect/mongo.h @@ -82,7 +82,6 @@ protected: PSZ Wrapname; /* Java wrapper name */ PCSZ Colist; /* Options list */ PCSZ Filter; /* Filtering query */ - int Level; /* Used for catalog table */ int Base; /* The array index base */ int Version; /* The Java driver version */ bool Pipe; /* True is Colist is a pipeline */ diff --git a/storage/connect/plgxml.cpp b/storage/connect/plgxml.cpp index f3d3a010266..8c5cc261899 100644 --- a/storage/connect/plgxml.cpp +++ b/storage/connect/plgxml.cpp @@ -49,7 +49,7 @@ bool XMLDOCUMENT::InitZip(PGLOBAL g, PCSZ entry) { #if defined(ZIP_SUPPORT) bool mul = (entry) ? strchr(entry, '*') || strchr(entry, '?') : false; - zip = new(g) UNZIPUTL(entry, mul); + zip = new(g) UNZIPUTL(entry, NULL, mul); return zip == NULL; #else // !ZIP_SUPPORT sprintf(g->Message, MSG(NO_FEAT_SUPPORT), "ZIP"); diff --git a/storage/connect/tabcmg.cpp b/storage/connect/tabcmg.cpp index b9b7f6e4b60..f2ff721627c 100644 --- a/storage/connect/tabcmg.cpp +++ b/storage/connect/tabcmg.cpp @@ -26,6 +26,8 @@ #include "tabmul.h" #include "filter.h" +PQRYRES MGOColumns(PGLOBAL g, PCSZ db, PCSZ uri, PTOS topt, bool info); + /* -------------------------- Class CMGDISC -------------------------- */ /***********************************************************************/ diff --git a/storage/connect/tabdos.cpp b/storage/connect/tabdos.cpp index 8efe2aad702..b3147bb7357 100644 --- a/storage/connect/tabdos.cpp +++ b/storage/connect/tabdos.cpp @@ -1,11 +1,11 @@ /************* TabDos C++ Program Source Code File (.CPP) **************/ /* PROGRAM NAME: TABDOS */ /* ------------- */ -/* Version 4.9.4 */ +/* Version 4.9.5 */ /* */ /* COPYRIGHT: */ /* ---------- */ -/* (C) Copyright to the author Olivier BERTRAND 1998-2019 */ +/* (C) Copyright to the author Olivier BERTRAND 1998-2020 */ /* */ /* WHAT THIS PROGRAM DOES: */ /* ----------------------- */ @@ -359,7 +359,26 @@ PTDB DOSDEF::GetTable(PGLOBAL g, MODE mode) /* Allocate table and file processing class of the proper type. */ /* Column blocks will be allocated only when needed. */ /*********************************************************************/ - if (Zipped) { + if (Recfm == RECFM_DBF) { + if (Catfunc == FNC_NO) { + if (Zipped) { + if (mode == MODE_READ || mode == MODE_ANY || mode == MODE_ALTER) { + txfp = new(g) UZDFAM(this); + } else { + strcpy(g->Message, "Zipped DBF tables are read only"); + return NULL; + } // endif's mode + + } else if (map) + txfp = new(g) DBMFAM(this); + else + txfp = new(g) DBFFAM(this); + + tdbp = new(g) TDBFIX(this, txfp); + } else + tdbp = new(g) TDBDCL(this); // Catfunc should be 'C' + + } else if (Zipped) { #if defined(ZIP_SUPPORT) if (Recfm == RECFM_VAR) { if (mode == MODE_READ || mode == MODE_ANY || mode == MODE_ALTER) { @@ -389,17 +408,6 @@ PTDB DOSDEF::GetTable(PGLOBAL g, MODE mode) sprintf(g->Message, MSG(NO_FEAT_SUPPORT), "ZIP"); return NULL; #endif // !ZIP_SUPPORT - } else if (Recfm == RECFM_DBF) { - if (Catfunc == FNC_NO) { - if (map) - txfp = new(g) DBMFAM(this); - else - txfp = new(g) DBFFAM(this); - - tdbp = new(g) TDBFIX(this, txfp); - } else // Catfunc should be 'C' - tdbp = new(g) TDBDCL(this); - } else if (Recfm != RECFM_VAR && Compressed < 2) { if (Huge) txfp = new(g) BGXFAM(this); diff --git a/storage/connect/tabdos.h b/storage/connect/tabdos.h index 207a1277fce..80dfe63845d 100644 --- a/storage/connect/tabdos.h +++ b/storage/connect/tabdos.h @@ -30,6 +30,7 @@ class DllExport DOSDEF : public TABDEF { /* Logical table description */ friend class DBFBASE; friend class UNZIPUTL; friend class JSONCOL; + friend class TDBDCL; public: // Constructor DOSDEF(void); diff --git a/storage/connect/tabfix.h b/storage/connect/tabfix.h index 53c0af1c422..5f859a2bffe 100644 --- a/storage/connect/tabfix.h +++ b/storage/connect/tabfix.h @@ -98,18 +98,20 @@ class DllExport BINCOL : public DOSCOL { /* This is the class declaration for the DBF columns catalog table. */ /***********************************************************************/ class TDBDCL : public TDBCAT { - public: - // Constructor - TDBDCL(PDOSDEF tdp) : TDBCAT(tdp) {Fn = tdp->GetFn();} +public: + // Constructor + TDBDCL(PDOSDEF tdp) : TDBCAT(tdp) + {Fn = tdp->GetFn(); Topt = tdp->GetTopt();} - protected: +protected: // Specific routines - virtual PQRYRES GetResult(PGLOBAL g) - {return DBFColumns(g, ((PTABDEF)To_Def)->GetPath(), Fn, false);} + virtual PQRYRES GetResult(PGLOBAL g) + {return DBFColumns(g, ((PTABDEF)To_Def)->GetPath(), Fn, Topt, false);} - // Members + // Members PCSZ Fn; // The DBF file (path) name - }; // end of class TDBOCL + PTOS Topt; +}; // end of class TDBOCL #endif // __TABFIX__ diff --git a/storage/connect/tabjson.cpp b/storage/connect/tabjson.cpp index 7e8d6c8d9f0..3b0d458a7a6 100644 --- a/storage/connect/tabjson.cpp +++ b/storage/connect/tabjson.cpp @@ -739,6 +739,7 @@ PTDB JSONDEF::GetTable(PGLOBAL g, MODE m) /***********************************************************************/ TDBJSN::TDBJSN(PJDEF tdp, PTXF txfp) : TDBDOS(tdp, txfp) { + G = NULL; Top = NULL; Row = NULL; Val = NULL; diff --git a/storage/connect/tabjson.h b/storage/connect/tabjson.h index 8721a2a5ab7..8c3f1013919 100644 --- a/storage/connect/tabjson.h +++ b/storage/connect/tabjson.h @@ -104,7 +104,6 @@ public: PCSZ Xcol; /* Name of expandable column */ int Limit; /* Limit of multiple values */ int Pretty; /* Depends on file structure */ - int Level; /* Used for catalog table */ int Base; /* The array index base */ bool Strict; /* Strict syntax checking */ char Sep; /* The Jpath separator */ diff --git a/storage/connect/tabzip.cpp b/storage/connect/tabzip.cpp index c026744dba8..d9c13e2a58a 100644 --- a/storage/connect/tabzip.cpp +++ b/storage/connect/tabzip.cpp @@ -23,6 +23,7 @@ #include "filamzip.h" #include "resource.h" // for IDS_COLUMNS #include "tabdos.h" +#include "tabmul.h" #include "tabzip.h" /* -------------------------- Class ZIPDEF --------------------------- */ @@ -41,7 +42,14 @@ bool ZIPDEF::DefineAM(PGLOBAL g, LPCSTR am, int poff) /***********************************************************************/ PTDB ZIPDEF::GetTable(PGLOBAL g, MODE m) { - return new(g) TDBZIP(this); + PTDB tdbp = NULL; + + tdbp = new(g) TDBZIP(this); + + if (Multiple) + tdbp = new(g) TDBMUL(tdbp); + + return tdbp; } // end of GetTable /* ------------------------------------------------------------------- */ @@ -108,7 +116,7 @@ int TDBZIP::Cardinality(PGLOBAL g) Cardinal = (err == UNZ_OK) ? (int)ginfo.number_entry : 0; } else - Cardinal = 0; + Cardinal = 10; // Dummy for multiple tables } // endif Cardinal @@ -187,6 +195,7 @@ int TDBZIP::DeleteDB(PGLOBAL g, int irc) void TDBZIP::CloseDB(PGLOBAL g) { close(); + nexterr = UNZ_OK; // For multiple tables Use = USE_READY; // Just to be clean } // end of CloseDB diff --git a/storage/connect/tabzip.h b/storage/connect/tabzip.h index 32b15281f81..d36e4dc01d0 100644 --- a/storage/connect/tabzip.h +++ b/storage/connect/tabzip.h @@ -48,6 +48,8 @@ public: // Implementation virtual AMT GetAmType(void) {return TYPE_AM_ZIP;} + virtual PCSZ GetFile(PGLOBAL) {return zfn;} + virtual void SetFile(PGLOBAL, PCSZ fn) {zfn = fn;} // Methods virtual PCOL MakeCol(PGLOBAL g, PCOLDEF cdp, PCOL cprec, int n); From 43ec9370b328fc9bf82e318bf992953a71925cd3 Mon Sep 17 00:00:00 2001 From: Sujatha Date: Thu, 22 Oct 2020 07:16:29 +0530 Subject: [PATCH 02/31] MDEV-10149: sys_vars.rpl_init_slave_func fails sporadically in buildbot problem: ======== mysqltest: In included file "./include/assert.inc": included from mysql-test/suite/sys_vars/t/rpl_init_slave_func.test at line 69: Assertion text: '@@global.max_connections = @start_max_connections' Assertion result: '0' mysqltest: In included file "./include/assert.inc": included from mysql-test/suite/sys_vars/t/rpl_init_slave_func.test at line 86: Assertion text: '@@global.max_connections = @start_max_connections + 1' Assertion result: '0' Analysis: ========= A slave SQL thread sets its Running state to Yes very early in its initialisation, before the majority of initialisation actions, including executing the init_slave command, are done. Thus the testcase has a race condition where the initial replication setup might finish executing later than the testcase SET GLOBAL init_slave, making the testcase see its effect where it checks for its absence. Fix: === Include 'sync_slave_sql_with_master.inc' at the beginning of the test to ensure that slave applier has completed the execution of 'init_slave' command and proceeded to event application. Replace the apparently needless RESET MASTER / RESET SLAVE etc. Patch is based on: https://github.com/percona/percona-server/pull/1464/commits/b91e2e6f90611aa299c302929fb8b068e8ac0dee Author: laurynas-biveinis --- .../sys_vars/r/rpl_init_slave_func.result | 15 +++------ .../suite/sys_vars/t/rpl_init_slave_func.test | 33 ++++++++++--------- 2 files changed, 21 insertions(+), 27 deletions(-) diff --git a/mysql-test/suite/sys_vars/r/rpl_init_slave_func.result b/mysql-test/suite/sys_vars/r/rpl_init_slave_func.result index 691f6f10e02..d63a540fe15 100644 --- a/mysql-test/suite/sys_vars/r/rpl_init_slave_func.result +++ b/mysql-test/suite/sys_vars/r/rpl_init_slave_func.result @@ -1,6 +1,6 @@ include/master-slave.inc [connection master] -connection slave +include/sync_slave_sql_with_master.inc SET @start_max_connections= @@global.max_connections; SET @start_init_slave= @@global.init_slave; SET NAMES utf8; @@ -19,18 +19,11 @@ SELECT @@global.init_slave = 'SET @@global.max_connections = @@global.max_connec 1 Expect 1 include/assert.inc [@@global.max_connections = @start_max_connections] -STOP SLAVE; -RESET MASTER; -RESET SLAVE; -START SLAVE; -include/wait_for_slave_to_start.inc +include/restart_slave.inc +include/sync_slave_sql_with_master.inc include/assert.inc [@@global.max_connections = @start_max_connections + 1] SET @@global.init_slave = "SET @a=5"; -STOP SLAVE; -RESET MASTER; -RESET SLAVE; -START SLAVE; -include/wait_for_slave_to_start.inc +include/restart_slave.inc SHOW VARIABLES LIKE 'init_slave'; Variable_name Value init_slave SET @a=5 diff --git a/mysql-test/suite/sys_vars/t/rpl_init_slave_func.test b/mysql-test/suite/sys_vars/t/rpl_init_slave_func.test index 1d57bfeddc5..0e5ff7a3f39 100644 --- a/mysql-test/suite/sys_vars/t/rpl_init_slave_func.test +++ b/mysql-test/suite/sys_vars/t/rpl_init_slave_func.test @@ -29,8 +29,13 @@ ############################################################################### source include/master-slave.inc; ---echo connection slave -connection slave; + +# Since a part of slave SQL thread initialisation happens after Slave_SQL_Running +# has been set to Yes, there is a race condition between initialisation above and +# init_slave setting given below. Synchronise slave applier with master to ensure +# init_slave is complete and applier had processed few events like FD. +--source include/sync_slave_sql_with_master.inc + --disable_query_log call mtr.add_suppression("Unsafe statement written to the binary log using statement format since BINLOG_FORMAT = STATEMENT"); --enable_query_log @@ -67,14 +72,15 @@ let $wait_condition= SELECT @@global.max_connections = @start_max_connections; --let $assert_text= @@global.max_connections = @start_max_connections --let $assert_cond= @@global.max_connections = @start_max_connections --source include/assert.inc -# -# reset of the server -STOP SLAVE; ---wait_for_slave_to_stop -RESET MASTER; -RESET SLAVE; -START SLAVE; -source include/wait_for_slave_to_start.inc; + +--source include/restart_slave_sql.inc + +# Upon slave start, sync the applier with master, to ensure slave has +# completed init_slave command execution and processed FD event from the +# master. +--connection master +--source include/sync_slave_sql_with_master.inc + # # wait for the slave threads have set the global variable. let $wait_timeout= 90; @@ -88,12 +94,7 @@ let $wait_condition= SELECT @@global.max_connections = @start_max_connections + # Setting a variable(which is local to a session) and must not be visible SET @@global.init_slave = "SET @a=5"; # -STOP SLAVE; ---wait_for_slave_to_stop -RESET MASTER; -RESET SLAVE; -START SLAVE; -source include/wait_for_slave_to_start.inc; +--source include/restart_slave_sql.inc # SHOW VARIABLES LIKE 'init_slave'; # expect NULL From 897ea21e57034e45bf3bc5997ba1b560eabc8221 Mon Sep 17 00:00:00 2001 From: Sergei Golubchik Date: Wed, 26 Aug 2020 13:08:33 +0200 Subject: [PATCH 03/31] MDEV-23358 main.upgrade_MDEV-19650 fails with result difference When including a generated file, always use <...>. We need the compiler to find it in the BINDIR, not in the SRCDIR. But when including as "..." SRCDIR is always searched first. The bug can only happen in out-of-source builds, if there was an in-source build before. --- client/mysql_upgrade.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/client/mysql_upgrade.c b/client/mysql_upgrade.c index d18dc97c9b8..8da7a4b01f2 100644 --- a/client/mysql_upgrade.c +++ b/client/mysql_upgrade.c @@ -18,7 +18,7 @@ #include "client_priv.h" #include -#include "../scripts/mysql_fix_privilege_tables_sql.c" +#include <../scripts/mysql_fix_privilege_tables_sql.c> #include /* ORACLE_WELCOME_COPYRIGHT_NOTICE */ From 94b493571a8ba8f30c97a5c30fc641171ca48e8a Mon Sep 17 00:00:00 2001 From: Alexey Botchkov Date: Fri, 23 Oct 2020 12:20:17 +0400 Subject: [PATCH 04/31] MDEV-20744 SET GLOBAL `replicate_do_db` = DEFAULT causes crash. DEFAULT for the replicate_do_db is the "" as our documentation states. --- mysql-test/suite/sys_vars/r/replicate_do_db_basic.result | 4 ++++ mysql-test/suite/sys_vars/t/replicate_do_db_basic.test | 3 +++ sql/sys_vars.ic | 6 +++++- 3 files changed, 12 insertions(+), 1 deletion(-) diff --git a/mysql-test/suite/sys_vars/r/replicate_do_db_basic.result b/mysql-test/suite/sys_vars/r/replicate_do_db_basic.result index a05b85a9bfd..54adf835962 100644 --- a/mysql-test/suite/sys_vars/r/replicate_do_db_basic.result +++ b/mysql-test/suite/sys_vars/r/replicate_do_db_basic.result @@ -37,5 +37,9 @@ SET @@GLOBAL.replicate_do_db=null; SELECT @@GLOBAL.replicate_do_db; @@GLOBAL.replicate_do_db +SET @@GLOBAL.replicate_do_db=DEFAULT; +SELECT @@GLOBAL.replicate_do_db; +@@GLOBAL.replicate_do_db + # Cleanup. SET @@GLOBAL.replicate_do_db = @save_replicate_do_db; diff --git a/mysql-test/suite/sys_vars/t/replicate_do_db_basic.test b/mysql-test/suite/sys_vars/t/replicate_do_db_basic.test index 59d0176add2..b7004d1938b 100644 --- a/mysql-test/suite/sys_vars/t/replicate_do_db_basic.test +++ b/mysql-test/suite/sys_vars/t/replicate_do_db_basic.test @@ -38,5 +38,8 @@ SELECT @@GLOBAL.replicate_do_db; SET @@GLOBAL.replicate_do_db=null; SELECT @@GLOBAL.replicate_do_db; +SET @@GLOBAL.replicate_do_db=DEFAULT; +SELECT @@GLOBAL.replicate_do_db; + --echo # Cleanup. SET @@GLOBAL.replicate_do_db = @save_replicate_do_db; diff --git a/sql/sys_vars.ic b/sql/sys_vars.ic index ae4568866b7..737f9bb545c 100644 --- a/sql/sys_vars.ic +++ b/sql/sys_vars.ic @@ -614,7 +614,11 @@ public: { DBUG_ASSERT(FALSE); } void global_save_default(THD *thd, set_var *var) - { DBUG_ASSERT(FALSE); } + { + char *ptr= (char*)(intptr)option.def_value; + var->save_result.string_value.str= ptr; + var->save_result.string_value.length= ptr ? strlen(ptr) : 0; + } bool session_update(THD *thd, set_var *var) { From f679d72679376d04c863e80bc68d084eb56795a5 Mon Sep 17 00:00:00 2001 From: Vlad Lesin Date: Thu, 30 Nov 2017 00:41:43 +0300 Subject: [PATCH 05/31] MDEV-24017: Blackhole : Specified key was too long; max key length is 1000 bytes The maximum innodb key length is 3500 what is hardcoded in ha_innobase::max_supported_key_length()). The maximum number of innodb indexes is configured with MAX_INDEXES macro (see also MAX_KEY definition). The same is currently implemented for blackhole storage engine. Cherry picked from percona-server 0d90d81c3c507a6b1476246a405504f6e4ef9d4d Original lp bug 1733049 Reviewed-by: daniel@mariadb.org --- mysql-test/r/blackhole.result | 6 ++++++ mysql-test/t/blackhole.test | 16 ++++++++++++++++ storage/blackhole/ha_blackhole.h | 5 +++-- 3 files changed, 25 insertions(+), 2 deletions(-) diff --git a/mysql-test/r/blackhole.result b/mysql-test/r/blackhole.result index 36f5459ff85..a7281c42ca7 100644 --- a/mysql-test/r/blackhole.result +++ b/mysql-test/r/blackhole.result @@ -24,3 +24,9 @@ SELECT 0 FROM t1 FORCE INDEX FOR GROUP BY(a) WHERE a = 0 OR b = 0 AND c = 0; 0 DROP TABLE t1; End of 5.6 tests +CREATE TABLE `t` ( +`a` varchar(3000) NOT NULL default '', +PRIMARY KEY (`a`) +) ENGINE=BLACKHOLE; +DROP TABLE `t`; +End of 10.1 tests diff --git a/mysql-test/t/blackhole.test b/mysql-test/t/blackhole.test index 7f394e0f846..c80ceffef4c 100644 --- a/mysql-test/t/blackhole.test +++ b/mysql-test/t/blackhole.test @@ -38,3 +38,19 @@ SELECT 0 FROM t1 FORCE INDEX FOR GROUP BY(a) WHERE a = 0 OR b = 0 AND c = 0; DROP TABLE t1; --echo End of 5.6 tests + +# +# MDEV-24017 / bug 53588 test case. +# +# Create long enough index (between 1000 and 3500). 1000 is the old value, +# 3500 is innodb value (see ha_innobase::max_supported_key_length()). Without +# the fix the test will fail with "Specified key was too long" error. +# +CREATE TABLE `t` ( + `a` varchar(3000) NOT NULL default '', + PRIMARY KEY (`a`) +) ENGINE=BLACKHOLE; + +DROP TABLE `t`; + +--echo End of 10.1 tests diff --git a/storage/blackhole/ha_blackhole.h b/storage/blackhole/ha_blackhole.h index e34386ddf33..07275b8eec1 100644 --- a/storage/blackhole/ha_blackhole.h +++ b/storage/blackhole/ha_blackhole.h @@ -20,6 +20,7 @@ #include "thr_lock.h" /* THR_LOCK */ #include "handler.h" /* handler */ #include "table.h" /* TABLE_SHARE */ +#include "sql_const.h" /* MAX_KEY */ /* Shared structure for correct LOCK operation @@ -65,9 +66,9 @@ public: HA_READ_ORDER | HA_KEYREAD_ONLY); } /* The following defines can be increased if necessary */ -#define BLACKHOLE_MAX_KEY 64 /* Max allowed keys */ +#define BLACKHOLE_MAX_KEY MAX_KEY /* Max allowed keys */ #define BLACKHOLE_MAX_KEY_SEG 16 /* Max segments for key */ -#define BLACKHOLE_MAX_KEY_LENGTH 1000 +#define BLACKHOLE_MAX_KEY_LENGTH 3500 /* Like in InnoDB */ uint max_supported_keys() const { return BLACKHOLE_MAX_KEY; } uint max_supported_key_length() const { return BLACKHOLE_MAX_KEY_LENGTH; } uint max_supported_key_part_length() const { return BLACKHOLE_MAX_KEY_LENGTH; } From 44c958dd7b454ebbdbf7ac8b066592c82dd3409f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20Lindstr=C3=B6m?= Date: Sat, 24 Oct 2020 14:57:16 +0300 Subject: [PATCH 06/31] Fix test failure on wsrep/variables test case. --- mysql-test/include/galera_have_debug_sync.inc | 9 +++++++++ mysql-test/suite/wsrep/t/variables.test | 1 + 2 files changed, 10 insertions(+) create mode 100644 mysql-test/include/galera_have_debug_sync.inc diff --git a/mysql-test/include/galera_have_debug_sync.inc b/mysql-test/include/galera_have_debug_sync.inc new file mode 100644 index 00000000000..21e7b3c88c3 --- /dev/null +++ b/mysql-test/include/galera_have_debug_sync.inc @@ -0,0 +1,9 @@ +--disable_query_log + +--let $galera_have_debug_sync = `SELECT 1 FROM INFORMATION_SCHEMA.GLOBAL_STATUS WHERE VARIABLE_NAME = 'wsrep_debug_sync_waiters'` + +--if (!$galera_have_debug_sync) { + --skip Test requires Galera debug library with debug_sync functionality +} + +--enable_query_log diff --git a/mysql-test/suite/wsrep/t/variables.test b/mysql-test/suite/wsrep/t/variables.test index 1a2ab2579a5..15b69f8b7f6 100644 --- a/mysql-test/suite/wsrep/t/variables.test +++ b/mysql-test/suite/wsrep/t/variables.test @@ -1,4 +1,5 @@ --source include/have_wsrep.inc +--source include/galera_have_debug_sync.inc SET @wsrep_provider_options_saved= @@global.wsrep_provider_options; SET @wsrep_cluster_address_saved= @@global.wsrep_cluster_address; From 858434910835117215890f8368b91b2905e74815 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Vicen=C8=9Biu=20Ciorbaru?= Date: Fri, 23 Oct 2020 18:09:01 +0300 Subject: [PATCH 07/31] MDEV-14945 possible buffer overflow in stack resolver According to https://stackoverflow.com/questions/22827510/how-to-avoid-bad-fd-set-buffer-overflow-crash it seems that using select instead of poll can cause additional memory allocations. As we are in a crashed state, we must prevent allocating any memory (if possible). Thus, switch select call to poll. Also move some bigger datastructures to global space. The code is not run in a multithreaded context so best we don't use up stack space if it's not needed. --- mysys/my_addr_resolve.c | 27 ++++++++++++++++++--------- 1 file changed, 18 insertions(+), 9 deletions(-) diff --git a/mysys/my_addr_resolve.c b/mysys/my_addr_resolve.c index ecb512572b1..e0ebbe699be 100644 --- a/mysys/my_addr_resolve.c +++ b/mysys/my_addr_resolve.c @@ -147,10 +147,18 @@ err: #include +#if defined(HAVE_POLL_H) +#include +#elif defined(HAVE_SYS_POLL_H) +#include +#endif /* defined(HAVE_POLL_H) */ + static int in[2], out[2]; static pid_t pid; static char addr2line_binary[1024]; static char output[1024]; +static struct pollfd poll_fds; +Dl_info info; int start_addr2line_fork(const char *binary_path) { @@ -200,15 +208,16 @@ int my_addr_resolve(void *ptr, my_addr_loc *loc) ssize_t extra_bytes_read = 0; ssize_t parsed = 0; - fd_set set; - struct timeval timeout; + int ret; int filename_start = -1; int line_number_start = -1; - Dl_info info; void *offset; + poll_fds.fd = out[0]; + poll_fds.events = POLLIN | POLLRDBAND; + if (!dladdr(ptr, &info)) return 1; @@ -230,16 +239,16 @@ int my_addr_resolve(void *ptr, my_addr_loc *loc) if (write(in[1], input, len) <= 0) return 1; - FD_ZERO(&set); - FD_SET(out[0], &set); - /* 100 ms should be plenty of time for addr2line to issue a response. */ - timeout.tv_sec = 0; - timeout.tv_usec = 100000; + /* 500 ms should be plenty of time for addr2line to issue a response. */ /* Read in a loop till all the output from addr2line is complete. */ while (parsed == total_bytes_read && - select(out[0] + 1, &set, NULL, NULL, &timeout) > 0) + (ret= poll(&poll_fds, 1, 500))) { + /* error during poll */ + if (ret < 0) + return 1; + extra_bytes_read= read(out[0], output + total_bytes_read, sizeof(output) - total_bytes_read); if (extra_bytes_read < 0) From 320a73f6a29c6d6adf8576651263812d42796235 Mon Sep 17 00:00:00 2001 From: Sergei Golubchik Date: Sun, 25 Oct 2020 18:16:24 +0100 Subject: [PATCH 08/31] cleanup: PRIV_LOCK_TABLES (10.5 style) --- sql/sql_parse.cc | 35 +++++++++++++++++------------------ 1 file changed, 17 insertions(+), 18 deletions(-) diff --git a/sql/sql_parse.cc b/sql/sql_parse.cc index 14ac657862f..fb7552e5929 100644 --- a/sql/sql_parse.cc +++ b/sql/sql_parse.cc @@ -100,6 +100,8 @@ #include "my_json_writer.h" +#define PRIV_LOCK_TABLES (SELECT_ACL | LOCK_TABLES_ACL) + #define FLAGSTR(V,F) ((V)&(F)?#F" ":"") #ifdef WITH_ARIA_STORAGE_ENGINE @@ -4560,7 +4562,7 @@ mysql_execute_command(THD *thd) if (first_table && lex->type & (REFRESH_READ_LOCK|REFRESH_FOR_EXPORT)) { /* Check table-level privileges. */ - if (check_table_access(thd, LOCK_TABLES_ACL | SELECT_ACL, all_tables, + if (check_table_access(thd, PRIV_LOCK_TABLES, all_tables, FALSE, UINT_MAX, FALSE)) goto error; @@ -6065,7 +6067,7 @@ check_access(THD *thd, ulong want_access, const char *db, ulong *save_priv, @param thd Thread handler @param privilege requested privilege - @param all_tables global table list of query + @param tables global table list of query @param no_errors FALSE/TRUE - report/don't report error to the client (using my_error() call). @@ -6075,32 +6077,29 @@ check_access(THD *thd, ulong want_access, const char *db, ulong *save_priv, 1 access denied, error is sent to client */ -bool check_single_table_access(THD *thd, ulong privilege, - TABLE_LIST *all_tables, bool no_errors) +bool check_single_table_access(THD *thd, ulong privilege, TABLE_LIST *tables, + bool no_errors) { Security_context * backup_ctx= thd->security_ctx; /* we need to switch to the saved context (if any) */ - if (all_tables->security_ctx) - thd->security_ctx= all_tables->security_ctx; + if (tables->security_ctx) + thd->security_ctx= tables->security_ctx; const char *db_name; - if ((all_tables->view || all_tables->field_translation) && - !all_tables->schema_table) - db_name= all_tables->view_db.str; + if ((tables->view || tables->field_translation) && !tables->schema_table) + db_name= tables->view_db.str; else - db_name= all_tables->db; + db_name= tables->db; - if (check_access(thd, privilege, db_name, - &all_tables->grant.privilege, - &all_tables->grant.m_internal, - 0, no_errors)) + if (check_access(thd, privilege, db_name, &tables->grant.privilege, + &tables->grant.m_internal, 0, no_errors)) goto deny; /* Show only 1 table for check_grant */ - if (!(all_tables->belong_to_view && - (thd->lex->sql_command == SQLCOM_SHOW_FIELDS)) && - check_grant(thd, privilege, all_tables, FALSE, 1, no_errors)) + if (!(tables->belong_to_view && + (thd->lex->sql_command == SQLCOM_SHOW_FIELDS)) && + check_grant(thd, privilege, tables, FALSE, 1, no_errors)) goto deny; thd->security_ctx= backup_ctx; @@ -9060,7 +9059,7 @@ static bool lock_tables_precheck(THD *thd, TABLE_LIST *tables) if (is_temporary_table(table)) continue; - if (check_table_access(thd, LOCK_TABLES_ACL | SELECT_ACL, table, + if (check_table_access(thd, PRIV_LOCK_TABLES, table, FALSE, 1, FALSE)) return TRUE; } From 0c3723e1d50e61303b63f6264c2c193397ee4475 Mon Sep 17 00:00:00 2001 From: Sergei Golubchik Date: Sun, 25 Oct 2020 18:17:34 +0100 Subject: [PATCH 09/31] Bug#31304432 "INSUFFICIENT PRIVILEGE CHECK BY LOCK TABLES" `LOCK TABLES view_name` should require * invoker to have SELECT and LOCK TABLES privileges on the view * either invoker or definer (only if sql security definer) to have SELECT and LOCK TABLES privileges on the used tables/views. --- mysql-test/r/lock_view.result | 216 ++++++++++++++++++++++++++++++++++ mysql-test/t/lock_view.test | 76 ++++++++++++ sql/sql_parse.cc | 30 +++++ 3 files changed, 322 insertions(+) create mode 100644 mysql-test/r/lock_view.result create mode 100644 mysql-test/t/lock_view.test diff --git a/mysql-test/r/lock_view.result b/mysql-test/r/lock_view.result new file mode 100644 index 00000000000..aa76dd73609 --- /dev/null +++ b/mysql-test/r/lock_view.result @@ -0,0 +1,216 @@ +create database mysqltest1; +create database mysqltest2; +create database mysqltest3; +create user invoker@localhost; +create user definer@localhost; +grant select,show view on mysqltest1.* to invoker@localhost; +grant select,show view on mysqltest1.* to definer@localhost; +grant select,show view on mysqltest2.* to invoker@localhost; +grant select,show view on mysqltest2.* to definer@localhost; +grant select,show view on mysqltest3.* to invoker@localhost; +grant select on performance_schema.* to definer@localhost; +create table mysqltest1.t1 (a int); +create definer=definer@localhost view mysqltest2.v2 as select * from mysqltest1.t1; +create definer=definer@localhost view mysqltest3.v3 as select * from mysqltest2.v2; +create definer=definer@localhost view mysqltest3.v3is as select schema_name from information_schema.schemata order by schema_name; +create definer=definer@localhost view mysqltest3.v3ps as select user from performance_schema.users where current_connections>0 order by user; +create definer=definer@localhost view mysqltest3.v3nt as select 1; +create definer=definer@localhost sql security invoker view mysqltest3.v3i as select * from mysqltest1.t1; + +CREATE DATABASE /*!32312 IF NOT EXISTS*/ `mysqltest1` /*!40100 DEFAULT CHARACTER SET latin1 */; + +USE `mysqltest1`; +/*!40101 SET @saved_cs_client = @@character_set_client */; +/*!40101 SET character_set_client = utf8 */; +CREATE TABLE `t1` ( + `a` int(11) DEFAULT NULL +) ENGINE=MyISAM DEFAULT CHARSET=latin1; +/*!40101 SET character_set_client = @saved_cs_client */; + +CREATE DATABASE /*!32312 IF NOT EXISTS*/ `mysqltest2` /*!40100 DEFAULT CHARACTER SET latin1 */; + +USE `mysqltest2`; +SET @saved_cs_client = @@character_set_client; +SET character_set_client = utf8; +/*!50001 CREATE TABLE `v2` ( + `a` tinyint NOT NULL +) ENGINE=MyISAM */; +SET character_set_client = @saved_cs_client; + +CREATE DATABASE /*!32312 IF NOT EXISTS*/ `mysqltest3` /*!40100 DEFAULT CHARACTER SET latin1 */; + +USE `mysqltest3`; +SET @saved_cs_client = @@character_set_client; +SET character_set_client = utf8; +/*!50001 CREATE TABLE `v3` ( + `a` tinyint NOT NULL +) ENGINE=MyISAM */; +SET character_set_client = @saved_cs_client; +SET @saved_cs_client = @@character_set_client; +SET character_set_client = utf8; +/*!50001 CREATE TABLE `v3i` ( + `a` tinyint NOT NULL +) ENGINE=MyISAM */; +SET character_set_client = @saved_cs_client; +SET @saved_cs_client = @@character_set_client; +SET character_set_client = utf8; +/*!50001 CREATE TABLE `v3is` ( + `schema_name` tinyint NOT NULL +) ENGINE=MyISAM */; +SET character_set_client = @saved_cs_client; +SET @saved_cs_client = @@character_set_client; +SET character_set_client = utf8; +/*!50001 CREATE TABLE `v3nt` ( + `1` tinyint NOT NULL +) ENGINE=MyISAM */; +SET character_set_client = @saved_cs_client; +SET @saved_cs_client = @@character_set_client; +SET character_set_client = utf8; +/*!50001 CREATE TABLE `v3ps` ( + `user` tinyint NOT NULL +) ENGINE=MyISAM */; +SET character_set_client = @saved_cs_client; + +USE `mysqltest1`; + +USE `mysqltest2`; +/*!50001 DROP TABLE IF EXISTS `v2`*/; +/*!50001 SET @saved_cs_client = @@character_set_client */; +/*!50001 SET @saved_cs_results = @@character_set_results */; +/*!50001 SET @saved_col_connection = @@collation_connection */; +/*!50001 SET character_set_client = latin1 */; +/*!50001 SET character_set_results = latin1 */; +/*!50001 SET collation_connection = latin1_swedish_ci */; +/*!50001 CREATE ALGORITHM=UNDEFINED */ +/*!50013 DEFINER=`definer`@`localhost` SQL SECURITY DEFINER */ +/*!50001 VIEW `v2` AS select `mysqltest1`.`t1`.`a` AS `a` from `mysqltest1`.`t1` */; +/*!50001 SET character_set_client = @saved_cs_client */; +/*!50001 SET character_set_results = @saved_cs_results */; +/*!50001 SET collation_connection = @saved_col_connection */; + +USE `mysqltest3`; +/*!50001 DROP TABLE IF EXISTS `v3`*/; +/*!50001 SET @saved_cs_client = @@character_set_client */; +/*!50001 SET @saved_cs_results = @@character_set_results */; +/*!50001 SET @saved_col_connection = @@collation_connection */; +/*!50001 SET character_set_client = latin1 */; +/*!50001 SET character_set_results = latin1 */; +/*!50001 SET collation_connection = latin1_swedish_ci */; +/*!50001 CREATE ALGORITHM=UNDEFINED */ +/*!50013 DEFINER=`definer`@`localhost` SQL SECURITY DEFINER */ +/*!50001 VIEW `v3` AS select `v2`.`a` AS `a` from `mysqltest2`.`v2` */; +/*!50001 SET character_set_client = @saved_cs_client */; +/*!50001 SET character_set_results = @saved_cs_results */; +/*!50001 SET collation_connection = @saved_col_connection */; +/*!50001 DROP TABLE IF EXISTS `v3i`*/; +/*!50001 SET @saved_cs_client = @@character_set_client */; +/*!50001 SET @saved_cs_results = @@character_set_results */; +/*!50001 SET @saved_col_connection = @@collation_connection */; +/*!50001 SET character_set_client = latin1 */; +/*!50001 SET character_set_results = latin1 */; +/*!50001 SET collation_connection = latin1_swedish_ci */; +/*!50001 CREATE ALGORITHM=UNDEFINED */ +/*!50013 DEFINER=`definer`@`localhost` SQL SECURITY INVOKER */ +/*!50001 VIEW `v3i` AS select `mysqltest1`.`t1`.`a` AS `a` from `mysqltest1`.`t1` */; +/*!50001 SET character_set_client = @saved_cs_client */; +/*!50001 SET character_set_results = @saved_cs_results */; +/*!50001 SET collation_connection = @saved_col_connection */; +/*!50001 DROP TABLE IF EXISTS `v3is`*/; +/*!50001 SET @saved_cs_client = @@character_set_client */; +/*!50001 SET @saved_cs_results = @@character_set_results */; +/*!50001 SET @saved_col_connection = @@collation_connection */; +/*!50001 SET character_set_client = latin1 */; +/*!50001 SET character_set_results = latin1 */; +/*!50001 SET collation_connection = latin1_swedish_ci */; +/*!50001 CREATE ALGORITHM=UNDEFINED */ +/*!50013 DEFINER=`definer`@`localhost` SQL SECURITY DEFINER */ +/*!50001 VIEW `v3is` AS select `information_schema`.`schemata`.`SCHEMA_NAME` AS `schema_name` from `information_schema`.`schemata` order by `information_schema`.`schemata`.`SCHEMA_NAME` */; +/*!50001 SET character_set_client = @saved_cs_client */; +/*!50001 SET character_set_results = @saved_cs_results */; +/*!50001 SET collation_connection = @saved_col_connection */; +/*!50001 DROP TABLE IF EXISTS `v3nt`*/; +/*!50001 SET @saved_cs_client = @@character_set_client */; +/*!50001 SET @saved_cs_results = @@character_set_results */; +/*!50001 SET @saved_col_connection = @@collation_connection */; +/*!50001 SET character_set_client = latin1 */; +/*!50001 SET character_set_results = latin1 */; +/*!50001 SET collation_connection = latin1_swedish_ci */; +/*!50001 CREATE ALGORITHM=UNDEFINED */ +/*!50013 DEFINER=`definer`@`localhost` SQL SECURITY DEFINER */ +/*!50001 VIEW `v3nt` AS select 1 AS `1` */; +/*!50001 SET character_set_client = @saved_cs_client */; +/*!50001 SET character_set_results = @saved_cs_results */; +/*!50001 SET collation_connection = @saved_col_connection */; +/*!50001 DROP TABLE IF EXISTS `v3ps`*/; +/*!50001 SET @saved_cs_client = @@character_set_client */; +/*!50001 SET @saved_cs_results = @@character_set_results */; +/*!50001 SET @saved_col_connection = @@collation_connection */; +/*!50001 SET character_set_client = latin1 */; +/*!50001 SET character_set_results = latin1 */; +/*!50001 SET collation_connection = latin1_swedish_ci */; +/*!50001 CREATE ALGORITHM=UNDEFINED */ +/*!50013 DEFINER=`definer`@`localhost` SQL SECURITY DEFINER */ +/*!50001 VIEW `v3ps` AS select `performance_schema`.`users`.`USER` AS `user` from `performance_schema`.`users` where (`performance_schema`.`users`.`CURRENT_CONNECTIONS` > 0) order by `performance_schema`.`users`.`USER` */; +/*!50001 SET character_set_client = @saved_cs_client */; +/*!50001 SET character_set_results = @saved_cs_results */; +/*!50001 SET collation_connection = @saved_col_connection */; +lock table mysqltest3.v3 write; +ERROR 42000: Access denied for user 'invoker'@'localhost' to database 'mysqltest3' +grant lock tables on mysqltest3.* to invoker@localhost; +show create view mysqltest3.v3; +View Create View character_set_client collation_connection +v3 CREATE ALGORITHM=UNDEFINED DEFINER=`definer`@`localhost` SQL SECURITY DEFINER VIEW `mysqltest3`.`v3` AS select `v2`.`a` AS `a` from `mysqltest2`.`v2` latin1 latin1_swedish_ci +show create view mysqltest3.v3is; +View Create View character_set_client collation_connection +v3is CREATE ALGORITHM=UNDEFINED DEFINER=`definer`@`localhost` SQL SECURITY DEFINER VIEW `mysqltest3`.`v3is` AS select `information_schema`.`schemata`.`SCHEMA_NAME` AS `schema_name` from `information_schema`.`schemata` order by `information_schema`.`schemata`.`SCHEMA_NAME` latin1 latin1_swedish_ci +show create view mysqltest3.v3ps; +View Create View character_set_client collation_connection +v3ps CREATE ALGORITHM=UNDEFINED DEFINER=`definer`@`localhost` SQL SECURITY DEFINER VIEW `mysqltest3`.`v3ps` AS select `performance_schema`.`users`.`USER` AS `user` from `performance_schema`.`users` where (`performance_schema`.`users`.`CURRENT_CONNECTIONS` > 0) order by `performance_schema`.`users`.`USER` latin1 latin1_swedish_ci +show create view mysqltest3.v3nt; +View Create View character_set_client collation_connection +v3nt CREATE ALGORITHM=UNDEFINED DEFINER=`definer`@`localhost` SQL SECURITY DEFINER VIEW `mysqltest3`.`v3nt` AS select 1 AS `1` latin1 latin1_swedish_ci +show create view mysqltest3.v3i; +View Create View character_set_client collation_connection +v3i CREATE ALGORITHM=UNDEFINED DEFINER=`definer`@`localhost` SQL SECURITY INVOKER VIEW `mysqltest3`.`v3i` AS select `mysqltest1`.`t1`.`a` AS `a` from `mysqltest1`.`t1` latin1 latin1_swedish_ci +lock table mysqltest3.v3 write; +ERROR HY000: View 'mysqltest3.v3' references invalid table(s) or column(s) or function(s) or definer/invoker of view lack rights to use them +lock table mysqltest3.v3i write; +ERROR HY000: View 'mysqltest3.v3i' references invalid table(s) or column(s) or function(s) or definer/invoker of view lack rights to use them +lock table mysqltest3.v3is write; +select * from mysqltest3.v3is; +schema_name +information_schema +mysqltest1 +mysqltest2 +mysqltest3 +test +lock table mysqltest3.v3ps write; +select * from mysqltest3.v3ps; +user +NULL +invoker +root +lock table mysqltest3.v3nt write; +select * from mysqltest3.v3nt; +1 +1 +grant lock tables on mysqltest2.* to invoker@localhost; +lock table mysqltest3.v3 write; +ERROR HY000: View 'mysqltest3.v3' references invalid table(s) or column(s) or function(s) or definer/invoker of view lack rights to use them +lock table mysqltest3.v3i write; +ERROR HY000: View 'mysqltest3.v3i' references invalid table(s) or column(s) or function(s) or definer/invoker of view lack rights to use them +grant lock tables on mysqltest1.* to definer@localhost; +lock table mysqltest3.v3 write; +select * from mysqltest3.v3; +a +lock table mysqltest3.v3i write; +ERROR HY000: View 'mysqltest3.v3i' references invalid table(s) or column(s) or function(s) or definer/invoker of view lack rights to use them +grant lock tables on mysqltest1.* to invoker@localhost; +lock table mysqltest3.v3i write; +select * from mysqltest3.v3i; +a +drop user invoker@localhost; +drop user definer@localhost; +drop database mysqltest1; +drop database mysqltest2; +drop database mysqltest3; diff --git a/mysql-test/t/lock_view.test b/mysql-test/t/lock_view.test new file mode 100644 index 00000000000..dd8809ab89d --- /dev/null +++ b/mysql-test/t/lock_view.test @@ -0,0 +1,76 @@ +source include/not_embedded.inc; +# +# LOCK TABLES and privileges on views +# +create database mysqltest1; +create database mysqltest2; +create database mysqltest3; +create user invoker@localhost; +create user definer@localhost; +grant select,show view on mysqltest1.* to invoker@localhost; +grant select,show view on mysqltest1.* to definer@localhost; +grant select,show view on mysqltest2.* to invoker@localhost; +grant select,show view on mysqltest2.* to definer@localhost; +grant select,show view on mysqltest3.* to invoker@localhost; +grant select on performance_schema.* to definer@localhost; +create table mysqltest1.t1 (a int); +create definer=definer@localhost view mysqltest2.v2 as select * from mysqltest1.t1; +create definer=definer@localhost view mysqltest3.v3 as select * from mysqltest2.v2; +create definer=definer@localhost view mysqltest3.v3is as select schema_name from information_schema.schemata order by schema_name; +create definer=definer@localhost view mysqltest3.v3ps as select user from performance_schema.users where current_connections>0 order by user; +create definer=definer@localhost view mysqltest3.v3nt as select 1; +create definer=definer@localhost sql security invoker view mysqltest3.v3i as select * from mysqltest1.t1; + +exec $MYSQL_DUMP --compact -B mysqltest1 mysqltest2 mysqltest3; + +connect inv,localhost,invoker; +error ER_DBACCESS_DENIED_ERROR; +lock table mysqltest3.v3 write; +disconnect inv; +connection default; + +grant lock tables on mysqltest3.* to invoker@localhost; +connect inv,localhost,invoker; +show create view mysqltest3.v3; +show create view mysqltest3.v3is; +show create view mysqltest3.v3ps; +show create view mysqltest3.v3nt; +show create view mysqltest3.v3i; +error ER_VIEW_INVALID; +lock table mysqltest3.v3 write; +error ER_VIEW_INVALID; +lock table mysqltest3.v3i write; +lock table mysqltest3.v3is write; select * from mysqltest3.v3is; +lock table mysqltest3.v3ps write; select * from mysqltest3.v3ps; +lock table mysqltest3.v3nt write; select * from mysqltest3.v3nt; +disconnect inv; +connection default; + +grant lock tables on mysqltest2.* to invoker@localhost; +connect inv,localhost,invoker; +error ER_VIEW_INVALID; +lock table mysqltest3.v3 write; +error ER_VIEW_INVALID; +lock table mysqltest3.v3i write; +disconnect inv; +connection default; + +grant lock tables on mysqltest1.* to definer@localhost; +connect inv,localhost,invoker; +lock table mysqltest3.v3 write; select * from mysqltest3.v3; +error ER_VIEW_INVALID; +lock table mysqltest3.v3i write; +disconnect inv; +connection default; + +grant lock tables on mysqltest1.* to invoker@localhost; +connect inv,localhost,invoker; +lock table mysqltest3.v3i write; select * from mysqltest3.v3i; +disconnect inv; +connection default; + +drop user invoker@localhost; +drop user definer@localhost; +drop database mysqltest1; +drop database mysqltest2; +drop database mysqltest3; diff --git a/sql/sql_parse.cc b/sql/sql_parse.cc index fb7552e5929..bf1dad804e3 100644 --- a/sql/sql_parse.cc +++ b/sql/sql_parse.cc @@ -2392,10 +2392,40 @@ static bool lock_tables_open_and_lock_tables(THD *thd, TABLE_LIST *tables) We don't set TABLE_LIST::lock_type in this case as this might result in extra warnings from THD::decide_logging_format() even though binary logging is totally irrelevant for LOCK TABLES. + + Check privileges of view tables here, after views were opened. + Either definer or invoker has to have PRIV_LOCK_TABLES to be able to + lock view and its tables. For mysqldump (that locks views before dumping + their structures) compatibility we allow locking views that select + from I_S or P_S tables, but downrade the lock to TL_READ */ for (table= tables; table; table= table->next_global) + { if (!table->placeholder() && table->table->s->tmp_table) table->table->reginfo.lock_type= TL_WRITE; + else if (table->belong_to_view && + check_single_table_access(thd, PRIV_LOCK_TABLES, table, 1)) + { + if (table->grant.m_internal.m_schema_access) + table->lock_type= TL_READ; + else + { + bool error= true; + if (Security_context *sctx= table->security_ctx) + { + table->security_ctx= 0; + error= check_single_table_access(thd, PRIV_LOCK_TABLES, table, 1); + table->security_ctx= sctx; + } + if (error) + { + my_error(ER_VIEW_INVALID, MYF(0), table->belong_to_view->view_db.str, + table->belong_to_view->view_name.str); + goto err; + } + } + } + } if (lock_tables(thd, tables, counter, 0) || thd->locked_tables_list.init_locked_tables(thd)) From 1269fd420dbcac2cedb345944a35aebe3562ddc1 Mon Sep 17 00:00:00 2001 From: Karthik Kamath Date: Mon, 31 Aug 2020 12:21:07 +0530 Subject: [PATCH 10/31] BUG#31650096: MYSQL SERVER HEAP-USE-AFTER-FREE IN TRANS_SAVEPOINT ANALYSIS: ========= During Bootstrap, while executing the statements from sql file passed to the init-file server option, transaction mem_root was being freed for every statement. This creates an issue with multi statement transactions especially when a statement in the transaction has to access the memory used by the previous statement in the transaction. FIX: ==== Transaction mem_root is freed whenever a transaction is committed or rolled-back. Hence explicitly freeing it is not necessary in the bootstrap implementation. Change-Id: I40f71d49781bf7ad32d474bb176bd6060c9377dc --- sql/sql_parse.cc | 1 - 1 file changed, 1 deletion(-) diff --git a/sql/sql_parse.cc b/sql/sql_parse.cc index bf1dad804e3..64b9f35d664 100644 --- a/sql/sql_parse.cc +++ b/sql/sql_parse.cc @@ -811,7 +811,6 @@ static void handle_bootstrap_impl(THD *thd) thd->reset_kill_query(); /* Ensure that killed_errmsg is released */ free_root(thd->mem_root,MYF(MY_KEEP_PREALLOC)); - free_root(&thd->transaction.mem_root,MYF(MY_KEEP_PREALLOC)); thd->lex->restore_set_statement_var(); } From a7d5e85c4935080458fea21c718f56c7bf6f02bd Mon Sep 17 00:00:00 2001 From: Sergei Golubchik Date: Mon, 26 Oct 2020 17:20:59 +0100 Subject: [PATCH 11/31] cleanup: have_static_innodb.inc and remove unused files --- mysql-test/include/have_not_innodb_plugin.inc | 4 ---- mysql-test/include/have_static_innodb.inc | 7 +++++++ mysql-test/r/not_true.require | 2 -- mysql-test/t/plugin_innodb.test | 9 +-------- 4 files changed, 8 insertions(+), 14 deletions(-) delete mode 100644 mysql-test/include/have_not_innodb_plugin.inc create mode 100644 mysql-test/include/have_static_innodb.inc delete mode 100644 mysql-test/r/not_true.require diff --git a/mysql-test/include/have_not_innodb_plugin.inc b/mysql-test/include/have_not_innodb_plugin.inc deleted file mode 100644 index e40fd811021..00000000000 --- a/mysql-test/include/have_not_innodb_plugin.inc +++ /dev/null @@ -1,4 +0,0 @@ -disable_query_log; ---require r/not_true.require -select (PLUGIN_LIBRARY LIKE 'ha_innodb_plugin%' OR PLUGIN_DESCRIPTION LIKE '%xtradb%') as `TRUE` from information_schema.plugins where PLUGIN_NAME='InnoDB'; -enable_query_log; diff --git a/mysql-test/include/have_static_innodb.inc b/mysql-test/include/have_static_innodb.inc new file mode 100644 index 00000000000..0d7bb856f4f --- /dev/null +++ b/mysql-test/include/have_static_innodb.inc @@ -0,0 +1,7 @@ +source include/have_innodb.inc; + +if (!`select count(*) from information_schema.plugins + where plugin_name = 'innodb' and plugin_status = 'active' and + plugin_library is null`) { + skip Need compiled-in InnoDB; +} diff --git a/mysql-test/r/not_true.require b/mysql-test/r/not_true.require deleted file mode 100644 index 0032832f3d1..00000000000 --- a/mysql-test/r/not_true.require +++ /dev/null @@ -1,2 +0,0 @@ -TRUE -NULL diff --git a/mysql-test/t/plugin_innodb.test b/mysql-test/t/plugin_innodb.test index fb5dd84b997..03afa190400 100644 --- a/mysql-test/t/plugin_innodb.test +++ b/mysql-test/t/plugin_innodb.test @@ -1,13 +1,6 @@ --source include/not_embedded.inc --source include/have_example_plugin.inc ---source include/have_innodb.inc - -if (!`select count(*) from information_schema.plugins - where plugin_name = 'innodb' and plugin_status = 'active' and - plugin_library is null`) { - skip Need compiled-in InnoDB; -} - +--source include/have_static_innodb.inc --replace_regex /\.dll/.so/ eval install plugin example soname '$HA_EXAMPLE_SO'; From d03ea82759aef7734f162c3137b10aa2476b35e9 Mon Sep 17 00:00:00 2001 From: Sergei Golubchik Date: Mon, 26 Oct 2020 17:21:26 +0100 Subject: [PATCH 12/31] test case for BUG#31650096 --- mysql-test/r/bootstrap_innodb.result | 7 +++++++ mysql-test/t/bootstrap_innodb.test | 27 +++++++++++++++++++++++++++ 2 files changed, 34 insertions(+) create mode 100644 mysql-test/r/bootstrap_innodb.result create mode 100644 mysql-test/t/bootstrap_innodb.test diff --git a/mysql-test/r/bootstrap_innodb.result b/mysql-test/r/bootstrap_innodb.result new file mode 100644 index 00000000000..2fc7800843b --- /dev/null +++ b/mysql-test/r/bootstrap_innodb.result @@ -0,0 +1,7 @@ +create table t1(a int) engine=innodb; +select * from t1; +a +1 +2 +5 +drop table t1; diff --git a/mysql-test/t/bootstrap_innodb.test b/mysql-test/t/bootstrap_innodb.test new file mode 100644 index 00000000000..ddaefb32155 --- /dev/null +++ b/mysql-test/t/bootstrap_innodb.test @@ -0,0 +1,27 @@ +source include/have_static_innodb.inc; +source include/not_embedded.inc; + +let $datadir= `select @@datadir`; + +create table t1(a int) engine=innodb; +source include/shutdown_mysqld.inc; + +write_file $MYSQLTEST_VARDIR/tmp/bootstrap_test.sql; +use test; +insert t1 values (1); +start transaction; +insert t1 values (2); +savepoint s1; +insert t1 values (3); +savepoint s2; +insert t1 values (4); +rollback to savepoint s1; +insert t1 values (5); +commit; +EOF +exec $MYSQLD_BOOTSTRAP_CMD --datadir=$datadir --innodb < $MYSQLTEST_VARDIR/tmp/bootstrap_test.sql >> $MYSQLTEST_VARDIR/tmp/bootstrap.log 2>&1; +remove_file $MYSQLTEST_VARDIR/tmp/bootstrap_test.sql; + +source include/start_mysqld.inc; +select * from t1; +drop table t1; From 3829b408d689182f05804ec045c9705da8de4e34 Mon Sep 17 00:00:00 2001 From: Vladislav Vaintroub Date: Tue, 27 Oct 2020 20:31:16 +0100 Subject: [PATCH 13/31] MDEV-24040 Named pipe permission issue Tighten access control - deny FILE_CREATE_PIPE_INSTANCE permission to everyone except current user (the one that runs mysqld) --- sql/mysqld.cc | 70 +++++++++++++++++++++++++++++++++++++++++---------- 1 file changed, 57 insertions(+), 13 deletions(-) diff --git a/sql/mysqld.cc b/sql/mysqld.cc index 8d00b5af948..3a5222d32de 100644 --- a/sql/mysqld.cc +++ b/sql/mysqld.cc @@ -2582,6 +2582,62 @@ static MYSQL_SOCKET activate_tcp_port(uint port) DBUG_RETURN(ip_sock); } +#ifdef _WIN32 +/* + Create a security descriptor for pipe. + - Use low integrity level, so that it is possible to connect + from any process. + - Give current user read/write access to pipe. + - Give Everyone read/write access to pipe minus FILE_CREATE_PIPE_INSTANCE +*/ +static void init_pipe_security_descriptor() +{ +#define SDDL_FMT "S:(ML;; NW;;; LW) D:(A;; 0x%08x;;; WD)(A;; FRFW;;; %s)" +#define EVERYONE_PIPE_ACCESS_MASK \ + (FILE_READ_DATA | FILE_READ_EA | FILE_READ_ATTRIBUTES | READ_CONTROL | \ + SYNCHRONIZE | FILE_WRITE_DATA | FILE_WRITE_EA | FILE_WRITE_ATTRIBUTES) + + /* + Figure out SID of the user that runs the server, then create SDDL string + for pipe permissions, and convert it to the security descriptor. + */ + char sddl_string[sizeof(SDDL_FMT) + 8 + SECURITY_MAX_SID_STRING_CHARACTERS]; + struct + { + TOKEN_USER token_user; + BYTE buffer[SECURITY_MAX_SID_SIZE]; + } token_buffer; + HANDLE token; + DWORD tmp; + + if (!OpenProcessToken(GetCurrentProcess(), TOKEN_QUERY, &token)) + goto fail; + + if (!GetTokenInformation(token, TokenUser, &token_buffer, + (DWORD) sizeof(token_buffer), &tmp)) + goto fail; + + CloseHandle(token); + + char *current_user_string_sid; + if (!ConvertSidToStringSid(token_buffer.token_user.User.Sid, + ¤t_user_string_sid)) + goto fail; + + snprintf(sddl_string, sizeof(sddl_string), SDDL_FMT, + EVERYONE_PIPE_ACCESS_MASK, current_user_string_sid); + LocalFree(current_user_string_sid); + + if (ConvertStringSecurityDescriptorToSecurityDescriptor(sddl_string, + SDDL_REVISION_1, &saPipeSecurity.lpSecurityDescriptor, 0)) + return; + +fail: + sql_perror("Can't start server : Initialize security descriptor"); + unireg_abort(1); +} +#endif + static void network_init(void) { #ifdef HAVE_SYS_UN_H @@ -2619,19 +2675,7 @@ static void network_init(void) strxnmov(pipe_name, sizeof(pipe_name)-1, "\\\\.\\pipe\\", mysqld_unix_port, NullS); - /* - Create a security descriptor for pipe. - - Use low integrity level, so that it is possible to connect - from any process. - - Give Everyone read/write access to pipe. - */ - if (!ConvertStringSecurityDescriptorToSecurityDescriptor( - "S:(ML;; NW;;; LW) D:(A;; FRFW;;; WD)", - SDDL_REVISION_1, &saPipeSecurity.lpSecurityDescriptor, NULL)) - { - sql_perror("Can't start server : Initialize security descriptor"); - unireg_abort(1); - } + init_pipe_security_descriptor(); saPipeSecurity.nLength = sizeof(SECURITY_ATTRIBUTES); saPipeSecurity.bInheritHandle = FALSE; if ((hPipe= CreateNamedPipe(pipe_name, From d5c9f84dfc72787c94ca773f83cceee9d93b2014 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20Lindstr=C3=B6m?= Date: Wed, 28 Oct 2020 14:00:37 +0200 Subject: [PATCH 14/31] MDEV-22707 : galera got stuck after flush tables Remove unnecessary condition and add necessary include for non debug Galera library. --- mysql-test/suite/galera/t/MDEV-22707.test | 1 + sql/sql_base.cc | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/mysql-test/suite/galera/t/MDEV-22707.test b/mysql-test/suite/galera/t/MDEV-22707.test index 19755f7c5ac..feb44e5e44f 100644 --- a/mysql-test/suite/galera/t/MDEV-22707.test +++ b/mysql-test/suite/galera/t/MDEV-22707.test @@ -6,6 +6,7 @@ --source include/have_innodb.inc --source include/have_debug.inc --source include/have_debug_sync.inc +--source include/galera_have_debug_sync.inc CREATE TABLE t1(f2 INT) ENGINE=InnoDB; diff --git a/sql/sql_base.cc b/sql/sql_base.cc index 497c6a0322f..635e17d1865 100644 --- a/sql/sql_base.cc +++ b/sql/sql_base.cc @@ -1861,7 +1861,7 @@ retry_share: #ifdef WITH_WSREP if (!((flags & MYSQL_OPEN_IGNORE_FLUSH) || - (wsrep_on(thd) && thd->wsrep_applier))) + (thd->wsrep_applier))) #else if (!(flags & MYSQL_OPEN_IGNORE_FLUSH)) #endif From cb04c1bc64031c7a2e5e0af16c97766b2a331c4a Mon Sep 17 00:00:00 2001 From: Vladislav Vaintroub Date: Wed, 28 Oct 2020 13:35:43 +0100 Subject: [PATCH 15/31] MDEV-24040 - fix appveyor build Old SDK is missing #define SECURITY_MAX_SID_STRING_CHARACTERS --- sql/mysqld.cc | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/sql/mysqld.cc b/sql/mysqld.cc index 3a5222d32de..433a6dacc01 100644 --- a/sql/mysqld.cc +++ b/sql/mysqld.cc @@ -2597,6 +2597,11 @@ static void init_pipe_security_descriptor() (FILE_READ_DATA | FILE_READ_EA | FILE_READ_ATTRIBUTES | READ_CONTROL | \ SYNCHRONIZE | FILE_WRITE_DATA | FILE_WRITE_EA | FILE_WRITE_ATTRIBUTES) +#ifndef SECURITY_MAX_SID_STRING_CHARACTERS +/* Old SDK does not have this constant */ +#define SECURITY_MAX_SID_STRING_CHARACTERS 187 +#endif + /* Figure out SID of the user that runs the server, then create SDDL string for pipe permissions, and convert it to the security descriptor. From 9e3e4c0e04509ebdac9017c334927fe7e2fbda5e Mon Sep 17 00:00:00 2001 From: Oleksandr Byelkin Date: Wed, 28 Oct 2020 20:54:29 +0100 Subject: [PATCH 16/31] new CC --- libmariadb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libmariadb b/libmariadb index 0cdc1656a70..8e5be108991 160000 --- a/libmariadb +++ b/libmariadb @@ -1 +1 @@ -Subproject commit 0cdc1656a70c52103b4329debf9ed02ccacfb3c2 +Subproject commit 8e5be108991ffc29887660fd6540ed0fd3428d1b From 6cb88685c46e74e7b4644b0bc64d283c969468fe Mon Sep 17 00:00:00 2001 From: Vlad Lesin Date: Tue, 27 Oct 2020 20:03:41 +0300 Subject: [PATCH 17/31] MDEV-24026: InnoDB: Failing assertion: os_total_large_mem_allocated >= size upon incremental backup mariabackup deallocated uninitialized write_filt_ctxt.u.wf_incremental_ctxt in xtrabackup_copy_datafile() when some table should be skipped due to parsed DDL redo log record. --- extra/mariabackup/backup_copy.cc | 2 +- extra/mariabackup/backup_copy.h | 4 ++++ extra/mariabackup/xtrabackup.cc | 11 ++++++++++- .../mariabackup/incremental_ddl_during_backup.result | 1 + .../mariabackup/incremental_ddl_during_backup.test | 6 +++++- 5 files changed, 21 insertions(+), 3 deletions(-) diff --git a/extra/mariabackup/backup_copy.cc b/extra/mariabackup/backup_copy.cc index 1cde4a4cf13..0ba220364e2 100644 --- a/extra/mariabackup/backup_copy.cc +++ b/extra/mariabackup/backup_copy.cc @@ -1448,7 +1448,7 @@ out: void backup_fix_ddl(void); -static lsn_t get_current_lsn(MYSQL *connection) +lsn_t get_current_lsn(MYSQL *connection) { static const char lsn_prefix[] = "\nLog sequence number "; lsn_t lsn = 0; diff --git a/extra/mariabackup/backup_copy.h b/extra/mariabackup/backup_copy.h index fbc09eaded3..7c886719f37 100644 --- a/extra/mariabackup/backup_copy.h +++ b/extra/mariabackup/backup_copy.h @@ -3,6 +3,7 @@ #define XTRABACKUP_BACKUP_COPY_H #include +#include #include "datasink.h" /* special files */ @@ -48,4 +49,7 @@ is_path_separator(char); bool directory_exists(const char *dir, bool create); +lsn_t +get_current_lsn(MYSQL *connection); + #endif diff --git a/extra/mariabackup/xtrabackup.cc b/extra/mariabackup/xtrabackup.cc index 668a355f0fa..364d1242b29 100644 --- a/extra/mariabackup/xtrabackup.cc +++ b/extra/mariabackup/xtrabackup.cc @@ -2578,6 +2578,8 @@ static my_bool xtrabackup_copy_datafile(fil_node_t *node, uint thread_n, return(FALSE); } + memset(&write_filt_ctxt, 0, sizeof(xb_write_filt_ctxt_t)); + bool was_dropped; pthread_mutex_lock(&backup_mutex); was_dropped = (ddl_tracker.drops.find(node->space->id) != ddl_tracker.drops.end()); @@ -2605,7 +2607,6 @@ static my_bool xtrabackup_copy_datafile(fil_node_t *node, uint thread_n, sizeof dst_name - 1); dst_name[sizeof dst_name - 1] = '\0'; - memset(&write_filt_ctxt, 0, sizeof(xb_write_filt_ctxt_t)); ut_a(write_filter.process != NULL); if (write_filter.init != NULL && @@ -2947,8 +2948,14 @@ static void dbug_mariabackup_event(const char *event,const char *key) } #define DBUG_MARIABACKUP_EVENT(A, B) DBUG_EXECUTE_IF("mariabackup_events", dbug_mariabackup_event(A,B);); +#define DBUG_MB_INJECT_CODE(EVENT, KEY, CODE) \ + DBUG_EXECUTE_IF("mariabackup_inject_code", {\ + char *env = getenv(EVENT); \ + if (env && !strcmp(env, KEY)) { CODE } \ + }) #else #define DBUG_MARIABACKUP_EVENT(A,B) +#define DBUG_MB_INJECT_CODE(EVENT, KEY, CODE) #endif /************************************************************************** @@ -2973,6 +2980,8 @@ data_copy_thread_func( while ((node = datafiles_iter_next(ctxt->it)) != NULL) { DBUG_MARIABACKUP_EVENT("before_copy", node->space->name); + DBUG_MB_INJECT_CODE("wait_innodb_redo_before_copy", node->space->name, + backup_wait_for_lsn(get_current_lsn(mysql_connection));); /* copy the datafile */ if (xtrabackup_copy_datafile(node, num, NULL, xtrabackup_incremental ? wf_incremental : wf_write_through)) diff --git a/mysql-test/suite/mariabackup/incremental_ddl_during_backup.result b/mysql-test/suite/mariabackup/incremental_ddl_during_backup.result index ffca1ef0e1f..e9746083178 100644 --- a/mysql-test/suite/mariabackup/incremental_ddl_during_backup.result +++ b/mysql-test/suite/mariabackup/incremental_ddl_during_backup.result @@ -2,6 +2,7 @@ call mtr.add_suppression("InnoDB: New log files created"); CREATE TABLE t1(i INT PRIMARY KEY) ENGINE INNODB; CREATE TABLE t2(i INT PRIMARY KEY) ENGINE INNODB; CREATE TABLE t3(i INT) ENGINE INNODB; +CREATE TABLE t10(i INT PRIMARY KEY) ENGINE INNODB; # Create full backup , modify table, then create incremental/differential backup INSERT into t1 values(1); # Prepare full backup, apply incremental one diff --git a/mysql-test/suite/mariabackup/incremental_ddl_during_backup.test b/mysql-test/suite/mariabackup/incremental_ddl_during_backup.test index 60780b13920..49e952eefea 100644 --- a/mysql-test/suite/mariabackup/incremental_ddl_during_backup.test +++ b/mysql-test/suite/mariabackup/incremental_ddl_during_backup.test @@ -8,6 +8,7 @@ let $incremental_dir=$MYSQLTEST_VARDIR/tmp/backup_inc1; CREATE TABLE t1(i INT PRIMARY KEY) ENGINE INNODB; CREATE TABLE t2(i INT PRIMARY KEY) ENGINE INNODB; CREATE TABLE t3(i INT) ENGINE INNODB; +CREATE TABLE t10(i INT PRIMARY KEY) ENGINE INNODB; echo # Create full backup , modify table, then create incremental/differential backup; --disable_result_log @@ -20,8 +21,11 @@ INSERT into t1 values(1); --let after_copy_test_t1=RENAME TABLE test.t1 TO test.t1_renamed --let after_copy_test_t2=DROP TABLE test.t2 --let after_copy_test_t3=CREATE INDEX a_i ON test.t3(i); +--let before_copy_test_t10=DROP TABLE test.t10 +--let wait_innodb_redo_before_copy=test/t10 -exec $XTRABACKUP --defaults-file=$MYSQLTEST_VARDIR/my.cnf --backup --target-dir=$incremental_dir --incremental-basedir=$basedir --dbug=+d,mariabackup_events; +# mariabackup should crash with assertion if MDEV-24026 is not fixed +exec $XTRABACKUP --defaults-file=$MYSQLTEST_VARDIR/my.cnf --backup --target-dir=$incremental_dir --incremental-basedir=$basedir --dbug=+d,mariabackup_events,mariabackup_inject_code; --let after_load_tablespaces= --disable_result_log echo # Prepare full backup, apply incremental one; From 4b854d47957173317294ed129690e11defdc8a8d Mon Sep 17 00:00:00 2001 From: Lawrin Novitsky Date: Tue, 27 Oct 2020 22:26:41 +0100 Subject: [PATCH 18/31] MDEV-19838 Wrong direxec param data caused crash In case of direct execution(stmtid=-1, mariadb_stmt_execute_direct in C API) application is in control of how many parameters client sends to the server. In case this number is not equal to actual query parameters number, the server may start to interprete packet data incorrectly, e.g. starting from the size of null bitmap. And that could cause it to crash at some point. The commit introduces some additional COM_STMT_EXECUTE packet sanity checks: - checking that "types sent" byte is set, and the value is equal to 1. if it's not direct execution, then that value is 0 or 1. - checking that parameter type value is a valid type, and parameter flags value is 0 or only "unsigned" bit is set - added more checks that read does not go beyond the end of the packet --- sql/sql_prepare.cc | 199 +++++++++++++++++++++++++++++++++++--- tests/mysql_client_test.c | 149 ++++++++++++++++++++++++++++ 2 files changed, 336 insertions(+), 12 deletions(-) diff --git a/sql/sql_prepare.cc b/sql/sql_prepare.cc index 1efb9d713bc..0df2617bb9a 100644 --- a/sql/sql_prepare.cc +++ b/sql/sql_prepare.cc @@ -123,6 +123,9 @@ When one supplies long data for a placeholder: #include "transaction.h" // trans_rollback_implicit #include "wsrep_mysqld.h" +/* Constants defining bits in parameter type flags. Flags are read from high byte of short value */ +static const uint PARAMETER_FLAG_UNSIGNED = 128U << 8; + /** A result class used to send cursor rows using the binary protocol. */ @@ -1003,11 +1006,73 @@ static bool insert_bulk_params(Prepared_statement *stmt, DBUG_RETURN(0); } -static bool set_conversion_functions(Prepared_statement *stmt, - uchar **data, uchar *data_end) + +/** + Checking if parameter type and flags are valid + + @param typecode ushort value with type in low byte, and flags in high byte + + @retval true this parameter is wrong + @retval false this parameter is OK +*/ + +static bool +parameter_type_sanity_check(ushort typecode) +{ + /* Checking if type in lower byte is valid */ + switch (typecode & 0xff) { + case MYSQL_TYPE_DECIMAL: + case MYSQL_TYPE_NEWDECIMAL: + case MYSQL_TYPE_TINY: + case MYSQL_TYPE_SHORT: + case MYSQL_TYPE_LONG: + case MYSQL_TYPE_LONGLONG: + case MYSQL_TYPE_INT24: + case MYSQL_TYPE_YEAR: + case MYSQL_TYPE_BIT: + case MYSQL_TYPE_FLOAT: + case MYSQL_TYPE_DOUBLE: + case MYSQL_TYPE_NULL: + case MYSQL_TYPE_VARCHAR: + case MYSQL_TYPE_TINY_BLOB: + case MYSQL_TYPE_MEDIUM_BLOB: + case MYSQL_TYPE_LONG_BLOB: + case MYSQL_TYPE_BLOB: + case MYSQL_TYPE_VAR_STRING: + case MYSQL_TYPE_STRING: + case MYSQL_TYPE_ENUM: + case MYSQL_TYPE_SET: + case MYSQL_TYPE_GEOMETRY: + case MYSQL_TYPE_TIMESTAMP: + case MYSQL_TYPE_DATE: + case MYSQL_TYPE_TIME: + case MYSQL_TYPE_DATETIME: + case MYSQL_TYPE_NEWDATE: + break; + /* + This types normally cannot be sent by client, so maybe it'd be + better to treat them like an error here. + */ + case MYSQL_TYPE_TIMESTAMP2: + case MYSQL_TYPE_TIME2: + case MYSQL_TYPE_DATETIME2: + default: + return true; + }; + + // In Flags in high byte only unsigned bit may be set + if (typecode & ((~PARAMETER_FLAG_UNSIGNED) & 0x0000ff00)) + { + return true; + } + return false; +} + +static bool +set_conversion_functions(Prepared_statement *stmt, uchar **data) { uchar *read_pos= *data; - const uint signed_bit= 1 << 15; + DBUG_ENTER("set_conversion_functions"); /* First execute or types altered by the client, setup the @@ -1020,12 +1085,17 @@ static bool set_conversion_functions(Prepared_statement *stmt, { ushort typecode; - if (read_pos >= data_end) - DBUG_RETURN(1); - + /* + stmt_execute_packet_sanity_check has already verified, that there + are enough data in the packet for data types + */ typecode= sint2korr(read_pos); read_pos+= 2; - (**it).unsigned_flag= MY_TEST(typecode & signed_bit); + if (parameter_type_sanity_check(typecode)) + { + DBUG_RETURN(1); + } + (**it).unsigned_flag= MY_TEST(typecode & PARAMETER_FLAG_UNSIGNED); setup_one_conversion_function(thd, *it, (uchar) (typecode & 0xff)); (*it)->sync_clones(); } @@ -1035,7 +1105,7 @@ static bool set_conversion_functions(Prepared_statement *stmt, static bool setup_conversion_functions(Prepared_statement *stmt, - uchar **data, uchar *data_end, + uchar **data, bool bulk_protocol= 0) { /* skip null bits */ @@ -1048,7 +1118,7 @@ static bool setup_conversion_functions(Prepared_statement *stmt, if (*read_pos++) //types supplied / first execute { *data= read_pos; - bool res= set_conversion_functions(stmt, data, data_end); + bool res= set_conversion_functions(stmt, data); DBUG_RETURN(res); } *data= read_pos; @@ -3159,11 +3229,20 @@ static void mysql_stmt_execute_common(THD *thd, void mysqld_stmt_execute(THD *thd, char *packet_arg, uint packet_length) { + const uint packet_min_lenght= 9; uchar *packet= (uchar*)packet_arg; // GCC 4.0.1 workaround + + DBUG_ENTER("mysqld_stmt_execute"); + + if (packet_length < packet_min_lenght) + { + my_error(ER_MALFORMED_PACKET, MYF(0), 0, + "", "mysqld_stmt_execute"); + DBUG_VOID_RETURN; + } ulong stmt_id= uint4korr(packet); ulong flags= (ulong) packet[4]; uchar *packet_end= packet + packet_length; - DBUG_ENTER("mysqld_stmt_execute"); packet+= 9; /* stmt_id + 5 bytes of flags */ @@ -3219,6 +3298,84 @@ void mysqld_stmt_bulk_execute(THD *thd, char *packet_arg, uint packet_length) DBUG_VOID_RETURN; } +/** + Additional packet checks for direct execution + + @param thd THD handle + @param stmt prepared statement being directly executed + @param paket packet with parameters to bind + @param packet_end pointer to the byte after parameters end + @param bulk_op is it bulk operation + @param direct_exec is it direct execution + @param read_bytes need to read types (only with bulk_op) + + @retval true this parameter is wrong + @retval false this parameter is OK +*/ + +static bool +stmt_execute_packet_sanity_check(Prepared_statement *stmt, + uchar *packet, uchar *packet_end, + bool bulk_op, bool direct_exec, + bool read_types) +{ + + DBUG_ASSERT((!read_types) || (read_types && bulk_op)); + if (stmt->param_count > 0) + { + uint packet_length= static_cast(packet_end - packet); + uint null_bitmap_bytes= (bulk_op ? 0 : (stmt->param_count + 7)/8); + uint min_len_for_param_count = null_bitmap_bytes + + (bulk_op ? 0 : 1); /* sent types byte */ + + if (!bulk_op && packet_length >= min_len_for_param_count) + { + if ((read_types= packet[null_bitmap_bytes])) + { + /* + Should be 0 or 1. If the byte is not 1, that could mean, + e.g. that we read incorrect byte due to incorrect number + of sent parameters for direct execution (i.e. null bitmap + is shorter or longer, than it should be) + */ + if (packet[null_bitmap_bytes] != '\1') + { + return true; + } + } + } + + if (read_types) + { + /* 2 bytes per parameter of the type and flags */ + min_len_for_param_count+= 2*stmt->param_count; + } + else + { + /* + If types are not sent, there is nothing to do here. + But for direct execution types should always be sent + */ + return direct_exec; + } + + /* + If true, the packet is guaranteed too short for the number of + parameters in the PS + */ + return (packet_length < min_len_for_param_count); + } + else + { + /* + If there is no parameters, this should be normally already end + of the packet. If it's not - then error + */ + return (packet_end > packet); + } + return false; +} + /** Common part of prepared statement execution @@ -3258,6 +3415,24 @@ static void mysql_stmt_execute_common(THD *thd, llstr(stmt_id, llbuf), "mysqld_stmt_execute"); DBUG_VOID_RETURN; } + + /* + In case of direct execution application decides how many parameters + to send. + + Thus extra checks are required to prevent crashes caused by incorrect + interpretation of the packet data. Plus there can be always a broken + evil client. + */ + if (stmt_execute_packet_sanity_check(stmt, packet, packet_end, bulk_op, + stmt_id == LAST_STMT_ID, read_types)) + { + char llbuf[22]; + my_error(ER_MALFORMED_PACKET, MYF(0), static_cast(sizeof(llbuf)), + llstr(stmt_id, llbuf), "mysqld_stmt_execute"); + DBUG_VOID_RETURN; + } + stmt->read_types= read_types; #if defined(ENABLED_PROFILING) @@ -4168,7 +4343,7 @@ Prepared_statement::set_parameters(String *expanded_query, { #ifndef EMBEDDED_LIBRARY uchar *null_array= packet; - res= (setup_conversion_functions(this, &packet, packet_end) || + res= (setup_conversion_functions(this, &packet) || set_params(this, null_array, packet, packet_end, expanded_query)); #else /* @@ -4400,7 +4575,7 @@ Prepared_statement::execute_bulk_loop(String *expanded_query, #ifndef EMBEDDED_LIBRARY if (read_types && - set_conversion_functions(this, &packet, packet_end)) + set_conversion_functions(this, &packet)) #else // bulk parameters are not supported for embedded, so it will an error #endif diff --git a/tests/mysql_client_test.c b/tests/mysql_client_test.c index 9a90862e93f..058168eedd5 100644 --- a/tests/mysql_client_test.c +++ b/tests/mysql_client_test.c @@ -19897,6 +19897,152 @@ static void test_ps_params_in_ctes() } +#ifndef EMBEDDED_LIBRARY +#define MDEV19838_MAX_PARAM_COUNT 32 +#define MDEV19838_FIELDS_COUNT 17 +static void test_mdev19838() +{ + int rc; + MYSQL_BIND bind[MDEV19838_MAX_PARAM_COUNT]; + unsigned int i, paramCount = 1; + char charvalue[] = "012345678901234567890123456789012345"; + MYSQL_STMT *stmt; + + myheader("test_mdev19838"); + + rc = mysql_query(mysql, "CREATE TABLE mdev19838(" + "f1 char(36)," + "f2 char(36)," + "f3 char(36)," + "f4 char(36)," + "f5 char(36)," + "f6 char(36)," + "f7 char(36)," + "f8 char(36)," + "f9 char(36)," + "f10 char(36)," + "f11 char(36)," + "f12 char(36)," + "f13 char(36)," + "f14 char(36)," + "f15 char(36)," + "f16 char(36)," + "f17 char(36)" + ")"); + myquery(rc); + + stmt = mysql_stmt_init(mysql); + check_stmt(stmt); + + memset(bind, 0, sizeof(bind)); + + for (i = 0; i < MDEV19838_MAX_PARAM_COUNT; ++i) + { + bind[i].buffer = charvalue; + bind[i].buffer_type = MYSQL_TYPE_STRING; + bind[i].buffer_length = strlen(charvalue) + 1; + bind[i].length = &bind[i].length_value; + bind[i].length_value = bind[i].buffer_length - 1; + } + + for (paramCount = 1; paramCount < MDEV19838_FIELDS_COUNT; ++paramCount) + { + mysql_stmt_attr_set(stmt, STMT_ATTR_PREBIND_PARAMS, ¶mCount); + + rc = mysql_stmt_bind_param(stmt, bind); + check_execute(stmt, rc); + + rc = mariadb_stmt_execute_direct(stmt, "INSERT INTO mdev19838" + "(f1, f2, f3, f4, f5, f6, f7, f8, f9, f10, f11, f12, f13, f14, f15, f16, f17)" + " VALUES " + "(?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)", -1); + + /* Expecting an error */ + DIE_UNLESS(rc != 0); + + mysql_stmt_close(stmt); + stmt = mysql_stmt_init(mysql); + check_stmt(stmt); + } + + paramCount = 0; + mysql_stmt_attr_set(stmt, STMT_ATTR_PREBIND_PARAMS, ¶mCount); + rc = mariadb_stmt_execute_direct(stmt, "INSERT INTO mdev19838(f1)" + " VALUES (?)", -1); + /* Expecting an error */ + DIE_UNLESS(rc != 0); + mysql_stmt_close(stmt); + + stmt = mysql_stmt_init(mysql); + check_stmt(stmt); + /* Correct number of parameters */ + paramCount = MDEV19838_FIELDS_COUNT; + mysql_stmt_attr_set(stmt, STMT_ATTR_PREBIND_PARAMS, ¶mCount); + mysql_stmt_bind_param(stmt, bind); + + rc = mariadb_stmt_execute_direct(stmt, "INSERT INTO mdev19838" + "(f1, f2, f3, f4, f5, f6, f7, f8, f9, f10, f11, f12, f13, f14, f15, f16, f17)" + " VALUES " + "(?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)", -1); + check_execute(stmt, rc); + + /* MYSQL_TYPE_TINY = 1. This parameter byte can be read as "parameters send" flag byte. + Checking that wrong packet is still detected */ + bind[0].buffer_type = MYSQL_TYPE_TINY; + bind[0].length_value = 1; + bind[0].buffer_length = 1; + + for (paramCount = 8; paramCount > 0; --paramCount) + { + mysql_stmt_close(stmt); + stmt = mysql_stmt_init(mysql); + check_stmt(stmt); + + mysql_stmt_attr_set(stmt, STMT_ATTR_PREBIND_PARAMS, ¶mCount); + + rc = mysql_stmt_bind_param(stmt, bind); + check_execute(stmt, rc); + + rc = mariadb_stmt_execute_direct(stmt, "INSERT INTO mdev19838" + "(f1, f2, f3, f4, f5, f6, f7, f8, f9)" + " VALUES " + "(?, ?, ?, ?, ?, ?, ?, ?, ?)", -1); + + /* Expecting an error */ + DIE_UNLESS(rc != 0); + } + + /* Test of query w/out parameters, with parameter sent and not sent */ + for (paramCount = MDEV19838_MAX_PARAM_COUNT; paramCount != (unsigned int)-1; --paramCount) + { + mysql_stmt_close(stmt); + stmt = mysql_stmt_init(mysql); + check_stmt(stmt); + + mysql_stmt_attr_set(stmt, STMT_ATTR_PREBIND_PARAMS, ¶mCount); + + if (paramCount > 0) + { + rc = mysql_stmt_bind_param(stmt, bind); + check_execute(stmt, rc); + } + + rc = mariadb_stmt_execute_direct(stmt, "INSERT INTO mdev19838" + "(f1)" + " VALUES " + "(0x1111111111111111)", -1); + + /* Expecting an error if parameters are sent */ + DIE_UNLESS(rc != 0 || paramCount == 0); + } + + mysql_stmt_close(stmt); + + rc = mysql_query(mysql, "drop table mdev19838"); + myquery(rc); +} +#endif // EMBEDDED_LIBRARY + static struct my_tests_st my_tests[]= { { "disable_query_logs", disable_query_logs }, { "test_view_sp_list_fields", test_view_sp_list_fields }, @@ -20182,6 +20328,9 @@ static struct my_tests_st my_tests[]= { { "test_bulk_replace", test_bulk_replace }, #endif { "test_ps_params_in_ctes", test_ps_params_in_ctes }, +#ifndef EMBEDDED_LIBRARY + { "test_mdev19838", test_mdev19838 }, +#endif { 0, 0 } }; From eae10a87ff60d1b419f103533d739caa8c5e467c Mon Sep 17 00:00:00 2001 From: Oleksandr Byelkin Date: Thu, 29 Oct 2020 08:29:03 +0100 Subject: [PATCH 19/31] Move result files at the correct place. --- mysql-test/{t => r}/aria_icp_debug.result | 0 mysql-test/{t => r}/innodb_icp_debug.result | 0 mysql-test/{t => r}/myisam_icp_debug.result | 0 3 files changed, 0 insertions(+), 0 deletions(-) rename mysql-test/{t => r}/aria_icp_debug.result (100%) rename mysql-test/{t => r}/innodb_icp_debug.result (100%) rename mysql-test/{t => r}/myisam_icp_debug.result (100%) diff --git a/mysql-test/t/aria_icp_debug.result b/mysql-test/r/aria_icp_debug.result similarity index 100% rename from mysql-test/t/aria_icp_debug.result rename to mysql-test/r/aria_icp_debug.result diff --git a/mysql-test/t/innodb_icp_debug.result b/mysql-test/r/innodb_icp_debug.result similarity index 100% rename from mysql-test/t/innodb_icp_debug.result rename to mysql-test/r/innodb_icp_debug.result diff --git a/mysql-test/t/myisam_icp_debug.result b/mysql-test/r/myisam_icp_debug.result similarity index 100% rename from mysql-test/t/myisam_icp_debug.result rename to mysql-test/r/myisam_icp_debug.result From 313cf9de2c9036e5acd5fef3be6afebbad471998 Mon Sep 17 00:00:00 2001 From: Sergei Golubchik Date: Thu, 29 Oct 2020 01:42:07 +0100 Subject: [PATCH 20/31] update result files after backport followup for 3e807d255e0e and eae10a87ff60 --- mysql-test/r/aria_icp_debug.result | 4 ---- mysql-test/r/innodb_icp_debug.result | 4 ---- mysql-test/r/myisam_icp_debug.result | 4 ---- 3 files changed, 12 deletions(-) diff --git a/mysql-test/r/aria_icp_debug.result b/mysql-test/r/aria_icp_debug.result index fc01ee4fb3b..26881d5a4cc 100644 --- a/mysql-test/r/aria_icp_debug.result +++ b/mysql-test/r/aria_icp_debug.result @@ -21,14 +21,10 @@ id select_type table type possible_keys key key_len ref rows Extra 1 SIMPLE t2 range kp1 kp1 5 NULL 10 Using index condition set debug_sync='handler_index_cond_check SIGNAL at_icp_check WAIT_FOR go'; select * from t2 where kp1 between 10 and 20 and kp2 +1 >100; -connect con1, localhost, root,,; -connection con1; set debug_sync='now WAIT_FOR at_icp_check'; kill query $target_id; set debug_sync='now SIGNAL go'; -connection default; ERROR 70100: Query execution was interrupted set debug_sync='RESET'; -disconnect con1; drop table t0,t1,t2; set default_storage_engine=default; diff --git a/mysql-test/r/innodb_icp_debug.result b/mysql-test/r/innodb_icp_debug.result index 5a169650c8e..f9f5dcc794c 100644 --- a/mysql-test/r/innodb_icp_debug.result +++ b/mysql-test/r/innodb_icp_debug.result @@ -21,14 +21,10 @@ id select_type table type possible_keys key key_len ref rows Extra 1 SIMPLE t2 range kp1 kp1 5 NULL 11 Using index condition set debug_sync='handler_index_cond_check SIGNAL at_icp_check WAIT_FOR go'; select * from t2 where kp1 between 10 and 20 and kp2 +1 >100; -connect con1, localhost, root,,; -connection con1; set debug_sync='now WAIT_FOR at_icp_check'; kill query $target_id; set debug_sync='now SIGNAL go'; -connection default; ERROR 70100: Query execution was interrupted set debug_sync='RESET'; -disconnect con1; drop table t0,t1,t2; set default_storage_engine=default; diff --git a/mysql-test/r/myisam_icp_debug.result b/mysql-test/r/myisam_icp_debug.result index cb45a0e2274..8c5caef7874 100644 --- a/mysql-test/r/myisam_icp_debug.result +++ b/mysql-test/r/myisam_icp_debug.result @@ -20,13 +20,9 @@ id select_type table type possible_keys key key_len ref rows Extra 1 SIMPLE t2 range kp1 kp1 5 NULL 11 Using index condition set debug_sync='handler_index_cond_check SIGNAL at_icp_check WAIT_FOR go'; select * from t2 where kp1 between 10 and 20 and kp2 +1 >100; -connect con1, localhost, root,,; -connection con1; set debug_sync='now WAIT_FOR at_icp_check'; kill query $target_id; set debug_sync='now SIGNAL go'; -connection default; ERROR 70100: Query execution was interrupted set debug_sync='RESET'; -disconnect con1; drop table t0,t1,t2; From d6302c9a47615019a3bca5304d3f61c587d826b8 Mon Sep 17 00:00:00 2001 From: Sergei Golubchik Date: Wed, 28 Oct 2020 20:44:03 +0100 Subject: [PATCH 21/31] MDEV-23702 calculating(auto rounding) issue Implement a different fix for "MDEV-19232: Floating point precision / value comparison problem" Instead of truncating decimal values after every division, truncate them for comparison purposes. This reverts commit 62d73df6b270 but keeps the test. --- mysql-test/r/func_group.result | 4 ++-- mysql-test/r/parser_precedence.result | 2 +- mysql-test/r/subselect4.result | 2 +- mysql-test/r/type_newdecimal.result | 7 +++++-- mysql-test/r/type_ranges.result | 6 ++++-- .../r/div_precision_increment_func.result | 16 ++++++++-------- mysql-test/suite/vcol/r/not_supported.result | 2 +- sql/item_cmpfunc.cc | 6 +++++- sql/item_func.cc | 2 -- sql/my_decimal.h | 11 +++++++++++ .../mysql-test/tokudb/r/type_newdecimal.result | 7 +++++-- .../mysql-test/tokudb/r/type_ranges.result | 6 ++++-- 12 files changed, 47 insertions(+), 24 deletions(-) diff --git a/mysql-test/r/func_group.result b/mysql-test/r/func_group.result index d0d413331df..06323f5b3bb 100644 --- a/mysql-test/r/func_group.result +++ b/mysql-test/r/func_group.result @@ -1186,13 +1186,13 @@ i count(*) std(e1/e2) 3 4 0.00000000 select std(s1/s2) from bug22555; std(s1/s2) -0.21328517 +0.21325764 select std(o1/o2) from bug22555; std(o1/o2) 0.2132576358664934 select std(e1/e2) from bug22555; std(e1/e2) -0.21328517 +0.21325764 set @saved_div_precision_increment=@@div_precision_increment; set div_precision_increment=19; select i, count(*), std(s1/s2) from bug22555 group by i order by i; diff --git a/mysql-test/r/parser_precedence.result b/mysql-test/r/parser_precedence.result index f23295bd61b..4330c8a2045 100644 --- a/mysql-test/r/parser_precedence.result +++ b/mysql-test/r/parser_precedence.result @@ -619,7 +619,7 @@ select 4 - 3 * 2, (4 - 3) * 2, 4 - (3 * 2); Testing that / is left associative select 15 / 5 / 3, (15 / 5) / 3, 15 / (5 / 3); 15 / 5 / 3 (15 / 5) / 3 15 / (5 / 3) -1.00000000 1.00000000 8.9998 +1.00000000 1.00000000 9.0000 Testing that / has precedence over | select 105 / 5 | 2, (105 / 5) | 2, 105 / (5 | 2); 105 / 5 | 2 (105 / 5) | 2 105 / (5 | 2) diff --git a/mysql-test/r/subselect4.result b/mysql-test/r/subselect4.result index e7655131fcf..b7161151928 100644 --- a/mysql-test/r/subselect4.result +++ b/mysql-test/r/subselect4.result @@ -2596,7 +2596,7 @@ SELECT population, area, population/area, cast(population/area as DECIMAL(20,9)) FROM t1 LIMIT 1; population area population/area cast(population/area as DECIMAL(20,9)) -11797 91 129.6374 129.637400000 +11797 91 129.6374 129.637362637 SELECT * FROM t1 A WHERE population/area = (SELECT MAX(population/area) from t1 B where A.region = B.region); region area population diff --git a/mysql-test/r/type_newdecimal.result b/mysql-test/r/type_newdecimal.result index 0c8f4f55442..6c9f3467b97 100644 --- a/mysql-test/r/type_newdecimal.result +++ b/mysql-test/r/type_newdecimal.result @@ -1530,8 +1530,11 @@ select (1.20396873 * 0.89550000 * 0.68000000 * 1.08721696 * 0.99500000 * 1.01500000 * 1.01500000 * 0.99500000) 0.812988073953673124592306939480 create table t1 as select 5.05 / 0.014; +Warnings: +Note 1265 Data truncated for column '5.05 / 0.014' at row 1 show warnings; Level Code Message +Note 1265 Data truncated for column '5.05 / 0.014' at row 1 show create table t1; Table Create Table t1 CREATE TABLE `t1` ( @@ -1646,6 +1649,8 @@ my_col 0.123456789123456789123456789123 DROP TABLE t1; CREATE TABLE t1 SELECT 1 / .123456789123456789123456789123456789123456789123456789123456789123456789123456789 AS my_col; +Warnings: +Note 1265 Data truncated for column 'my_col' at row 1 DESCRIBE t1; Field Type Null Key Default Extra my_col decimal(65,4) YES NULL @@ -1831,8 +1836,6 @@ DROP TABLE t1; CREATE TABLE t1 (a DECIMAL(30,30)); INSERT INTO t1 VALUES (0.1),(0.2),(0.3); CREATE TABLE t2 SELECT MIN(a + 0.0000000000000000000000000000001) AS c1 FROM t1; -Warnings: -Note 1265 Data truncated for column 'c1' at row 4 DESC t2; Field Type Null Key Default Extra c1 decimal(33,30) YES NULL diff --git a/mysql-test/r/type_ranges.result b/mysql-test/r/type_ranges.result index a10d2a56eae..101bf0cfb2c 100644 --- a/mysql-test/r/type_ranges.result +++ b/mysql-test/r/type_ranges.result @@ -94,6 +94,8 @@ DROP INDEX test ON t1; insert into t1 values (10, 1,1,1,1,1,1,1,1,1,1,1,1,1,NULL,0,0,0,1,1,1,1,'one','one'); insert into t1 values (NULL,2,2,2,2,2,2,2,2,2,2,2,2,2,NULL,NULL,NULL,NULL,NULL,NULL,2,2,'two','two,one'); insert into t1 values (0,1/3,3,3,3,3,3,3,3,3,3,3,3,3,NULL,'19970303','10:10:10','19970303101010','','','','3',3,3); +Warnings: +Warning 1265 Data truncated for column 'string' at row 1 insert into t1 values (0,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,NULL,19970807,080706,19970403090807,-1,-1,-1,'-1',-1,-1); Warnings: Warning 1264 Out of range value for column 'utiny' at row 1 @@ -134,7 +136,7 @@ select auto,string,tiny,short,medium,long_int,longlong,real_float,real_double,ut auto string tiny short medium long_int longlong real_float real_double utiny ushort umedium ulong ulonglong mod(floor(time_stamp/1000000),1000000)-mod(curdate(),1000000) date_field time_field date_time blob_col tinyblob_col mediumblob_col longblob_col 10 1 1 1 1 1 1 1.0 1.0000 1 00001 1 1 1 0 0000-00-00 00:00:00 0000-00-00 00:00:00 1 1 1 1 11 2 2 2 2 2 2 2.0 2.0000 2 00002 2 2 2 0 NULL NULL NULL NULL NULL 2 2 -12 0.3333 3 3 3 3 3 3.0 3.0000 3 00003 3 3 3 0 1997-03-03 10:10:10 1997-03-03 10:10:10 3 +12 0.33333333 3 3 3 3 3 3.0 3.0000 3 00003 3 3 3 0 1997-03-03 10:10:10 1997-03-03 10:10:10 3 13 -1 -1 -1 -1 -1 -1 -1.0 -1.0000 0 00000 0 0 0 0 1997-08-07 08:07:06 1997-04-03 09:08:07 -1 -1 -1 -1 14 -429496729 -128 -32768 -8388608 -2147483648 -4294967295 -4294967296.0 -4294967295.0000 0 00000 0 0 0 0 0000-00-00 00:00:00 0000-00-00 00:00:00 -4294967295 -4294967295 -4294967295 -4294967295 15 4294967295 127 32767 8388607 2147483647 4294967295 4294967296.0 4294967295.0000 255 65535 16777215 4294967295 4294967295 0 0000-00-00 00:00:00 0000-00-00 00:00:00 4294967295 4294967295 4294967295 4294967295 @@ -186,7 +188,7 @@ Warning 1265 Data truncated for column 'new_field' at row 7 select * from t2; auto string mediumblob_col new_field 1 2 2 ne -2 0.3333 ne +2 0.33333333 ne 3 -1 -1 ne 4 -429496729 -4294967295 ne 5 4294967295 4294967295 ne diff --git a/mysql-test/suite/sys_vars/r/div_precision_increment_func.result b/mysql-test/suite/sys_vars/r/div_precision_increment_func.result index ffe23eb3cef..ee8b7c5691d 100644 --- a/mysql-test/suite/sys_vars/r/div_precision_increment_func.result +++ b/mysql-test/suite/sys_vars/r/div_precision_increment_func.result @@ -50,9 +50,9 @@ INSERT into t1(name, salary, income_tax) values('Record_2', 501, 501*2.5/1000); INSERT into t1(name, salary, income_tax) values('Record_3', 210, 210*2.5/1000); SELECT * from t1; id name salary income_tax -1 Record_1 100011 250.03 -2 Record_2 501 1.25 -3 Record_3 210 0.53 +1 Record_1 100011 250.027 +2 Record_2 501 1.2525 +3 Record_3 210 0.525 ## Creating new connection ## ## Verifying session & global value of variable ## SELECT @@global.div_precision_increment = 2; @@ -67,11 +67,11 @@ INSERT into t1(name, salary, income_tax) values('Record_5', 501, 501*2.5/1000); INSERT into t1(name, salary, income_tax) values('Record_6', 210, 210*2.5/1000); SELECT * from t1; id name salary income_tax -1 Record_1 100011 250.03 -2 Record_2 501 1.25 -3 Record_3 210 0.53 -4 Record_4 100011 250.028 -5 Record_5 501 1.253 +1 Record_1 100011 250.027 +2 Record_2 501 1.2525 +3 Record_3 210 0.525 +4 Record_4 100011 250.027 +5 Record_5 501 1.2525 6 Record_6 210 0.525 ## Dropping table t1 ## drop table t1; diff --git a/mysql-test/suite/vcol/r/not_supported.result b/mysql-test/suite/vcol/r/not_supported.result index a3b73d3444f..06627fccf8b 100644 --- a/mysql-test/suite/vcol/r/not_supported.result +++ b/mysql-test/suite/vcol/r/not_supported.result @@ -51,7 +51,7 @@ a b v flush tables; select * from t1; a b v -1 2 0.3333000000000000000 +1 2 0.3333333330000000000 select * from t5; a b v 20141010 2 October diff --git a/sql/item_cmpfunc.cc b/sql/item_cmpfunc.cc index 61c6b980551..7e99230723a 100644 --- a/sql/item_cmpfunc.cc +++ b/sql/item_cmpfunc.cc @@ -914,6 +914,8 @@ int Arg_comparator::compare_decimal() { if (set_null) owner->null_value= 0; + my_decimal_round_if_needed(E_DEC_FATAL_ERROR, val1, (*a)->decimals, 0); + my_decimal_round_if_needed(E_DEC_FATAL_ERROR, val2, (*b)->decimals, 0); return my_decimal_cmp(val1, val2); } } @@ -938,7 +940,9 @@ int Arg_comparator::compare_e_decimal() my_decimal *val2= (*b)->val_decimal(&decimal2); if ((*a)->null_value || (*b)->null_value) return MY_TEST((*a)->null_value && (*b)->null_value); - return MY_TEST(my_decimal_cmp(val1, val2) == 0); + my_decimal_round_if_needed(E_DEC_FATAL_ERROR, val1, (*a)->decimals, 0); + my_decimal_round_if_needed(E_DEC_FATAL_ERROR, val2, (*b)->decimals, 0); + return my_decimal_cmp(val1, val2) == 0; } diff --git a/sql/item_func.cc b/sql/item_func.cc index b4cffc84b80..034fb385f05 100644 --- a/sql/item_func.cc +++ b/sql/item_func.cc @@ -1719,8 +1719,6 @@ my_decimal *Item_func_div::decimal_op(my_decimal *decimal_value) null_value= 1; return 0; } - my_decimal_round(E_DEC_FATAL_ERROR, decimal_value, - decimals, FALSE, decimal_value); return decimal_value; } diff --git a/sql/my_decimal.h b/sql/my_decimal.h index f318bcd1cd3..6a73cd54d50 100644 --- a/sql/my_decimal.h +++ b/sql/my_decimal.h @@ -303,6 +303,17 @@ bool my_decimal_is_zero(const my_decimal *decimal_value) } +inline +int my_decimal_round_if_needed(uint mask, my_decimal *dec, int scale, + bool truncate) +{ + if (scale >= dec->frac) + return E_DEC_OK; + return check_result(mask, decimal_round(dec, dec, scale, + (truncate ? TRUNCATE : HALF_UP))); +} + + inline int my_decimal_round(uint mask, const my_decimal *from, int scale, bool truncate, my_decimal *to) diff --git a/storage/tokudb/mysql-test/tokudb/r/type_newdecimal.result b/storage/tokudb/mysql-test/tokudb/r/type_newdecimal.result index dd47b8a1a5c..1f173bbb796 100644 --- a/storage/tokudb/mysql-test/tokudb/r/type_newdecimal.result +++ b/storage/tokudb/mysql-test/tokudb/r/type_newdecimal.result @@ -1531,8 +1531,11 @@ select (1.20396873 * 0.89550000 * 0.68000000 * 1.08721696 * 0.99500000 * 1.01500000 * 1.01500000 * 0.99500000) 0.812988073953673124592306939480 create table t1 as select 5.05 / 0.014; +Warnings: +Note 1265 Data truncated for column '5.05 / 0.014' at row 1 show warnings; Level Code Message +Note 1265 Data truncated for column '5.05 / 0.014' at row 1 show create table t1; Table Create Table t1 CREATE TABLE `t1` ( @@ -1647,6 +1650,8 @@ my_col 0.123456789123456789123456789123 DROP TABLE t1; CREATE TABLE t1 SELECT 1 / .123456789123456789123456789123456789123456789123456789123456789123456789123456789 AS my_col; +Warnings: +Note 1265 Data truncated for column 'my_col' at row 1 DESCRIBE t1; Field Type Null Key Default Extra my_col decimal(65,4) YES NULL @@ -1832,8 +1837,6 @@ DROP TABLE t1; CREATE TABLE t1 (a DECIMAL(30,30)); INSERT INTO t1 VALUES (0.1),(0.2),(0.3); CREATE TABLE t2 SELECT MIN(a + 0.0000000000000000000000000000001) AS c1 FROM t1; -Warnings: -Note 1265 Data truncated for column 'c1' at row 4 DESC t2; Field Type Null Key Default Extra c1 decimal(33,30) YES NULL diff --git a/storage/tokudb/mysql-test/tokudb/r/type_ranges.result b/storage/tokudb/mysql-test/tokudb/r/type_ranges.result index 395f21a8a8f..bd8491336b3 100644 --- a/storage/tokudb/mysql-test/tokudb/r/type_ranges.result +++ b/storage/tokudb/mysql-test/tokudb/r/type_ranges.result @@ -95,6 +95,8 @@ DROP INDEX test ON t1; insert into t1 values (10, 1,1,1,1,1,1,1,1,1,1,1,1,1,NULL,0,0,0,1,1,1,1,'one','one'); insert into t1 values (NULL,2,2,2,2,2,2,2,2,2,2,2,2,2,NULL,NULL,NULL,NULL,NULL,NULL,2,2,'two','two,one'); insert into t1 values (0,1/3,3,3,3,3,3,3,3,3,3,3,3,3,NULL,'19970303','10:10:10','19970303101010','','','','3',3,3); +Warnings: +Warning 1265 Data truncated for column 'string' at row 1 insert into t1 values (0,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,NULL,19970807,080706,19970403090807,-1,-1,-1,'-1',-1,-1); Warnings: Warning 1264 Out of range value for column 'utiny' at row 1 @@ -135,7 +137,7 @@ select auto,string,tiny,short,medium,long_int,longlong,real_float,real_double,ut auto string tiny short medium long_int longlong real_float real_double utiny ushort umedium ulong ulonglong mod(floor(time_stamp/1000000),1000000)-mod(curdate(),1000000) date_field time_field date_time blob_col tinyblob_col mediumblob_col longblob_col 10 1 1 1 1 1 1 1.0 1.0000 1 00001 1 1 1 0 0000-00-00 00:00:00 0000-00-00 00:00:00 1 1 1 1 11 2 2 2 2 2 2 2.0 2.0000 2 00002 2 2 2 0 NULL NULL NULL NULL NULL 2 2 -12 0.3333 3 3 3 3 3 3.0 3.0000 3 00003 3 3 3 0 1997-03-03 10:10:10 1997-03-03 10:10:10 3 +12 0.33333333 3 3 3 3 3 3.0 3.0000 3 00003 3 3 3 0 1997-03-03 10:10:10 1997-03-03 10:10:10 3 13 -1 -1 -1 -1 -1 -1 -1.0 -1.0000 0 00000 0 0 0 0 1997-08-07 08:07:06 1997-04-03 09:08:07 -1 -1 -1 -1 14 -429496729 -128 -32768 -8388608 -2147483648 -4294967295 -4294967296.0 -4294967295.0000 0 00000 0 0 0 0 0000-00-00 00:00:00 0000-00-00 00:00:00 -4294967295 -4294967295 -4294967295 -4294967295 15 4294967295 127 32767 8388607 2147483647 4294967295 4294967296.0 4294967295.0000 255 65535 16777215 4294967295 4294967295 0 0000-00-00 00:00:00 0000-00-00 00:00:00 4294967295 4294967295 4294967295 4294967295 @@ -187,7 +189,7 @@ Warning 1265 Data truncated for column 'new_field' at row 7 select * from t2; auto string mediumblob_col new_field 1 2 2 ne -2 0.3333 ne +2 0.33333333 ne 3 -1 -1 ne 4 -429496729 -4294967295 ne 5 4294967295 4294967295 ne From 17cf27f5b61aace463c3ae52df54c2363d2d4ca8 Mon Sep 17 00:00:00 2001 From: Sergei Golubchik Date: Mon, 26 Oct 2020 18:08:58 +0100 Subject: [PATCH 22/31] remove non-working debug assert and restore the test modified in the same commit (the non-replication related deadlock will be reported separately) --- mysql-test/suite/rpl/t/rpl_slave_grp_exec.test | 3 +-- mysys/my_error.c | 3 --- 2 files changed, 1 insertion(+), 5 deletions(-) diff --git a/mysql-test/suite/rpl/t/rpl_slave_grp_exec.test b/mysql-test/suite/rpl/t/rpl_slave_grp_exec.test index 84f4438dd76..e69c28500ae 100644 --- a/mysql-test/suite/rpl/t/rpl_slave_grp_exec.test +++ b/mysql-test/suite/rpl/t/rpl_slave_grp_exec.test @@ -101,8 +101,7 @@ ALTER TABLE t1 ADD PRIMARY KEY (a); ALTER TABLE t2 ADD PRIMARY KEY (a); ALTER TABLE t3 ADD PRIMARY KEY (a); -#--sync_slave_with_master ---connection slave +--sync_slave_with_master RENAME TABLE t3 TO t3_bak; --connection master diff --git a/mysys/my_error.c b/mysys/my_error.c index 20836269939..2bb9305b2fb 100644 --- a/mysys/my_error.c +++ b/mysys/my_error.c @@ -112,9 +112,6 @@ void my_error(uint nr, myf MyFlags, ...) char ebuff[ERRMSGSIZE]; DBUG_ENTER("my_error"); DBUG_PRINT("my", ("nr: %d MyFlags: %lu errno: %d", nr, MyFlags, errno)); - - if (errno == 1213) - DBUG_ASSERT(0); if (!(format = my_get_err_msg(nr))) (void) my_snprintf(ebuff, sizeof(ebuff), "Unknown error %d", nr); From 14798d3cd139430c1d23f14d810a27c5410f7161 Mon Sep 17 00:00:00 2001 From: Monty Date: Thu, 29 Oct 2020 16:20:57 +0200 Subject: [PATCH 23/31] MDEV-23159 Assertion `table_share->tmp_table != NO_TMP_TABLE || m_lock_type != 2'... The problem was that opt_sum_query() was, as part of MIN/MAX optimization, doing read operations on constant tables that where already closed Fixed by ensuring we don't try to read from tables that are closed. --- mysql-test/suite/maria/create.result | 33 ++++++++++++++++++++++++++++ mysql-test/suite/maria/create.test | 33 ++++++++++++++++++++++++++++ sql/opt_sum.cc | 24 +++++++++++++------- 3 files changed, 82 insertions(+), 8 deletions(-) diff --git a/mysql-test/suite/maria/create.result b/mysql-test/suite/maria/create.result index 82c6b8c9871..462c148df8e 100644 --- a/mysql-test/suite/maria/create.result +++ b/mysql-test/suite/maria/create.result @@ -31,3 +31,36 @@ select * from t2; f1 f2 3 qux DROP TABLE t1, t2; +# +# MDEV-23159 Assertion `table_share->tmp_table != NO_TMP_TABLE || +# m_lock_type != 2' + SIGSEGV in trnman_can_read_from +# (on optimized builds) +# +SET @org_sql_mode=@@SQL_MODE; +CREATE TABLE t1 (a INT PRIMARY KEY) ENGINE=Aria ROW_FORMAT=COMPRESSED; +INSERT INTO t1 VALUES(1); +CREATE TEMPORARY TABLE t2(b INT); +EXPLAIN SELECT * FROM t1 WHERE a IN (SELECT MAX(a) FROM t2); +id select_type table type possible_keys key key_len ref rows Extra +1 PRIMARY t1 system NULL NULL NULL NULL 1 Using index +2 DEPENDENT SUBQUERY NULL NULL NULL NULL NULL NULL NULL Select tables optimized away +DROP TABLE t1,t2; +SET SQL_MODE=''; +CREATE TABLE t1 (c INT PRIMARY KEY) ENGINE=Aria; +CREATE TABLE t2 (d INT); +INSERT INTO t1 VALUES (1); +SELECT c FROM t1 WHERE (c) IN (SELECT MIN(c) FROM t2); +c +DROP TABLE t1,t2; +USE test; +SET SQL_MODE='ONLY_FULL_GROUP_BY'; +CREATE TABLE t3 (c1 DECIMAL(1,1) PRIMARY KEY,c2 DATE,c3 NUMERIC(10) UNSIGNED) ENGINE=Aria; +CREATE TABLE t2 (f1 INTEGER ) ENGINE=Aria; +INSERT INTO t3 VALUES (0,0,0); +SELECT c1 FROM t3 WHERE (c1) IN (SELECT MIN(DISTINCT c1) FROM t2); +c1 +DROP TABLE t2,t3; +SET @@SQL_MODE=@org_sql_mode; +# +# End of 10.3 tests +# diff --git a/mysql-test/suite/maria/create.test b/mysql-test/suite/maria/create.test index 8f2ffd7492f..0efae8cac1b 100644 --- a/mysql-test/suite/maria/create.test +++ b/mysql-test/suite/maria/create.test @@ -40,3 +40,36 @@ INSERT IGNORE INTO t1 VALUES (1),(2); CREATE OR REPLACE TABLE t2 ENGINE=Aria AS SELECT SUM(a) AS f1, IFNULL( 'qux', ExtractValue( 'foo', 'bar' ) ) AS f2 FROM t1; select * from t2; DROP TABLE t1, t2; + +--echo # +--echo # MDEV-23159 Assertion `table_share->tmp_table != NO_TMP_TABLE || +--echo # m_lock_type != 2' + SIGSEGV in trnman_can_read_from +--echo # (on optimized builds) +--echo # + +SET @org_sql_mode=@@SQL_MODE; +CREATE TABLE t1 (a INT PRIMARY KEY) ENGINE=Aria ROW_FORMAT=COMPRESSED; +INSERT INTO t1 VALUES(1); +CREATE TEMPORARY TABLE t2(b INT); +EXPLAIN SELECT * FROM t1 WHERE a IN (SELECT MAX(a) FROM t2); +DROP TABLE t1,t2; + +SET SQL_MODE=''; +CREATE TABLE t1 (c INT PRIMARY KEY) ENGINE=Aria; +CREATE TABLE t2 (d INT); +INSERT INTO t1 VALUES (1); +SELECT c FROM t1 WHERE (c) IN (SELECT MIN(c) FROM t2); +DROP TABLE t1,t2; + +USE test; +SET SQL_MODE='ONLY_FULL_GROUP_BY'; +CREATE TABLE t3 (c1 DECIMAL(1,1) PRIMARY KEY,c2 DATE,c3 NUMERIC(10) UNSIGNED) ENGINE=Aria; +CREATE TABLE t2 (f1 INTEGER ) ENGINE=Aria; +INSERT INTO t3 VALUES (0,0,0); +SELECT c1 FROM t3 WHERE (c1) IN (SELECT MIN(DISTINCT c1) FROM t2); +DROP TABLE t2,t3; +SET @@SQL_MODE=@org_sql_mode; + +--echo # +--echo # End of 10.3 tests +--echo # diff --git a/sql/opt_sum.cc b/sql/opt_sum.cc index adfad29bb34..525d7390a26 100644 --- a/sql/opt_sum.cc +++ b/sql/opt_sum.cc @@ -399,20 +399,28 @@ int opt_sum_query(THD *thd, break; } longlong info_limit= 1; - table->file->info_push(INFO_KIND_FORCE_LIMIT_BEGIN, &info_limit); - if (likely(!(error= table->file->ha_index_init((uint) ref.key, 1)))) - error= (is_max ? - get_index_max_value(table, &ref, range_fl) : - get_index_min_value(table, &ref, item_field, range_fl, - prefix_len)); + error= 0; + table->file->info_push(INFO_KIND_FORCE_LIMIT_BEGIN, &info_limit); + if (!table->const_table) + { + if (likely(!(error= table->file->ha_index_init((uint) ref.key, + 1)))) + error= (is_max ? + get_index_max_value(table, &ref, range_fl) : + get_index_min_value(table, &ref, item_field, range_fl, + prefix_len)); + } /* Verify that the read tuple indeed matches the search key */ if (!error && reckey_in_range(is_max, &ref, item_field->field, conds, range_fl, prefix_len)) error= HA_ERR_KEY_NOT_FOUND; - table->file->ha_end_keyread(); - table->file->ha_index_end(); + if (!table->const_table) + { + table->file->ha_end_keyread(); + table->file->ha_index_end(); + } table->file->info_push(INFO_KIND_FORCE_LIMIT_END, NULL); if (error) { From 4c99e3e948dd14cfee9209e1a6274d94ecb5e22f Mon Sep 17 00:00:00 2001 From: Monty Date: Thu, 29 Oct 2020 17:36:49 +0200 Subject: [PATCH 24/31] Fixed bug in detection of getgrouplist parameters. On my system, OpenSuse, I got a compilation error that some arguments to getgrouplist() where not initialized --- plugin/auth_pam/CMakeLists.txt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/plugin/auth_pam/CMakeLists.txt b/plugin/auth_pam/CMakeLists.txt index ff7ba4a4f68..740aedeb7d9 100644 --- a/plugin/auth_pam/CMakeLists.txt +++ b/plugin/auth_pam/CMakeLists.txt @@ -14,8 +14,8 @@ CHECK_C_SOURCE_COMPILES( #include #include int main() { - char *arg_1; - gid_t arg_2, arg_3; + char *arg_1= 0; + gid_t arg_2=0, arg_3; int arg_4; (void)getgrouplist(arg_1,arg_2,&arg_3,&arg_4); return 0; From 14d43f4fa691e3af113195a3608f1fc401b85090 Mon Sep 17 00:00:00 2001 From: Monty Date: Thu, 29 Oct 2020 18:34:26 +0200 Subject: [PATCH 25/31] MDEV-23222 SIGSEG in maria_create() because of double free The crash happens because a double free in the case CREATE TABLE fails because there is a conflicting tables on disk. Fixed by ensuring that the double free can't happen. --- mysql-test/suite/maria/create.result | 11 +++++++++++ mysql-test/suite/maria/create.test | 22 ++++++++++++++++++++++ storage/maria/ma_create.c | 1 + 3 files changed, 34 insertions(+) diff --git a/mysql-test/suite/maria/create.result b/mysql-test/suite/maria/create.result index 462c148df8e..f1da0d7105b 100644 --- a/mysql-test/suite/maria/create.result +++ b/mysql-test/suite/maria/create.result @@ -62,5 +62,16 @@ c1 DROP TABLE t2,t3; SET @@SQL_MODE=@org_sql_mode; # +# MDEV-23222 SIGSEGV in maria_status | Assertion `(longlong) +# thd->status_var.local_memory_used >= 0 +# +CREATE TABLE t1 (a INT); +INSERT INTO t1 VALUES (1); +CREATE TABLE MDEV_23222 (i INT) DATA DIRECTORY = 'MYSQL_TMP_DIR', ENGINE=Aria TRANSACTIONAL=1;; +flush tables; +CREATE TABLE MDEV_23222 (i INT) DATA DIRECTORY = 'MYSQL_TMP_DIR', ENGINE=Aria TRANSACTIONAL=1;; +Got one of the listed errors +DROP TABLE t1; +# # End of 10.3 tests # diff --git a/mysql-test/suite/maria/create.test b/mysql-test/suite/maria/create.test index 0efae8cac1b..cac1f8bd7e1 100644 --- a/mysql-test/suite/maria/create.test +++ b/mysql-test/suite/maria/create.test @@ -70,6 +70,28 @@ SELECT c1 FROM t3 WHERE (c1) IN (SELECT MIN(DISTINCT c1) FROM t2); DROP TABLE t2,t3; SET @@SQL_MODE=@org_sql_mode; +--echo # +--echo # MDEV-23222 SIGSEGV in maria_status | Assertion `(longlong) +--echo # thd->status_var.local_memory_used >= 0 +--echo # + +let $mysqld_datadir= `select @@datadir`; +CREATE TABLE t1 (a INT); +INSERT INTO t1 VALUES (1); +--replace_result $MYSQL_TMP_DIR MYSQL_TMP_DIR +--eval CREATE TABLE MDEV_23222 (i INT) DATA DIRECTORY = '$MYSQL_TMP_DIR', ENGINE=Aria TRANSACTIONAL=1; +flush tables; +--remove_file $mysqld_datadir/test/MDEV_23222.frm +--replace_result $MYSQL_TMP_DIR MYSQL_TMP_DIR +--error 1,ER_TABLE_EXISTS_ERROR +--eval CREATE TABLE MDEV_23222 (i INT) DATA DIRECTORY = '$MYSQL_TMP_DIR', ENGINE=Aria TRANSACTIONAL=1; +DROP TABLE t1; +--disable_warnings +--remove_file $mysqld_datadir/test/MDEV_23222.MAD +--replace_result $MYSQL_TMP_DIR MYSQL_TMP_DIR +--remove_file $MYSQL_TMP_DIR/MDEV_23222.MAD +--enable_warnings + --echo # --echo # End of 10.3 tests --echo # diff --git a/storage/maria/ma_create.c b/storage/maria/ma_create.c index 98c33f896ac..8d6eb6edc48 100644 --- a/storage/maria/ma_create.c +++ b/storage/maria/ma_create.c @@ -1163,6 +1163,7 @@ int maria_create(const char *name, enum data_file_type datafile_type, FALSE, TRUE)) goto err; my_free(log_data); + log_data= 0; } if (!(flags & HA_DONT_TOUCH_DATA)) From f9b0ee07ef066625403ef64bf3486c2db7baed10 Mon Sep 17 00:00:00 2001 From: Oleksandr Byelkin Date: Thu, 29 Oct 2020 22:19:32 +0100 Subject: [PATCH 26/31] MDEV-19838: followup, fix for PS & embedded Use 9 byte (min length packet) --- libmysqld/lib_sql.cc | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/libmysqld/lib_sql.cc b/libmysqld/lib_sql.cc index d278bd39834..229e4e69e6a 100644 --- a/libmysqld/lib_sql.cc +++ b/libmysqld/lib_sql.cc @@ -330,7 +330,7 @@ static my_bool emb_read_query_result(MYSQL *mysql) static int emb_stmt_execute(MYSQL_STMT *stmt) { DBUG_ENTER("emb_stmt_execute"); - uchar header[5]; + uchar header[9]; THD *thd; my_bool res; @@ -342,6 +342,7 @@ static int emb_stmt_execute(MYSQL_STMT *stmt) int4store(header, stmt->stmt_id); header[4]= (uchar) stmt->flags; + header[5]= header[6]= header[7]= header[8]= 0; // safety thd= (THD*)stmt->mysql->thd; thd->client_param_count= stmt->param_count; thd->client_params= stmt->params; From a90b15837cb1a2138a07266cb8df42fcaa905229 Mon Sep 17 00:00:00 2001 From: Oleksandr Byelkin Date: Thu, 29 Oct 2020 22:20:21 +0100 Subject: [PATCH 27/31] MDEV-19838: fix of error messages --- sql/sql_prepare.cc | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/sql/sql_prepare.cc b/sql/sql_prepare.cc index 0df2617bb9a..11edd577309 100644 --- a/sql/sql_prepare.cc +++ b/sql/sql_prepare.cc @@ -3236,8 +3236,7 @@ void mysqld_stmt_execute(THD *thd, char *packet_arg, uint packet_length) if (packet_length < packet_min_lenght) { - my_error(ER_MALFORMED_PACKET, MYF(0), 0, - "", "mysqld_stmt_execute"); + my_error(ER_MALFORMED_PACKET, MYF(0)); DBUG_VOID_RETURN; } ulong stmt_id= uint4korr(packet); @@ -3427,9 +3426,7 @@ static void mysql_stmt_execute_common(THD *thd, if (stmt_execute_packet_sanity_check(stmt, packet, packet_end, bulk_op, stmt_id == LAST_STMT_ID, read_types)) { - char llbuf[22]; - my_error(ER_MALFORMED_PACKET, MYF(0), static_cast(sizeof(llbuf)), - llstr(stmt_id, llbuf), "mysqld_stmt_execute"); + my_error(ER_MALFORMED_PACKET, MYF(0)); DBUG_VOID_RETURN; } From 5482d62760bcbdcf44f1340fb5846c3942419dc5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20Lindstr=C3=B6m?= Date: Fri, 30 Oct 2020 09:19:29 +0200 Subject: [PATCH 28/31] Fix sporadic test failure on galera_parallel_apply_3nodes. Test itself is not deterministic. --- .../r/galera_parallel_apply_3nodes.result | 16 +++++++++++----- .../t/galera_parallel_apply_3nodes.test | 16 ++++++++++++++-- 2 files changed, 25 insertions(+), 7 deletions(-) diff --git a/mysql-test/suite/galera_3nodes/r/galera_parallel_apply_3nodes.result b/mysql-test/suite/galera_3nodes/r/galera_parallel_apply_3nodes.result index 4f9951c382f..4742fc1af9c 100644 --- a/mysql-test/suite/galera_3nodes/r/galera_parallel_apply_3nodes.result +++ b/mysql-test/suite/galera_3nodes/r/galera_parallel_apply_3nodes.result @@ -7,12 +7,18 @@ UPDATE t1 SET f1 = f1 + 10;; connection node_2; UPDATE t1 SET f1 = f1 + 100;; connection node_1; -connection node_2; -connection node_3; -SELECT f1 = 111 FROM t1; -f1 = 111 +SELECT COUNT(*) FROM t1; +COUNT(*) 1 -SELECT COUNT(*) IN (1, 2) FROM INFORMATION_SCHEMA.PROCESSLIST WHERE USER = 'system user' AND STATE LIKE 'committed%'; +connection node_2; +SELECT COUNT(*) FROM t1; +COUNT(*) +1 +connection node_3; +SELECT COUNT(*) FROM t1; +COUNT(*) +1 +SELECT COUNT(*) IN (1, 2) FROM INFORMATION_SCHEMA.PROCESSLIST WHERE USER = 'system user' AND STATE LIKE '%committed%'; COUNT(*) IN (1, 2) 1 SET GLOBAL wsrep_slave_threads = 1;; diff --git a/mysql-test/suite/galera_3nodes/t/galera_parallel_apply_3nodes.test b/mysql-test/suite/galera_3nodes/t/galera_parallel_apply_3nodes.test index 7d80d8036a1..f1168e59193 100644 --- a/mysql-test/suite/galera_3nodes/t/galera_parallel_apply_3nodes.test +++ b/mysql-test/suite/galera_3nodes/t/galera_parallel_apply_3nodes.test @@ -24,14 +24,26 @@ SET GLOBAL wsrep_slave_threads = 2; --send UPDATE t1 SET f1 = f1 + 100; --connection node_1 +# +# Note that test is not deterministic. We have following cases possible +# (1) Both updates are certified locally and then executed by the applier +# (2) Certification of update in node_1 fails because applier has started +# update from node_2 +# (3) Certification of update in node_2 fails because applier has started +# update from node_1 +# +--error 0,ER_LOCK_DEADLOCK --reap +SELECT COUNT(*) FROM t1; --connection node_2 +--error 0,ER_LOCK_DEADLOCK --reap +SELECT COUNT(*) FROM t1; --connection node_3 -SELECT f1 = 111 FROM t1; -SELECT COUNT(*) IN (1, 2) FROM INFORMATION_SCHEMA.PROCESSLIST WHERE USER = 'system user' AND STATE LIKE 'committed%'; +SELECT COUNT(*) FROM t1; +SELECT COUNT(*) IN (1, 2) FROM INFORMATION_SCHEMA.PROCESSLIST WHERE USER = 'system user' AND STATE LIKE '%committed%'; --eval SET GLOBAL wsrep_slave_threads = $wsrep_slave_threads_orig; From 5a0c34e4c2fd951119efb432eedcaa65a1d36606 Mon Sep 17 00:00:00 2001 From: Varun Gupta Date: Fri, 30 Oct 2020 14:56:57 +0530 Subject: [PATCH 29/31] MDEV-24033: SIGSEGV in __memcmp_avx2_movbe from queue_insert | SIGSEGV in __memcmp_avx2_movbe from native_compare The issue here was the system variable max_sort_length was being applied to decimals and it was truncating the value for decimals to the number of bytes set by max_sort_length. This was leading to a buffer overflow as the values were written to the buffer without truncation and then we moved the offset to the number of bytes(set by max_sort_length), that are needed for comparison. The fix is to not apply max_sort_length for fixed size types like INT, DECIMALS and only apply max_sort_length for CHAR, VARCHARS, TEXT and BLOBS. --- mysql-test/r/order_by.result | 20 ++++++++++++++++++++ mysql-test/t/order_by.test | 21 +++++++++++++++++++++ sql/field.h | 3 +++ sql/filesort.cc | 14 +++++++++++++- sql/sql_class.h | 2 ++ sql/sql_type.h | 2 ++ 6 files changed, 61 insertions(+), 1 deletion(-) diff --git a/mysql-test/r/order_by.result b/mysql-test/r/order_by.result index 1daf523e052..7bc8f562f1b 100644 --- a/mysql-test/r/order_by.result +++ b/mysql-test/r/order_by.result @@ -3432,4 +3432,24 @@ NULLIF(GROUP_CONCAT(v1), null) C B DROP TABLE t1; +# +# MDEV-24033: SIGSEGV in __memcmp_avx2_movbe from queue_insert | SIGSEGV in __memcmp_avx2_movbe from native_compare +# +SET @save_max_length_for_sort_data=@@max_length_for_sort_data; +SET @save_max_sort_length= @@max_sort_length; +SET @save_sql_select_limit= @@sql_select_limit; +CREATE TABLE t1 (a DECIMAL(64,0), b INT); +INSERT INTO t1 VALUES (1,1), (2,2), (3,3), (4,4); +SET max_length_for_sort_data= 30; +SET sql_select_limit = 3; +SET max_sort_length=8; +SELECT * FROM t1 ORDER BY a+1; +a b +1 1 +2 2 +3 3 +SET max_length_for_sort_data=@save_max_length_for_sort_data; +SET max_sort_length= @save_max_sort_length; +SET sql_select_limit= @save_sql_select_limit; +DROP TABLE t1; # End of 10.2 tests diff --git a/mysql-test/t/order_by.test b/mysql-test/t/order_by.test index f6886d6d45c..e9802f95721 100644 --- a/mysql-test/t/order_by.test +++ b/mysql-test/t/order_by.test @@ -2272,4 +2272,25 @@ ORDER BY id+1 DESC; DROP TABLE t1; +--echo # +--echo # MDEV-24033: SIGSEGV in __memcmp_avx2_movbe from queue_insert | SIGSEGV in __memcmp_avx2_movbe from native_compare +--echo # + +SET @save_max_length_for_sort_data=@@max_length_for_sort_data; +SET @save_max_sort_length= @@max_sort_length; +SET @save_sql_select_limit= @@sql_select_limit; + +CREATE TABLE t1 (a DECIMAL(64,0), b INT); +INSERT INTO t1 VALUES (1,1), (2,2), (3,3), (4,4); + +SET max_length_for_sort_data= 30; +SET sql_select_limit = 3; +SET max_sort_length=8; +SELECT * FROM t1 ORDER BY a+1; + +SET max_length_for_sort_data=@save_max_length_for_sort_data; +SET max_sort_length= @save_max_sort_length; +SET sql_select_limit= @save_sql_select_limit; +DROP TABLE t1; + --echo # End of 10.2 tests diff --git a/sql/field.h b/sql/field.h index 18e44f1d9d4..fea40251587 100644 --- a/sql/field.h +++ b/sql/field.h @@ -1339,6 +1339,8 @@ public: virtual uint max_packed_col_length(uint max_length) { return max_length;} + virtual bool is_packable() const { return false; } + uint offset(uchar *record) const { return (uint) (ptr - record); @@ -1827,6 +1829,7 @@ public: bool can_optimize_range(const Item_bool_func *cond, const Item *item, bool is_eq_func) const; + bool is_packable() const { return true; } }; /* base class for float and double and decimal (old one) */ diff --git a/sql/filesort.cc b/sql/filesort.cc index d76c39c3bd4..96f9aa874da 100644 --- a/sql/filesort.cc +++ b/sql/filesort.cc @@ -1971,7 +1971,14 @@ sortlength(THD *thd, SORT_FIELD *sortorder, uint s_length, if (sortorder->field) { CHARSET_INFO *cs= sortorder->field->sort_charset(); + sortorder->type= sortorder->field->is_packable() ? + SORT_FIELD_ATTR::VARIABLE_SIZE : + SORT_FIELD_ATTR::FIXED_SIZE; + sortorder->length= sortorder->field->sort_length(); + if (sortorder->is_variable_sized()) + set_if_smaller(sortorder->length, thd->variables.max_sort_length); + if (use_strnxfrm((cs=sortorder->field->sort_charset()))) { *multi_byte_charset= true; @@ -1982,6 +1989,10 @@ sortlength(THD *thd, SORT_FIELD *sortorder, uint s_length, } else { + sortorder->type= sortorder->item->type_handler()->is_packable() ? + SORT_FIELD_ATTR::VARIABLE_SIZE : + SORT_FIELD_ATTR::FIXED_SIZE; + sortorder->item->sortlength(thd, sortorder->item, sortorder); if (use_strnxfrm(sortorder->item->collation.collation)) { @@ -1990,7 +2001,8 @@ sortlength(THD *thd, SORT_FIELD *sortorder, uint s_length, if (sortorder->item->maybe_null) length++; // Place for NULL marker } - set_if_smaller(sortorder->length, thd->variables.max_sort_length); + if (sortorder->is_variable_sized()) + set_if_smaller(sortorder->length, thd->variables.max_sort_length); length+=sortorder->length; } sortorder->field= (Field*) 0; // end marker diff --git a/sql/sql_class.h b/sql/sql_class.h index d693cfa2727..2727b4c84d5 100644 --- a/sql/sql_class.h +++ b/sql/sql_class.h @@ -5449,6 +5449,8 @@ struct SORT_FIELD_ATTR { uint length; /* Length of sort field */ uint suffix_length; /* Length suffix (0-4) */ + enum Type { FIXED_SIZE, VARIABLE_SIZE } type; + bool is_variable_sized() { return type == VARIABLE_SIZE; } }; diff --git a/sql/sql_type.h b/sql/sql_type.h index 42090037ead..046b42e4a83 100644 --- a/sql/sql_type.h +++ b/sql/sql_type.h @@ -92,6 +92,7 @@ public: virtual void sortlength(THD *thd, const Type_std_attributes *item, SORT_FIELD_ATTR *attr) const= 0; + virtual bool is_packable() const { return false; } }; @@ -169,6 +170,7 @@ public: void sortlength(THD *thd, const Type_std_attributes *item, SORT_FIELD_ATTR *attr) const; + bool is_packable()const { return true; } }; From fbcd7c0c0659108624422bc845272fab775bdce5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20M=C3=A4kel=C3=A4?= Date: Fri, 30 Oct 2020 12:22:23 +0200 Subject: [PATCH 30/31] Update Connector/C --- libmariadb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libmariadb b/libmariadb index 8e5be108991..62427520a5b 160000 --- a/libmariadb +++ b/libmariadb @@ -1 +1 @@ -Subproject commit 8e5be108991ffc29887660fd6540ed0fd3428d1b +Subproject commit 62427520a5ba20e42fe51f5045062a7a9cadb466 From 72eea39d4c4a8bcadccfdac457e61abc7b618ff8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20M=C3=A4kel=C3=A4?= Date: Fri, 30 Oct 2020 12:58:16 +0200 Subject: [PATCH 31/31] MDEV-23991 fixup: Initialize the memory This regression was introduced in commit afc9d00c66db946c8240fe1fa6b345a3a8b6fec1. This is a partial backport of commit 199863d72b7cccaa4c75641c50c45a83b568ab8c from 10.4. --- storage/innobase/dict/dict0stats.cc | 3 +++ 1 file changed, 3 insertions(+) diff --git a/storage/innobase/dict/dict0stats.cc b/storage/innobase/dict/dict0stats.cc index bf2645fd65c..f8a4cebfb16 100644 --- a/storage/innobase/dict/dict0stats.cc +++ b/storage/innobase/dict/dict0stats.cc @@ -2249,6 +2249,9 @@ dict_stats_update_persistent( } ut_ad(!dict_index_is_ibuf(index)); + mutex_enter(&dict_sys->mutex); + dict_stats_empty_index(index, false); + mutex_exit(&dict_sys->mutex); index_stats_t stats = dict_stats_analyze_index(index);