From 85e8aee47d2e1cc58857148293f84ccd7e2ec620 Mon Sep 17 00:00:00 2001 From: Olivier Bertrand Date: Mon, 10 Mar 2014 12:21:17 +0100 Subject: [PATCH 001/279] - Temporary modified: storage/connect/catalog.h storage/connect/colblk.cpp storage/connect/colblk.h storage/connect/connect.cc storage/connect/filamap.cpp storage/connect/filamfix.cpp storage/connect/filamfix.h storage/connect/filamtxt.cpp storage/connect/filamvct.cpp storage/connect/filamzip.cpp storage/connect/filamzip.h storage/connect/ha_connect.cc storage/connect/ha_connect.h storage/connect/plgdbsem.h storage/connect/plgdbutl.cpp storage/connect/reldef.cpp storage/connect/reldef.h storage/connect/tabdos.cpp storage/connect/tabdos.h storage/connect/tabfix.cpp storage/connect/tabfix.h storage/connect/tabfmt.cpp storage/connect/tabfmt.h storage/connect/table.cpp storage/connect/tabmac.h storage/connect/tabmul.h storage/connect/tabmysql.cpp storage/connect/tabodbc.cpp storage/connect/tabsys.cpp storage/connect/tabsys.h storage/connect/tabtbl.cpp storage/connect/tabtbl.h storage/connect/tabvct.cpp storage/connect/tabvct.h storage/connect/tabwmi.cpp storage/connect/tabwmi.h storage/connect/tabxml.cpp storage/connect/tabxml.h storage/connect/valblk.cpp storage/connect/valblk.h storage/connect/value.cpp storage/connect/value.h storage/connect/xobject.cpp storage/connect/xobject.h storage/connect/xtable.h --- storage/connect/catalog.h | 1 + storage/connect/colblk.cpp | 2 + storage/connect/colblk.h | 7 + storage/connect/connect.cc | 19 +- storage/connect/filamap.cpp | 69 +- storage/connect/filamfix.cpp | 34 +- storage/connect/filamfix.h | 20 +- storage/connect/filamtxt.cpp | 62 +- storage/connect/filamvct.cpp | 29 +- storage/connect/filamzip.cpp | 611 ++++++++++++- storage/connect/filamzip.h | 4 +- storage/connect/ha_connect.cc | 400 +++++++-- storage/connect/ha_connect.h | 5 +- storage/connect/plgdbsem.h | 55 +- storage/connect/plgdbutl.cpp | 4 +- storage/connect/reldef.cpp | 21 +- storage/connect/reldef.h | 32 +- storage/connect/tabdos.cpp | 1571 +++++++++++++++++++++++++++++++-- storage/connect/tabdos.h | 121 ++- storage/connect/tabfix.cpp | 75 +- storage/connect/tabfix.h | 2 +- storage/connect/tabfmt.cpp | 7 +- storage/connect/tabfmt.h | 2 + storage/connect/table.cpp | 39 +- storage/connect/tabmac.h | 2 +- storage/connect/tabmul.h | 2 +- storage/connect/tabmysql.cpp | 12 +- storage/connect/tabodbc.cpp | 14 +- storage/connect/tabsys.cpp | 2 + storage/connect/tabsys.h | 2 +- storage/connect/tabtbl.cpp | 23 +- storage/connect/tabtbl.h | 4 +- storage/connect/tabvct.cpp | 19 +- storage/connect/tabvct.h | 2 +- storage/connect/tabwmi.cpp | 20 +- storage/connect/tabwmi.h | 2 +- storage/connect/tabxml.cpp | 2 + storage/connect/tabxml.h | 2 +- storage/connect/valblk.cpp | 143 ++- storage/connect/valblk.h | 48 +- storage/connect/value.cpp | 118 ++- storage/connect/value.h | 21 +- storage/connect/xobject.cpp | 4 +- storage/connect/xobject.h | 8 +- storage/connect/xtable.h | 75 +- 45 files changed, 3328 insertions(+), 389 deletions(-) diff --git a/storage/connect/catalog.h b/storage/connect/catalog.h index a61b1a53653..86d973e0036 100644 --- a/storage/connect/catalog.h +++ b/storage/connect/catalog.h @@ -44,6 +44,7 @@ typedef struct _colinfo { int Precision; int Scale; int Opt; + int Freq; char *Remark; char *Datefmt; char *Fieldfmt; diff --git a/storage/connect/colblk.cpp b/storage/connect/colblk.cpp index bce26e25387..364628bfca6 100644 --- a/storage/connect/colblk.cpp +++ b/storage/connect/colblk.cpp @@ -39,6 +39,7 @@ COLBLK::COLBLK(PCOLDEF cdp, PTDB tdbp, int i) Opt = cdp->Opt; Long = cdp->Long; Precision = cdp->Precision; + Freq = cdp->Freq; Buf_Type = cdp->Buf_Type; ColUse |= cdp->Flags; // Used by CONNECT Nullable = !!(cdp->Flags & U_NULLS); @@ -49,6 +50,7 @@ COLBLK::COLBLK(PCOLDEF cdp, PTDB tdbp, int i) Opt = 0; Long = 0; Precision = 0; + Freq = 0; Buf_Type = TYPE_ERROR; Nullable = false; Unsigned = false; diff --git a/storage/connect/colblk.h b/storage/connect/colblk.h index 7b84d237bd2..1204c4623f7 100644 --- a/storage/connect/colblk.h +++ b/storage/connect/colblk.h @@ -36,6 +36,10 @@ class DllExport COLBLK : public XOBJECT { virtual int GetAmType() {return TYPE_AM_ERROR;} virtual void SetOk(void) {Status |= BUF_EMPTY;} virtual PTDB GetTo_Tdb(void) {return To_Tdb;} +#if defined(BLK_INDX) + virtual int GetClustered(void) {return 0;} + virtual int IsClustered(void) {return FALSE;} +#endif // BLK_INDX PCOL GetNext(void) {return Next;} PSZ GetName(void) {return Name;} int GetIndex(void) {return Index;} @@ -76,7 +80,9 @@ class DllExport COLBLK : public XOBJECT { virtual void WriteColumn(PGLOBAL g); virtual void Print(PGLOBAL g, FILE *, uint); virtual void Print(PGLOBAL g, char *, uint); +#if defined(BLK_INDX) virtual bool VarSize(void) {return false;} +#endif // BLK_INDX virtual bool IsColInside(PCOL colp) {return this == colp;} bool InitValue(PGLOBAL g); @@ -94,6 +100,7 @@ class DllExport COLBLK : public XOBJECT { int Buf_Type; // Data type int Long; // Internal length in table int Precision; // Column length (as for ODBC) + int Freq; // Evaluated ceiling of distinct values FORMAT Format; // Output format ushort ColUse; // Column usage ushort Status; // Column read status diff --git a/storage/connect/connect.cc b/storage/connect/connect.cc index d111ff07044..f6d25b47ce7 100644 --- a/storage/connect/connect.cc +++ b/storage/connect/connect.cc @@ -433,7 +433,7 @@ RCODE EvalColumns(PGLOBAL g, PTDB tdbp, bool mrr) /***********************************************************************/ /* ReadNext: Read next record sequentially. */ /***********************************************************************/ -RCODE CntReadNext(PGLOBAL g, PTDB tdbp) +RCODE CntReadNext(PGLOBAL g, PTDB tdbp) { RCODE rc; @@ -449,8 +449,21 @@ RCODE CntReadNext(PGLOBAL g, PTDB tdbp) ((PTDBASE)tdbp)->SetKindex(NULL); } // endif index + // Save stack and allocation environment and prepare error return + if (g->jump_level == MAX_JUMP) { + strcpy(g->Message, MSG(TOO_MANY_JUMPS)); + return RC_FX; + } // endif jump_level + + if ((setjmp(g->jumper[++g->jump_level])) != 0) { + rc= RC_FX; + goto err; + } // endif rc + while ((rc= (RCODE)tdbp->ReadDB(g)) == RC_NF) ; - + + err: + g->jump_level--; return (rc != RC_OK) ? rc : EvalColumns(g, tdbp); } // end of CntReadNext @@ -578,7 +591,7 @@ int CntCloseTable(PGLOBAL g, PTDB tdbp) tbxp= (TDBDOX*)tdbp; tbxp->SetKindex(NULL); tbxp->To_Key_Col= NULL; - rc= tbxp->ResetTableOpt(g, ((PTDBASE)tdbp)->GetDef()->Indexable()); + rc= tbxp->ResetTableOpt(g, false, ((PTDBASE)tdbp)->GetDef()->Indexable()); err: if (xtrace > 1) diff --git a/storage/connect/filamap.cpp b/storage/connect/filamap.cpp index 5a67c5d2dd2..64e9ed15f40 100644 --- a/storage/connect/filamap.cpp +++ b/storage/connect/filamap.cpp @@ -1,11 +1,11 @@ /*********** File AM Map C++ Program Source Code File (.CPP) ***********/ /* PROGRAM NAME: FILAMAP */ /* ------------- */ -/* Version 1.4 */ +/* Version 1.5 */ /* */ /* COPYRIGHT: */ /* ---------- */ -/* (C) Copyright to the author Olivier BERTRAND 2005-2013 */ +/* (C) Copyright to the author Olivier BERTRAND 2005-2014 */ /* */ /* WHAT THIS PROGRAM DOES: */ /* ----------------------- */ @@ -322,8 +322,30 @@ int MAPFAM::ReadBuffer(PGLOBAL g) /*******************************************************************/ /* Record file position in case of UPDATE or DELETE. */ /*******************************************************************/ - Fpos = Mempos; - CurBlk = (int)Rows++; +#if defined(BLK_INDX) + int rc; + + next: +#endif // BLK_INDX + Fpos = Mempos; + CurBlk = (int)Rows++; + +#if defined(BLK_INDX) + /*******************************************************************/ + /* Check whether optimization on ROWID */ + /* can be done, as well as for join as for local filtering. */ + /*******************************************************************/ + switch (Tdbp->TestBlock(g)) { + case RC_EF: + return RC_EF; + case RC_NF: + // Skip this record + if ((rc = SkipRecord(g, FALSE)) != RC_OK) + return rc; + + goto next; + } // endswitch rc +#endif // BLK_INDX } else Placed = false; @@ -491,7 +513,11 @@ MBKFAM::MBKFAM(PDOSDEF tdp) : MAPFAM(tdp) Block = tdp->GetBlock(); Last = tdp->GetLast(); Nrec = tdp->GetElemt(); +#if defined(BLK_INDX) BlkPos = tdp->GetTo_Pos(); +#else // !BLK_INDX + BlkPos = NULL; +#endif // !BLK_INDX CurNum = Nrec; } // end of MBKFAM standard constructor @@ -537,6 +563,7 @@ int MBKFAM::GetRowID(void) /***********************************************************************/ int MBKFAM::ReadBuffer(PGLOBAL g) { +#if defined(BLK_INDX) int len; /*********************************************************************/ @@ -554,9 +581,21 @@ int MBKFAM::ReadBuffer(PGLOBAL g) /*******************************************************************/ CurNum = 0; + next: if (++CurBlk >= Block) return RC_EF; + /*******************************************************************/ + /* Before reading a new block, check whether block optimization */ + /* can be done, as well as for join as for local filtering. */ + /*******************************************************************/ + switch (Tdbp->TestBlock(g)) { + case RC_EF: + return RC_EF; + case RC_NF: + goto next; + } // endswitch rc + Fpos = Mempos = Memory + BlkPos[CurBlk]; } // endif's @@ -568,6 +607,10 @@ int MBKFAM::ReadBuffer(PGLOBAL g) memcpy(Tdbp->GetLine(), Fpos, len); Tdbp->GetLine()[len] = '\0'; return RC_OK; +#else // !BLK_POS + strcpy(g->Message, "This AM cannot be used in this version"); + return RC_FX; +#endif // !BLK_POS } // end of ReadBuffer /***********************************************************************/ @@ -657,10 +700,26 @@ int MPXFAM::ReadBuffer(PGLOBAL g) /* New block. */ /*******************************************************************/ CurNum = 0; - + +#if defined(BLK_INDX) + next: +#endif // BLK_INDX if (++CurBlk >= Block) return RC_EF; +#if defined(BLK_INDX) + /*******************************************************************/ + /* Before reading a new block, check whether block optimization */ + /* can be done, as well as for join as for local filtering. */ + /*******************************************************************/ + switch (Tdbp->TestBlock(g)) { + case RC_EF: + return RC_EF; + case RC_NF: + goto next; + } // endswitch rc +#endif // BLK_INDX + Fpos = Mempos = Headlen + Memory + CurBlk * Blksize; } // endif's diff --git a/storage/connect/filamfix.cpp b/storage/connect/filamfix.cpp index a92d9766933..295c281a478 100644 --- a/storage/connect/filamfix.cpp +++ b/storage/connect/filamfix.cpp @@ -1,11 +1,11 @@ /*********** File AM Fix C++ Program Source Code File (.CPP) ***********/ /* PROGRAM NAME: FILAMFIX */ /* ------------- */ -/* Version 1.4 */ +/* Version 1.5 */ /* */ /* COPYRIGHT: */ /* ---------- */ -/* (C) Copyright to the author Olivier BERTRAND 2005-2013 */ +/* (C) Copyright to the author Olivier BERTRAND 2005-2014 */ /* */ /* WHAT THIS PROGRAM DOES: */ /* ----------------------- */ @@ -168,9 +168,24 @@ int FIXFAM::ReadBuffer(PGLOBAL g) CurNum = 0; Tdbp->SetLine(To_Buf); +#if defined(BLK_INDX) + next: +#endif // BLK_INDX if (++CurBlk >= Block) return RC_EF; +#if defined(BLK_INDX) + /*****************************************************************/ + /* Before reading a new block, check whether block indexing */ + /* can be done, as well as for join as for local filtering. */ + /*****************************************************************/ + switch (Tdbp->TestBlock(g)) { + case RC_EF: + return RC_EF; + case RC_NF: + goto next; + } // endswitch rc +#endif // BLK_INDX } // endif's if (OldBlk == CurBlk) { @@ -1028,9 +1043,24 @@ int BGXFAM::ReadBuffer(PGLOBAL g) CurNum = 0; Tdbp->SetLine(To_Buf); +#if defined(BLK_INDX) + next: +#endif // BLK_INDX if (++CurBlk >= Block) return RC_EF; +#if defined(BLK_INDX) + /*****************************************************************/ + /* Before reading a new block, check whether block optimization */ + /* can be done, as well as for join as for local filtering. */ + /*****************************************************************/ + switch (Tdbp->TestBlock(g)) { + case RC_EF: + return RC_EF; + case RC_NF: + goto next; + } // endswitch rc +#endif // BLK_INDX } // endif's if (OldBlk == CurBlk) { diff --git a/storage/connect/filamfix.h b/storage/connect/filamfix.h index 758d891bf2c..b2c5019c635 100644 --- a/storage/connect/filamfix.h +++ b/storage/connect/filamfix.h @@ -33,16 +33,16 @@ class DllExport FIXFAM : public BLKFAM { virtual int Cardinality(PGLOBAL g) {return TXTFAM::Cardinality(g);} virtual int MaxBlkSize(PGLOBAL g, int s) {return TXTFAM::MaxBlkSize(g, s);} - virtual bool AllocateBuffer(PGLOBAL g); - virtual void ResetBuffer(PGLOBAL g); - virtual int ReadBuffer(PGLOBAL g); - virtual int WriteBuffer(PGLOBAL g); - virtual int DeleteRecords(PGLOBAL g, int irc); - virtual void CloseTableFile(PGLOBAL g); + virtual bool AllocateBuffer(PGLOBAL g); + virtual void ResetBuffer(PGLOBAL g); + virtual int ReadBuffer(PGLOBAL g); + virtual int WriteBuffer(PGLOBAL g); + virtual int DeleteRecords(PGLOBAL g, int irc); + virtual void CloseTableFile(PGLOBAL g); protected: - virtual bool CopyHeader(PGLOBAL g) {return false;} - virtual bool MoveIntermediateLines(PGLOBAL g, bool *b); + virtual bool CopyHeader(PGLOBAL g) {return false;} + virtual bool MoveIntermediateLines(PGLOBAL g, bool *b); // No additional members }; // end of class FIXFAM @@ -69,7 +69,7 @@ class BGXFAM : public FIXFAM { virtual bool OpenTableFile(PGLOBAL g); virtual int ReadBuffer(PGLOBAL g); virtual int WriteBuffer(PGLOBAL g); - virtual int DeleteRecords(PGLOBAL g, int irc); + virtual int DeleteRecords(PGLOBAL g, int irc); virtual void CloseTableFile(PGLOBAL g); virtual void Rewind(void); @@ -78,7 +78,7 @@ class BGXFAM : public FIXFAM { , int org = FILE_BEGIN); int BigRead(PGLOBAL g, HANDLE h, void *inbuf, int req); bool BigWrite(PGLOBAL g, HANDLE h, void *inbuf, int req); - virtual bool OpenTempFile(PGLOBAL g); + virtual bool OpenTempFile(PGLOBAL g); virtual bool MoveIntermediateLines(PGLOBAL g, bool *b = NULL); // Members diff --git a/storage/connect/filamtxt.cpp b/storage/connect/filamtxt.cpp index 1d3f17e2228..d7310f34e6b 100644 --- a/storage/connect/filamtxt.cpp +++ b/storage/connect/filamtxt.cpp @@ -1,11 +1,11 @@ /*********** File AM Txt C++ Program Source Code File (.CPP) ***********/ /* PROGRAM NAME: FILAMTXT */ /* ------------- */ -/* Version 1.4 */ +/* Version 1.5 */ /* */ /* COPYRIGHT: */ /* ---------- */ -/* (C) Copyright to the author Olivier BERTRAND 2005-2013 */ +/* (C) Copyright to the author Olivier BERTRAND 2005-2014 */ /* */ /* WHAT THIS PROGRAM DOES: */ /* ----------------------- */ @@ -235,13 +235,20 @@ int TXTFAM::Cardinality(PGLOBAL g) /***********************************************************************/ int TXTFAM::MaxBlkSize(PGLOBAL g, int s) { - int savcur = CurBlk, blm1 = Block - 1; + int rc = RC_OK, savcur = CurBlk, blm1 = Block - 1; int size, last = s - blm1 * Nrec; // Roughly estimate the table size as the sum of blocks // that can contain good rows for (size = 0, CurBlk = 0; CurBlk < Block; CurBlk++) +#if defined(BLK_INDX) + if ((rc = Tdbp->TestBlock(g)) == RC_OK) + size += (CurBlk == blm1) ? last : Nrec; + else if (rc == RC_EF) + break; +#else // !BLK_INDX size += (CurBlk == blm1) ? last : Nrec; +#endif // !BLK_INDX CurBlk = savcur; return size; @@ -543,6 +550,9 @@ int DOSFAM::ReadBuffer(PGLOBAL g) /*******************************************************************/ /* Record file position in case of UPDATE or DELETE. */ /*******************************************************************/ +#if defined(BLK_INDX) + next: +#endif // BLK_INDX if (RecordPos(g)) return RC_FX; @@ -551,6 +561,22 @@ int DOSFAM::ReadBuffer(PGLOBAL g) if (trace > 1) htrc("ReadBuffer: CurBlk=%d\n", CurBlk); +#if defined(BLK_INDX) + /*******************************************************************/ + /* Check whether optimization on ROWID */ + /* can be done, as well as for join as for local filtering. */ + /*******************************************************************/ + switch (Tdbp->TestBlock(g)) { + case RC_EF: + return RC_EF; + case RC_NF: + // Skip this record + if ((rc = SkipRecord(g, FALSE)) != RC_OK) + return rc; + + goto next; + } // endswitch rc +#endif // BLK_INDX } else Placed = false; @@ -993,7 +1019,11 @@ BLKFAM::BLKFAM(PDOSDEF tdp) : DOSFAM(tdp) Last = tdp->GetLast(); Nrec = tdp->GetElemt(); Closing = false; +#if defined(BLK_INDX) BlkPos = tdp->GetTo_Pos(); +#else // !BLK_INDX + BlkPos = NULL; +#endif // !BLK_INDX CurLine = NULL; NxtLine = NULL; OutBuf = NULL; @@ -1033,13 +1063,20 @@ int BLKFAM::Cardinality(PGLOBAL g) /***********************************************************************/ int BLKFAM::MaxBlkSize(PGLOBAL g, int s) { - int savcur = CurBlk; + int rc = RC_OK, savcur = CurBlk; int size; // Roughly estimate the table size as the sum of blocks // that can contain good rows for (size = 0, CurBlk = 0; CurBlk < Block; CurBlk++) +#if defined(BLK_INDX) + if ((rc = Tdbp->TestBlock(g)) == RC_OK) + size += (CurBlk == Block - 1) ? Last : Nrec; + else if (rc == RC_EF) + break; +#else // !BLK_INDX size += (CurBlk == Block - 1) ? Last : Nrec; +#endif // !BLK_INDX CurBlk = savcur; return size; @@ -1150,6 +1187,7 @@ int BLKFAM::SkipRecord(PGLOBAL g, bool header) /***********************************************************************/ int BLKFAM::ReadBuffer(PGLOBAL g) { +#if defined(BLK_INDX) int i, n, rc = RC_OK; /*********************************************************************/ @@ -1176,9 +1214,21 @@ int BLKFAM::ReadBuffer(PGLOBAL g) /*******************************************************************/ CurNum = 0; + next: if (++CurBlk >= Block) return RC_EF; + /*******************************************************************/ + /* Before reading a new block, check whether block optimization */ + /* can be done, as well as for join as for local filtering. */ + /*******************************************************************/ + switch (Tdbp->TestBlock(g)) { + case RC_EF: + return RC_EF; + case RC_NF: + goto next; + } // endswitch rc + } // endif's if (OldBlk == CurBlk) @@ -1241,6 +1291,10 @@ int BLKFAM::ReadBuffer(PGLOBAL g) // Store the current record file position for Delete and Update Fpos = BlkPos[CurBlk] + CurLine - To_Buf; return rc; +#else // !BLK_POS + strcpy(g->Message, "This AM cannot be used in this version"); + return RC_FX; +#endif // !BLK_POS } // end of ReadBuffer /***********************************************************************/ diff --git a/storage/connect/filamvct.cpp b/storage/connect/filamvct.cpp index 11c9ac69d4d..4887d7e52fd 100755 --- a/storage/connect/filamvct.cpp +++ b/storage/connect/filamvct.cpp @@ -1,11 +1,11 @@ /*********** File AM Vct C++ Program Source Code File (.CPP) ***********/ /* PROGRAM NAME: FILAMVCT */ /* ------------- */ -/* Version 2.4 */ +/* Version 2.5 */ /* */ /* COPYRIGHT: */ /* ---------- */ -/* (C) Copyright to the author Olivier BERTRAND 2005-2013 */ +/* (C) Copyright to the author Olivier BERTRAND 2005-2014 */ /* */ /* WHAT THIS PROGRAM DOES: */ /* ----------------------- */ @@ -249,13 +249,20 @@ bool VCTFAM::SetBlockInfo(PGLOBAL g) /***********************************************************************/ int VCTFAM::MaxBlkSize(PGLOBAL g, int s) { - int savcur = CurBlk; + int rc = RC_OK, savcur = CurBlk; int size; // Roughly estimate the table size as the sum of blocks // that can contain good rows for (size = 0, CurBlk = 0; CurBlk < Block; CurBlk++) +#if defined(BLK_INDX) + if ((rc = Tdbp->TestBlock(g)) == RC_OK) + size += (CurBlk == Block - 1) ? Last : Nrec; + else if (rc == RC_EF) + break; +#else // !BLK_INDX size += (CurBlk == Block - 1) ? Last : Nrec; +#endif // !BLK_INDX CurBlk = savcur; return size; @@ -572,9 +579,25 @@ int VCTFAM::ReadBuffer(PGLOBAL g) /*******************************************************************/ CurNum = 0; +#if defined(BLK_INDX) + next: +#endif // BLK_INDX if (++CurBlk == Block) return RC_EF; // End of file +#if defined(BLK_INDX) + /*******************************************************************/ + /* Before reading a new block, check whether block optimizing */ + /* can be done, as well as for join as for local filtering. */ + /*******************************************************************/ + switch (Tdbp->TestBlock(g)) { + case RC_EF: + return RC_EF; + case RC_NF: + goto next; + } // endswitch rc +#endif // BLK_INDX + num_there++; } // endif CurNum diff --git a/storage/connect/filamzip.cpp b/storage/connect/filamzip.cpp index 4f1a02d8257..04184bdda71 100644 --- a/storage/connect/filamzip.cpp +++ b/storage/connect/filamzip.cpp @@ -306,10 +306,30 @@ int ZIPFAM::ReadBuffer(PGLOBAL g) /*******************************************************************/ /* Record file position in case of UPDATE or DELETE. */ /*******************************************************************/ +#if defined(BLK_INDX) + next: +#endif // BLK_INDX if (RecordPos(g)) return RC_FX; CurBlk = Rows++; // Update RowID + +#if defined(BLK_INDX) + /*******************************************************************/ + /* Check whether optimization on ROWID */ + /* can be done, as well as for join as for local filtering. */ + /*******************************************************************/ + switch (Tdbp->TestBlock(g)) { + case RC_EF: + return RC_EF; + case RC_NF: + // Skip this record + if ((rc = SkipRecord(g, FALSE)) != RC_OK) + return rc; + + goto next; + } // endswitch rc +#endif // BLK_INDX } else Placed = false; @@ -403,7 +423,11 @@ ZBKFAM::ZBKFAM(PDOSDEF tdp) : ZIPFAM(tdp) CurLine = NULL; NxtLine = NULL; Closing = false; +#if defined(BLK_INDX) BlkPos = tdp->GetTo_Pos(); +#else // !BLK_INDX + BlkPos = NULL; +#endif // !BLK_INDX } // end of ZBKFAM standard constructor ZBKFAM::ZBKFAM(PZBKFAM txfp) : ZIPFAM(txfp) @@ -418,13 +442,20 @@ ZBKFAM::ZBKFAM(PZBKFAM txfp) : ZIPFAM(txfp) /***********************************************************************/ int ZBKFAM::MaxBlkSize(PGLOBAL g, int s) { - int savcur = CurBlk; + int rc = RC_OK, savcur = CurBlk; int size; // Roughly estimate the table size as the sum of blocks // that can contain good rows for (size = 0, CurBlk = 0; CurBlk < Block; CurBlk++) +#if defined(BLK_INDX) + if ((rc = Tdbp->TestBlock(g)) == RC_OK) + size += (CurBlk == Block - 1) ? Last : Nrec; + else if (rc == RC_EF) + break; +#else // !BLK_INDX size += (CurBlk == Block - 1) ? Last : Nrec; +#endif // !BLK_INDX CurBlk = savcur; return size; @@ -509,7 +540,8 @@ int ZBKFAM::SkipRecord(PGLOBAL g, bool header) /***********************************************************************/ int ZBKFAM::ReadBuffer(PGLOBAL g) { - int n, rc = RC_OK; +#if defined(BLK_INDX) + int n, skip, rc = RC_OK; /*********************************************************************/ /* Sequential reading when Placed is not true. */ @@ -532,10 +564,34 @@ int ZBKFAM::ReadBuffer(PGLOBAL g) /* New block. */ /*********************************************************************/ CurNum = 0; + skip = 0; + next: if (++CurBlk >= Block) return RC_EF; + /*********************************************************************/ + /* Before using the new block, check whether block optimization */ + /* can be done, as well as for join as for local filtering. */ + /*********************************************************************/ + switch (Tdbp->TestBlock(g)) { + case RC_EF: + return RC_EF; + case RC_NF: + skip++; + goto next; + } // endswitch rc + + if (skip) + // Skip blocks rejected by block optimization + for (int i = CurBlk - skip; i < CurBlk; i++) { + BlkLen = BlkPos[i + 1] - BlkPos[i]; + + if (gzseek(Zfile, (z_off_t)BlkLen, SEEK_CUR) < 0) + return Zerror(g); + + } // endfor i + BlkLen = BlkPos[CurBlk + 1] - BlkPos[CurBlk]; if (!(n = gzread(Zfile, To_Buf, BlkLen))) { @@ -559,6 +615,10 @@ int ZBKFAM::ReadBuffer(PGLOBAL g) rc = Zerror(g); return rc; +#else // !BLK_POS + strcpy(g->Message, "This AM cannot be used in this version"); + return RC_FX; +#endif // !BLK_POS } // end of ReadBuffer /***********************************************************************/ @@ -770,8 +830,33 @@ int ZIXFAM::ReadBuffer(PGLOBAL g) CurNum = 0; Tdbp->SetLine(To_Buf); -//if (++CurBlk >= Block) -// return RC_EF; +#if defined(BLK_INDX) + int skip = 0; + + next: + if (++CurBlk >= Block) + return RC_EF; + + /*********************************************************************/ + /* Before using the new block, check whether block optimization */ + /* can be done, as well as for join as for local filtering. */ + /*********************************************************************/ + switch (Tdbp->TestBlock(g)) { + case RC_EF: + return RC_EF; + case RC_NF: + skip++; + goto next; + } // endswitch rc + + if (skip) + // Skip blocks rejected by block optimization + for (int i = 0; i < skip; i++) { + if (gzseek(Zfile, (z_off_t)Buflen, SEEK_CUR) < 0) + return Zerror(g); + + } // endfor i +#endif // BLK_INDX if (!(n = gzread(Zfile, To_Buf, Buflen))) { rc = RC_EF; @@ -818,4 +903,522 @@ int ZIXFAM::WriteBuffer(PGLOBAL g) return RC_OK; } // end of WriteBuffer +#if defined(BLK_INDX) +/* --------------------------- Class ZLBFAM -------------------------- */ + +/***********************************************************************/ +/* Constructors. */ +/***********************************************************************/ +ZLBFAM::ZLBFAM(PDOSDEF tdp) : BLKFAM(tdp) + { + Zstream = NULL; + Zbuffer = NULL; + Zlenp = NULL; + Optimized = tdp->IsOptimized(); + } // end of ZLBFAM standard constructor + +ZLBFAM::ZLBFAM(PZLBFAM txfp) : BLKFAM(txfp) + { + Zstream = txfp->Zstream; + Zbuffer = txfp->Zbuffer; + Zlenp = txfp->Zlenp; + Optimized = txfp->Optimized; + } // end of ZLBFAM (dummy?) copy constructor + +/***********************************************************************/ +/* ZLB GetFileLength: returns an estimate of what would be the */ +/* uncompressed file size in number of bytes. */ +/***********************************************************************/ +int ZLBFAM::GetFileLength(PGLOBAL g) + { + int len = (Optimized) ? BlkPos[Block] : BLKFAM::GetFileLength(g); + + if (len > 0) + // Estimate size reduction to a max of 5 + len *= 5; + + return len; + } // end of GetFileLength + +/***********************************************************************/ +/* Allocate the line buffer. For mode Delete a bigger buffer has to */ +/* be allocated because is it also used to move lines into the file. */ +/***********************************************************************/ +bool ZLBFAM::AllocateBuffer(PGLOBAL g) + { + char *msg; + int n, zrc; + +#if 0 + if (!Optimized && Tdbp->NeedIndexing(g)) { + strcpy(g->Message, MSG(NOP_ZLIB_INDEX)); + return TRUE; + } // endif indexing +#endif // 0 + +#if defined(NOLIB) + if (!zlib && LoadZlib()) { + sprintf(g->Message, MSG(DLL_LOAD_ERROR), GetLastError(), "zlib.dll"); + return TRUE; + } // endif zlib +#endif + + BLKFAM::AllocateBuffer(g); +//Buflen = Nrec * (Lrecl + 2); +//Rbuf = Nrec; + + // Allocate the compressed buffer + n = Buflen + 16; // ????????????????????????????????? + Zlenp = (int*)PlugSubAlloc(g, NULL, n); + Zbuffer = (Byte*)(Zlenp + 1); + + // Allocate and initialize the Z stream + Zstream = (z_streamp)PlugSubAlloc(g, NULL, sizeof(z_stream)); + Zstream->zalloc = (alloc_func)0; + Zstream->zfree = (free_func)0; + Zstream->opaque = (voidpf)0; + Zstream->next_in = NULL; + Zstream->avail_in = 0; + + if (Tdbp->GetMode() == MODE_READ) { + msg = "inflateInit"; + zrc = inflateInit(Zstream); + } else { + msg = "deflateInit"; + zrc = deflateInit(Zstream, Z_DEFAULT_COMPRESSION); + } // endif Mode + + if (zrc != Z_OK) { + if (Zstream->msg) + sprintf(g->Message, "%s error: %s", msg, Zstream->msg); + else + sprintf(g->Message, "%s error: %d", msg, zrc); + + return TRUE; + } // endif zrc + + if (Tdbp->GetMode() == MODE_INSERT) { + // Write the file header block + if (Last == Nrec) { + CurBlk = Block; + CurNum = 0; + + if (!GetFileLength(g)) { + // Write the zlib header as an extra block + strcpy(To_Buf, "PlugDB"); + BlkLen = strlen("PlugDB") + 1; + + if (WriteCompressedBuffer(g)) + return TRUE; + + } // endif void file + + } else { + // In mode insert, if Last != Nrec, last block must be updated + CurBlk = Block - 1; + CurNum = Last; + + strcpy(g->Message, MSG(NO_PAR_BLK_INS)); + return TRUE; + } // endif Last + + } else { // MODE_READ + // First thing to do is to read the header block + void *rdbuf; + + if (Optimized) { + BlkLen = BlkPos[0]; + rdbuf = Zlenp; + } else { + // Get the stored length from the file itself + if (fread(Zlenp, sizeof(int), 1, Stream) != 1) + return FALSE; // Empty file + + BlkLen = *Zlenp; + rdbuf = Zbuffer; + } // endif Optimized + + switch (ReadCompressedBuffer(g, rdbuf)) { + case RC_EF: + return FALSE; + case RC_FX: +#if defined(UNIX) + sprintf(g->Message, MSG(READ_ERROR), To_File, strerror(errno)); +#else + sprintf(g->Message, MSG(READ_ERROR), To_File, _strerror(NULL)); +#endif + case RC_NF: + return TRUE; + } // endswitch + + // Some old tables can have PlugDB in their header + if (strcmp(To_Buf, "PlugDB")) { + sprintf(g->Message, MSG(BAD_HEADER), Tdbp->GetFile(g)); + return TRUE; + } // endif strcmp + + } // endif Mode + + return FALSE; + } // end of AllocateBuffer + +/***********************************************************************/ +/* GetPos: return the position of last read record. */ +/***********************************************************************/ +int ZLBFAM::GetPos(void) + { + return (Optimized) ? (CurNum + Nrec * CurBlk) : Fpos; + } // end of GetPos + +/***********************************************************************/ +/* GetNextPos: should not be called for this class. */ +/***********************************************************************/ +int ZLBFAM::GetNextPos(void) + { + if (Optimized) { + assert(FALSE); + return 0; + } else + return ftell(Stream); + + } // end of GetNextPos + +/***********************************************************************/ +/* ReadBuffer: Read one line for a text file. */ +/***********************************************************************/ +int ZLBFAM::ReadBuffer(PGLOBAL g) + { + int n; + void *rdbuf; + + /*********************************************************************/ + /* Sequential reading when Placed is not true. */ + /*********************************************************************/ + if (Placed) { + Placed = FALSE; + } else if (++CurNum < Rbuf) { + CurLine = NxtLine; + + // Get the position of the next line in the buffer + if (Tdbp->GetFtype() == RECFM_VAR) + while (*NxtLine++ != '\n') ; + else + NxtLine += Lrecl; + + // Set caller line buffer + n = NxtLine - CurLine - ((Tdbp->GetFtype() == RECFM_BIN) ? 0 : Ending); + memcpy(Tdbp->GetLine(), CurLine, n); + Tdbp->GetLine()[n] = '\0'; + return RC_OK; + } else if (Rbuf < Nrec && CurBlk != -1) { + CurNum--; // To have a correct Last value when optimizing + return RC_EF; + } else { + /*******************************************************************/ + /* New block. */ + /*******************************************************************/ + CurNum = 0; + + next: + if (++CurBlk >= Block) + return RC_EF; + + /*******************************************************************/ + /* Before reading a new block, check whether block optimization */ + /* can be done, as well as for join as for local filtering. */ + /*******************************************************************/ + if (Optimized) switch (Tdbp->TestBlock(g)) { + case RC_EF: + return RC_EF; + case RC_NF: + goto next; + } // endswitch rc + + } // endif's + + if (OldBlk == CurBlk) + goto ok; // Block is already there + + if (Optimized) { + // Store the position of next block + Fpos = BlkPos[CurBlk]; + + // fseek is required only in non sequential reading + if (CurBlk != OldBlk + 1) + if (fseek(Stream, Fpos, SEEK_SET)) { + sprintf(g->Message, MSG(FSETPOS_ERROR), Fpos); + return RC_FX; + } // endif fseek + + // Calculate the length of block to read + BlkLen = BlkPos[CurBlk + 1] - Fpos; + rdbuf = Zlenp; + } else { // !Optimized + if (CurBlk != OldBlk + 1) { + strcpy(g->Message, MSG(INV_RAND_ACC)); + return RC_FX; + } else + Fpos = ftell(Stream); // Used when optimizing + + // Get the stored length from the file itself + if (fread(Zlenp, sizeof(int), 1, Stream) != 1) { + if (feof(Stream)) + return RC_EF; + + goto err; + } // endif fread + + BlkLen = *Zlenp; + rdbuf = Zbuffer; + } // endif Optimized + + // Read the next block + switch (ReadCompressedBuffer(g, rdbuf)) { + case RC_FX: goto err; + case RC_NF: return RC_FX; + case RC_EF: return RC_EF; + default: Rbuf = (CurBlk == Block - 1) ? Last : Nrec; + } // endswitch ReadCompressedBuffer + + ok: + if (Tdbp->GetFtype() == RECFM_VAR) { + int i; + + // Get the position of the current line + for (i = 0, CurLine = To_Buf; i < CurNum; i++) + while (*CurLine++ != '\n') ; // What about Unix ??? + + // Now get the position of the next line + for (NxtLine = CurLine; *NxtLine++ != '\n';) ; + + // Set caller line buffer + n = NxtLine - CurLine - Ending; + } else { + CurLine = To_Buf + CurNum * Lrecl; + NxtLine = CurLine + Lrecl; + n = Lrecl - ((Tdbp->GetFtype() == RECFM_BIN) ? 0 : Ending); + } // endif Ftype + + memcpy(Tdbp->GetLine(), CurLine, n); + Tdbp->GetLine()[n] = '\0'; + + OldBlk = CurBlk; // Last block actually read + IsRead = TRUE; // Is read indeed + return RC_OK; + + err: +#if defined(UNIX) + sprintf(g->Message, MSG(READ_ERROR), To_File, strerror(errno)); +#else + sprintf(g->Message, MSG(READ_ERROR), To_File, _strerror(NULL)); +#endif + return RC_FX; + } // end of ReadBuffer + +/***********************************************************************/ +/* Read and decompress a block from the stream. */ +/***********************************************************************/ +int ZLBFAM::ReadCompressedBuffer(PGLOBAL g, void *rdbuf) + { + if (fread(rdbuf, 1, (size_t)BlkLen, Stream) == (unsigned)BlkLen) { + int zrc; + + num_read++; + + if (Optimized && BlkLen != signed(*Zlenp + sizeof(int))) { + sprintf(g->Message, MSG(BAD_BLK_SIZE), CurBlk + 1); + return RC_NF; + } // endif BlkLen + + // HERE WE MUST INFLATE THE BLOCK + Zstream->next_in = Zbuffer; + Zstream->avail_in = (uInt)(*Zlenp); + Zstream->next_out = (Byte*)To_Buf; + Zstream->avail_out = Buflen; + zrc = inflate(Zstream, Z_SYNC_FLUSH); + + if (zrc != Z_OK) { + if (Zstream->msg) + sprintf(g->Message, MSG(FUNC_ERR_S), "inflate", Zstream->msg); + else + sprintf(g->Message, MSG(FUNCTION_ERROR), "inflate", (int)zrc); + + return RC_NF; + } // endif zrc + + } else if (feof(Stream)) { + return RC_EF; + } else + return RC_FX; + + return RC_OK; + } // end of ReadCompressedBuffer + +/***********************************************************************/ +/* WriteBuffer: File write routine for DOS access method. */ +/* Update is directly written back into the file, */ +/* with this (fast) method, record size cannot change. */ +/***********************************************************************/ +int ZLBFAM::WriteBuffer(PGLOBAL g) + { + assert (Tdbp->GetMode() == MODE_INSERT); + + /*********************************************************************/ + /* Prepare the write buffer. */ + /*********************************************************************/ + if (!Closing) { + if (Tdbp->GetFtype() == RECFM_BIN) + memcpy(CurLine, Tdbp->GetLine(), Lrecl); + else + strcat(strcpy(CurLine, Tdbp->GetLine()), CrLf); + +#if defined(_DEBUG) + if (Tdbp->GetFtype() == RECFM_FIX && + (signed)strlen(CurLine) != Lrecl + (signed)strlen(CrLf)) { + strcpy(g->Message, MSG(BAD_LINE_LEN)); + Closing = TRUE; + return RC_FX; + } // endif Lrecl +#endif // _DEBUG + } // endif Closing + + /*********************************************************************/ + /* In Insert mode, blocs are added sequentialy to the file end. */ + /*********************************************************************/ + if (++CurNum != Rbuf) { + if (Tdbp->GetFtype() == RECFM_VAR) + CurLine += strlen(CurLine); + else + CurLine += Lrecl; + + return RC_OK; // We write only full blocks + } // endif CurNum + + // HERE WE MUST DEFLATE THE BLOCK + if (Tdbp->GetFtype() == RECFM_VAR) + NxtLine = CurLine + strlen(CurLine); + else + NxtLine = CurLine + Lrecl; + + BlkLen = NxtLine - To_Buf; + + if (WriteCompressedBuffer(g)) { + Closing = TRUE; // To tell CloseDB about a Write error + return RC_FX; + } // endif WriteCompressedBuffer + + CurBlk++; + CurNum = 0; + CurLine = To_Buf; + return RC_OK; + } // end of WriteBuffer + +/***********************************************************************/ +/* Compress the buffer and write the deflated output to stream. */ +/***********************************************************************/ +bool ZLBFAM::WriteCompressedBuffer(PGLOBAL g) + { + int zrc; + + Zstream->next_in = (Byte*)To_Buf; + Zstream->avail_in = (uInt)BlkLen; + Zstream->next_out = Zbuffer; + Zstream->avail_out = Buflen + 16; + Zstream->total_out = 0; + zrc = deflate(Zstream, Z_FULL_FLUSH); + + if (zrc != Z_OK) { + if (Zstream->msg) + sprintf(g->Message, MSG(FUNC_ERR_S), "deflate", Zstream->msg); + else + sprintf(g->Message, MSG(FUNCTION_ERROR), "deflate", (int)zrc); + + return TRUE; + } else + *Zlenp = Zstream->total_out; + + // Now start the writing process. + BlkLen = *Zlenp + sizeof(int); + + if (fwrite(Zlenp, 1, BlkLen, Stream) != (size_t)BlkLen) { + sprintf(g->Message, MSG(FWRITE_ERROR), strerror(errno)); + return TRUE; + } // endif size + + return FALSE; + } // end of WriteCompressedBuffer + +/***********************************************************************/ +/* Table file close routine for DOS access method. */ +/***********************************************************************/ +void ZLBFAM::CloseTableFile(PGLOBAL g) + { + int rc = RC_OK; + + if (Tdbp->GetMode() == MODE_INSERT) { + PCATLG cat = PlgGetCatalog(g); + LPCSTR name = Tdbp->GetName(); + PDOSDEF defp = (PDOSDEF)Tdbp->GetDef(); + + // Closing is True if last Write was in error + if (CurNum && !Closing) { + // Some more inserted lines remain to be written + Last = (Nrec - Rbuf) + CurNum; + Block = CurBlk + 1; + Rbuf = CurNum--; + Closing = TRUE; + rc = WriteBuffer(g); + } else if (Rbuf == Nrec) { + Last = Nrec; + Block = CurBlk; + } // endif CurNum + + if (rc != RC_FX) { + defp->SetBlock(Block); + defp->SetLast(Last); + cat->SetIntCatInfo("Blocks", Block); + cat->SetIntCatInfo("Last", Last); + } // endif + + fclose(Stream); + } else + rc = fclose(Stream); + +#ifdef DEBTRACE + htrc("ZLB CloseTableFile: closing %s mode=%d rc=%d\n", + To_File, Tdbp->GetMode(), rc); +#endif + + Stream = NULL; // So we can know whether table is open + To_Fb->Count = 0; // Avoid double closing by PlugCloseAll + + if (Tdbp->GetMode() == MODE_READ) + rc = inflateEnd(Zstream); + else + rc = deflateEnd(Zstream); + + } // end of CloseTableFile + +/***********************************************************************/ +/* Rewind routine for ZLIB access method. */ +/***********************************************************************/ +void ZLBFAM::Rewind(void) + { + // We must be positioned after the header block + if (CurBlk >= 0) { // Nothing to do if no block read yet + if (!Optimized) { // If optimized, fseek will be done in ReadBuffer + rewind(Stream); + fread(Zlenp, sizeof(int), 1, Stream); + fseek(Stream, *Zlenp + sizeof(int), SEEK_SET); + OldBlk = -1; + } // endif Optimized + + CurBlk = -1; + CurNum = Rbuf; + } // endif CurBlk + +//OldBlk = -1; +//Rbuf = 0; commented out in case we reuse last read block + } // end of Rewind +#endif // BLK_INDX /* ------------------------ End of ZipFam ---------------------------- */ diff --git a/storage/connect/filamzip.h b/storage/connect/filamzip.h index f6f28ca5315..67d9553a4e6 100644 --- a/storage/connect/filamzip.h +++ b/storage/connect/filamzip.h @@ -122,7 +122,7 @@ class DllExport ZIXFAM : public ZBKFAM { // No additional Members }; // end of class ZIXFAM -#ifdef NOT_USED +#if defined(BLK_INDX) /***********************************************************************/ /* This is the DOS/UNIX Access Method class declaration for PlugDB */ /* fixed/variable files compressed using the zlib library functions. */ @@ -166,6 +166,6 @@ class DllExport ZLBFAM : public BLKFAM { int *Zlenp; // Pointer to block length bool Optimized; // true when opt file is available }; // end of class ZLBFAM -#endif // NOT_USED +#endif // BLK_INDX #endif // __FILAMZIP_H diff --git a/storage/connect/ha_connect.cc b/storage/connect/ha_connect.cc index f19cac9da50..f301ed2dcae 100644 --- a/storage/connect/ha_connect.cc +++ b/storage/connect/ha_connect.cc @@ -252,8 +252,8 @@ ha_create_table_option connect_table_option_list[]= ha_create_table_option connect_field_option_list[]= { HA_FOPTION_NUMBER("FLAG", offset, (ulonglong) -1, 0, INT_MAX32, 1), - HA_FOPTION_NUMBER("FREQUENCY", freq, 0, 0, INT_MAX32, 1), // not used - HA_FOPTION_NUMBER("OPT_VALUE", opt, 0, 0, 2, 1), // used for indexing + HA_FOPTION_NUMBER("MAX_DIST", freq, 0, 0, INT_MAX32, 1), // BLK_INDX + HA_FOPTION_NUMBER("DISTRIB", opt, 0, 0, 2, 1), // used for BLK_INDX HA_FOPTION_NUMBER("FIELD_LENGTH", fldlen, 0, 0, INT_MAX32, 1), HA_FOPTION_STRING("DATE_FORMAT", dateformat), HA_FOPTION_STRING("FIELD_FORMAT", fieldformat), @@ -331,11 +331,12 @@ DllExport LPCSTR PlugSetPath(LPSTR to, LPCSTR name, LPCSTR dir) delete_table method in handler.cc */ static const char *ha_connect_exts[]= { - ".dos", ".fix", ".csv",".bin", ".fmt", ".dbf", ".xml", ".ini", ".vec", + ".dos", ".fix", ".csv", ".bin", ".fmt", ".dbf", ".xml", ".ini", ".vec", ".dnx", ".fnx", ".bnx", ".vnx", ".dbx", - NULL -}; - +#if defined(BLK_INDX) + ".dop", ".fop", ".bop", ".vop", +#endif // BLK_INDX + NULL}; /** @brief @@ -453,7 +454,7 @@ static handler* connect_create_handler(handlerton *hton, handler *h= new (mem_root) ha_connect(hton, table); if (xtrace) - printf("New CONNECT %p, table: %s\n", + htrc("New CONNECT %p, table: %s\n", h, table ? table->table_name.str : ""); return h; @@ -500,7 +501,7 @@ ha_connect::ha_connect(handlerton *hton, TABLE_SHARE *table_arg) ha_connect::~ha_connect(void) { if (xtrace) - printf("Delete CONNECT %p, table: %s, xp=%p count=%d\n", this, + htrc("Delete CONNECT %p, table: %s, xp=%p count=%d\n", this, table ? table->s->table_name.str : "", xp, xp ? xp->count : 0); @@ -932,12 +933,12 @@ void *ha_connect::GetColumnOption(PGLOBAL g, void *field, PCOLINFO pcf) if (fop) { pcf->Offset= (int)fop->offset; -// pcf->Freq= fop->freq; + pcf->Freq= (int)fop->freq; pcf->Datefmt= (char*)fop->dateformat; pcf->Fieldfmt= (char*)fop->fieldformat; } else { pcf->Offset= -1; -// pcf->Freq= 0; + pcf->Freq= 0; pcf->Datefmt= NULL; pcf->Fieldfmt= NULL; } // endif fop @@ -1046,7 +1047,7 @@ PIXDEF ha_connect::GetIndexInfo(TABLE_SHARE *s) for (int n= 0; (unsigned)n < s->keynames.count; n++) { if (xtrace) - printf("Getting created index %d info\n", n + 1); + htrc("Getting created index %d info\n", n + 1); // Find the index to describe kp= s->key_info[n]; @@ -1182,7 +1183,7 @@ PTDB ha_connect::GetTDB(PGLOBAL g) valid_query_id= xp->last_query_id; tp->SetMode(xmod); } else - printf("GetTDB: %s\n", g->Message); + htrc("GetTDB: %s\n", g->Message); return tp; } // end of GetTDB @@ -1197,7 +1198,7 @@ int ha_connect::OpenTable(PGLOBAL g, bool del) // Double test to be on the safe side if (!g || !table) { - printf("OpenTable logical error; g=%p table=%p\n", g, table); + htrc("OpenTable logical error; g=%p table=%p\n", g, table); return HA_ERR_INITIALIZATION; } // endif g @@ -1278,9 +1279,8 @@ int ha_connect::OpenTable(PGLOBAL g, bool del) PIXDEF oldpix= GetIndexInfo(); } // endif xmod -// tdbp->SetOrig((PTBX)table); // used by CheckCond } else - printf("OpenTable: %s\n", g->Message); + htrc("OpenTable: %s\n", g->Message); if (rc) { tdbp= NULL; @@ -1334,7 +1334,7 @@ int ha_connect::MakeRecord(char *buf) DBUG_ENTER("ha_connect::MakeRecord"); if (xtrace > 1) - printf("Maps: read=%08X write=%08X vcol=%08X defr=%08X defw=%08X\n", + htrc("Maps: read=%08X write=%08X vcol=%08X defr=%08X defw=%08X\n", *table->read_set->bitmap, *table->write_set->bitmap, *table->vcol_set->bitmap, *table->def_read_set.bitmap, *table->def_write_set.bitmap); @@ -1372,7 +1372,7 @@ int ha_connect::MakeRecord(char *buf) if (mrr) continue; #endif // MRRBKA_SUPPORT - printf("Column %s not found\n", fp->field_name); + htrc("Column %s not found\n", fp->field_name); dbug_tmp_restore_column_map(table->write_set, org_bitmap); DBUG_RETURN(HA_ERR_WRONG_IN_RECORD); } // endif colp @@ -1487,7 +1487,7 @@ int ha_connect::ScanRecord(PGLOBAL g, uchar *buf) break; if (!colp) { - printf("Column %s not found\n", fp->field_name); + htrc("Column %s not found\n", fp->field_name); rc= HA_ERR_WRONG_IN_RECORD; goto err; } else @@ -1612,16 +1612,16 @@ const char *ha_connect::GetValStr(OPVAL vop, bool neg) val= (neg) ? " NOT IN (" : " IN ("; break; case OP_NULL: - val= " IS NULL"; + val= (neg) ? " IS NOT NULL" : " IS NULL"; break; case OP_LIKE: val= " LIKE "; break; case OP_XX: - val= " BETWEEN "; + val= (neg) ? " NOT BETWEEN " : " BETWEEN "; break; case OP_EXIST: - val= " EXISTS "; + val= (neg) ? " NOT EXISTS " : " EXISTS "; break; case OP_AND: val= " AND "; @@ -1656,22 +1656,204 @@ const char *ha_connect::GetValStr(OPVAL vop, bool neg) } // end of GetValStr +#if defined(BLK_INDX) /***********************************************************************/ -/* Check the WHERE condition and return an ODBC/WQL filter. */ +/* Check the WHERE condition and return a CONNECT filter. */ /***********************************************************************/ -PFIL ha_connect::CheckCond(PGLOBAL g, PFIL filp, AMT tty, Item *cond) +PFIL ha_connect::CondFilter(PGLOBAL g, Item *cond) +{ + unsigned int i; + bool ismul= false; + OPVAL vop= OP_XX; + PFIL filp= NULL; + + if (!cond) + return NULL; + + if (xtrace) + htrc("Cond type=%d\n", cond->type()); + + if (cond->type() == COND::COND_ITEM) { + PFIL fp; + Item_cond *cond_item= (Item_cond *)cond; + + if (xtrace) + htrc("Cond: Ftype=%d name=%s\n", cond_item->functype(), + cond_item->func_name()); + + switch (cond_item->functype()) { + case Item_func::COND_AND_FUNC: vop= OP_AND; break; + case Item_func::COND_OR_FUNC: vop= OP_OR; break; + default: return NULL; + } // endswitch functype + + List* arglist= cond_item->argument_list(); + List_iterator li(*arglist); + Item *subitem; + + for (i= 0; i < arglist->elements; i++) + if ((subitem= li++)) { + if (!(fp= CondFilter(g, subitem))) { + if (vop == OP_OR) + return NULL; + } else + filp= (filp) ? MakeFilter(g, filp, vop, fp) : fp; + + } else + return NULL; + + } else if (cond->type() == COND::FUNC_ITEM) { + unsigned int i; + bool iscol, neg= FALSE; + PCOL colp[2]= {NULL,NULL}; + PPARM pfirst= NULL, pprec= NULL; + POPER pop; + Item_func *condf= (Item_func *)cond; + Item* *args= condf->arguments(); + + if (xtrace) + htrc("Func type=%d argnum=%d\n", condf->functype(), + condf->argument_count()); + + switch (condf->functype()) { + case Item_func::EQUAL_FUNC: + case Item_func::EQ_FUNC: vop= OP_EQ; break; + case Item_func::NE_FUNC: vop= OP_NE; break; + case Item_func::LT_FUNC: vop= OP_LT; break; + case Item_func::LE_FUNC: vop= OP_LE; break; + case Item_func::GE_FUNC: vop= OP_GE; break; + case Item_func::GT_FUNC: vop= OP_GT; break; + case Item_func::IN_FUNC: vop= OP_IN; + case Item_func::BETWEEN: + ismul= true; + neg= ((Item_func_opt_neg *)condf)->negated; + break; + default: return NULL; + } // endswitch functype + + pop= (POPER)PlugSubAlloc(g, NULL, sizeof(OPER)); + pop->Name= NULL; + pop->Val=vop; + pop->Mod= 0; + + if (condf->argument_count() < 2) + return NULL; + + for (i= 0; i < condf->argument_count(); i++) { + if (xtrace) + htrc("Argtype(%d)=%d\n", i, args[i]->type()); + + if (i >= 2 && !ismul) { + if (xtrace) + htrc("Unexpected arg for vop=%d\n", vop); + + continue; + } // endif i + + if ((iscol= args[i]->type() == COND::FIELD_ITEM)) { + Item_field *pField= (Item_field *)args[i]; + + // IN and BETWEEN clauses should be col VOP list + if (i && ismul) + return NULL; + + if (pField->field->table != table || + !(colp[i]= tdbp->ColDB(g, (PSZ)pField->field->field_name, 0))) + return NULL; // Column does not belong to this table + + if (xtrace) { + htrc("Field index=%d\n", pField->field->field_index); + htrc("Field name=%s\n", pField->field->field_name); + } // endif xtrace + + } else { + char buff[256]; + String *res, tmp(buff, sizeof(buff), &my_charset_bin); + Item_basic_constant *pval= (Item_basic_constant *)args[i]; + PPARM pp= (PPARM)PlugSubAlloc(g, NULL, sizeof(PARM)); + + // IN and BETWEEN clauses should be col VOP list + if (!i && (ismul)) + return NULL; + + if ((res= pval->val_str(&tmp)) == NULL) + return NULL; // To be clarified + + switch (args[i]->real_type()) { + case COND::STRING_ITEM: + pp->Type= TYPE_STRING; + pp->Value= PlugSubAlloc(g, NULL, res->length() + 1); + strncpy((char*)pp->Value, res->ptr(), res->length() + 1); + break; + case COND::INT_ITEM: + pp->Type= TYPE_INT; + pp->Value= PlugSubAlloc(g, NULL, sizeof(int)); + *((int*)pp->Value)= (int)pval->val_int(); + break; + case COND::DATE_ITEM: + pp->Type= TYPE_DATE; + pp->Value= PlugSubAlloc(g, NULL, sizeof(int)); + *((int*)pp->Value)= (int)pval->val_int_from_date(); + break; + case COND::REAL_ITEM: + pp->Type= TYPE_DOUBLE; + pp->Value= PlugSubAlloc(g, NULL, sizeof(double)); + *((double*)pp->Value)= pval->val_real(); + break; + case COND::DECIMAL_ITEM: + pp->Type= TYPE_DOUBLE; + pp->Value= PlugSubAlloc(g, NULL, sizeof(double)); + *((double*)pp->Value)= pval->val_real_from_decimal(); + break; + case COND::CACHE_ITEM: // Possible ??? + case COND::NULL_ITEM: // TODO: handle this + default: + return NULL; + } // endswitch type + + if (xtrace) + htrc("Value=%.*s\n", res->length(), res->ptr()); + + // Append the value to the argument list + if (pprec) + pprec->Next= pp; + else + pfirst= pp; + + pp->Domain= i; + pp->Next= NULL; + pprec= pp; + } // endif type + + } // endfor i + + filp= MakeFilter(g, colp, pop, pfirst, neg); + } else { + if (xtrace) + htrc("Unsupported condition\n"); + + return NULL; + } // endif's type + + return filp; +} // end of CondFilter +#endif // BLK_INDX + +/***********************************************************************/ +/* Check the WHERE condition and return a MYSQL/ODBC/WQL filter. */ +/***********************************************************************/ +PCFIL ha_connect::CheckCond(PGLOBAL g, PCFIL filp, AMT tty, Item *cond) { char *body= filp->Body; unsigned int i; bool ismul= false, x= (tty == TYPE_AM_MYX || tty == TYPE_AM_XDBC); - PPARM pfirst= NULL, pprec= NULL, pp[2]= {NULL, NULL}; OPVAL vop= OP_XX; if (!cond) return NULL; if (xtrace) - printf("Cond type=%d\n", cond->type()); + htrc("Cond type=%d\n", cond->type()); if (cond->type() == COND::COND_ITEM) { char *p1, *p2; @@ -1681,7 +1863,7 @@ PFIL ha_connect::CheckCond(PGLOBAL g, PFIL filp, AMT tty, Item *cond) return NULL; if (xtrace) - printf("Cond: Ftype=%d name=%s\n", cond_item->functype(), + htrc("Cond: Ftype=%d name=%s\n", cond_item->functype(), cond_item->func_name()); switch (cond_item->functype()) { @@ -1728,7 +1910,7 @@ PFIL ha_connect::CheckCond(PGLOBAL g, PFIL filp, AMT tty, Item *cond) Item* *args= condf->arguments(); if (xtrace) - printf("Func type=%d argnum=%d\n", condf->functype(), + htrc("Func type=%d argnum=%d\n", condf->functype(), condf->argument_count()); // neg= condf-> @@ -1742,8 +1924,10 @@ PFIL ha_connect::CheckCond(PGLOBAL g, PFIL filp, AMT tty, Item *cond) case Item_func::GE_FUNC: vop= OP_GE; break; case Item_func::GT_FUNC: vop= OP_GT; break; case Item_func::IN_FUNC: vop= OP_IN; + case Item_func::BETWEEN: + ismul= true; neg= ((Item_func_opt_neg *)condf)->negated; - case Item_func::BETWEEN: ismul= true; break; + break; default: return NULL; } // endswitch functype @@ -1757,11 +1941,11 @@ PFIL ha_connect::CheckCond(PGLOBAL g, PFIL filp, AMT tty, Item *cond) for (i= 0; i < condf->argument_count(); i++) { if (xtrace) - printf("Argtype(%d)=%d\n", i, args[i]->type()); + htrc("Argtype(%d)=%d\n", i, args[i]->type()); if (i >= 2 && !ismul) { if (xtrace) - printf("Unexpected arg for vop=%d\n", vop); + htrc("Unexpected arg for vop=%d\n", vop); continue; } // endif i @@ -1793,8 +1977,8 @@ PFIL ha_connect::CheckCond(PGLOBAL g, PFIL filp, AMT tty, Item *cond) fnm= pField->field->field_name; if (xtrace) { - printf("Field index=%d\n", pField->field->field_index); - printf("Field name=%s\n", pField->field->field_name); + htrc("Field index=%d\n", pField->field->field_index); + htrc("Field name=%s\n", pField->field->field_name); } // endif xtrace // IN and BETWEEN clauses should be col VOP list @@ -1815,7 +1999,7 @@ PFIL ha_connect::CheckCond(PGLOBAL g, PFIL filp, AMT tty, Item *cond) String *res, tmp(buff, sizeof(buff), &my_charset_bin); Item_basic_constant *pval= (Item_basic_constant *)args[i]; - switch (args[i]->type()) { + switch (args[i]->real_type()) { case COND::STRING_ITEM: case COND::INT_ITEM: case COND::REAL_ITEM: @@ -1832,7 +2016,7 @@ PFIL ha_connect::CheckCond(PGLOBAL g, PFIL filp, AMT tty, Item *cond) return NULL; // To be clarified if (xtrace) - printf("Value=%.*s\n", res->length(), res->ptr()); + htrc("Value=%.*s\n", res->length(), res->ptr()); // IN and BETWEEN clauses should be col VOP list if (!i && (x || ismul)) @@ -1877,7 +2061,7 @@ PFIL ha_connect::CheckCond(PGLOBAL g, PFIL filp, AMT tty, Item *cond) } else { if (xtrace) - printf("Unsupported condition\n"); + htrc("Unsupported condition\n"); return NULL; } // endif's type @@ -1912,32 +2096,45 @@ const COND *ha_connect::cond_push(const COND *cond) if (tdbp) { AMT tty= tdbp->GetAmType(); bool x= (tty == TYPE_AM_MYX || tty == TYPE_AM_XDBC); + bool b= (tty == TYPE_AM_WMI || tty == TYPE_AM_ODBC || + tty == TYPE_AM_TBL || tty == TYPE_AM_MYSQL || + tty == TYPE_AM_PLG || x); +#if defined(BLK_INDX) + bool go= true; +#else // !BLK_INDX) + bool go= b; +#endif // !BLK_INDX - if (tty == TYPE_AM_WMI || tty == TYPE_AM_ODBC || - tty == TYPE_AM_TBL || tty == TYPE_AM_MYSQL || - tty == TYPE_AM_PLG || x) { + if (go) { PGLOBAL& g= xp->g; - PFIL filp= (PFIL)PlugSubAlloc(g, NULL, sizeof(FILTER)); - filp->Body= (char*)PlugSubAlloc(g, NULL, (x) ? 128 : 0); - *filp->Body= 0; - filp->Op= OP_XX; - filp->Cmds= NULL; + if (b) { + PCFIL filp= (PCFIL)PlugSubAlloc(g, NULL, sizeof(CONDFIL)); - if (CheckCond(g, filp, tty, (Item *)cond)) { - if (xtrace) - printf("cond_push: %s\n", filp->Body); + filp->Body= (char*)PlugSubAlloc(g, NULL, (x) ? 128 : 0); + *filp->Body= 0; + filp->Op= OP_XX; + filp->Cmds= NULL; + + if (CheckCond(g, filp, tty, (Item *)cond)) { + if (xtrace) + htrc("cond_push: %s\n", filp->Body); + + if (!x) + PlugSubAlloc(g, NULL, strlen(filp->Body) + 1); + else + cond= NULL; // Does this work? + + tdbp->SetCondFil(filp); + } else if (x && cond) + tdbp->SetCondFil(filp); // Wrong filter - if (!x) - PlugSubAlloc(g, NULL, strlen(filp->Body) + 1); + } // endif b +#if defined(BLK_INDX) else - cond= NULL; // Does this work? - - tdbp->SetFilter(filp); - } else if (x && cond) - tdbp->SetFilter(filp); // Wrong filter - - } // endif tty + tdbp->SetFilter(CondFilter(g, (Item *)cond)); +#endif // BLK_INDX + } // endif go } // endif tdbp @@ -2019,7 +2216,7 @@ int ha_connect::open(const char *name, int mode, uint test_if_locked) DBUG_ENTER("ha_connect::open"); if (xtrace) - printf("open: name=%s mode=%d test=%u\n", name, mode, test_if_locked); + htrc("open: name=%s mode=%d test=%u\n", name, mode, test_if_locked); if (!(share= get_share())) DBUG_RETURN(1); @@ -2063,11 +2260,23 @@ int ha_connect::optimize(THD* thd, HA_CHECK_OPT* check_opt) dup->Check |= CHK_OPT; if (tdbp || (tdbp= GetTDB(g))) { +#if defined(BLK_INDX) + bool b= ((PTDBASE)tdbp)->GetDef()->Indexable(); + + if ((rc= ((PTDBASE)tdbp)->ResetTableOpt(g, true, b))) { + if (rc == RC_INFO) { + push_warning(thd, Sql_condition::WARN_LEVEL_WARN, 0, g->Message); + rc= 0; + } else + rc= HA_ERR_INTERNAL_ERROR; + + } // endif rc +#else // !BLK_INDX if (!((PTDBASE)tdbp)->GetDef()->Indexable()) { sprintf(g->Message, "optimize: Table %s is not indexable", tdbp->GetName()); my_message(ER_INDEX_REBUILD, g->Message, MYF(0)); rc= HA_ERR_UNSUPPORTED; - } else if ((rc= ((PTDBASE)tdbp)->ResetTableOpt(g, true))) { + } else if ((rc= ((PTDBASE)tdbp)->ResetTableOpt(g, false, true))) { if (rc == RC_INFO) { push_warning(thd, Sql_condition::WARN_LEVEL_WARN, 0, g->Message); rc= 0; @@ -2075,6 +2284,8 @@ int ha_connect::optimize(THD* thd, HA_CHECK_OPT* check_opt) rc= HA_ERR_INTERNAL_ERROR; } // endif's +#endif // !BLK_INDX + } else rc= HA_ERR_INTERNAL_ERROR; @@ -2179,7 +2390,7 @@ int ha_connect::write_row(uchar *buf) // Return result code from write operation if (CntWriteRow(g, tdbp)) { DBUG_PRINT("write_row", ("%s", g->Message)); - printf("write_row: %s\n", g->Message); + htrc("write_row: %s\n", g->Message); rc= HA_ERR_INTERNAL_ERROR; } // endif RC @@ -2216,7 +2427,7 @@ int ha_connect::update_row(const uchar *old_data, uchar *new_data) DBUG_ENTER("ha_connect::update_row"); if (xtrace > 1) - printf("update_row: old=%s new=%s\n", old_data, new_data); + htrc("update_row: old=%s new=%s\n", old_data, new_data); // Check values for possible change in indexed column if ((rc= CheckRecord(g, old_data, new_data))) @@ -2224,7 +2435,7 @@ int ha_connect::update_row(const uchar *old_data, uchar *new_data) if (CntUpdateRow(g, tdbp)) { DBUG_PRINT("update_row", ("%s", g->Message)); - printf("update_row CONNECT: %s\n", g->Message); + htrc("update_row CONNECT: %s\n", g->Message); rc= HA_ERR_INTERNAL_ERROR; } // endif RC @@ -2258,7 +2469,7 @@ int ha_connect::delete_row(const uchar *buf) if (CntDeleteRow(xp->g, tdbp, false)) { rc= HA_ERR_INTERNAL_ERROR; - printf("delete_row CONNECT: %s\n", xp->g->Message); + htrc("delete_row CONNECT: %s\n", xp->g->Message); } // endif DeleteRow DBUG_RETURN(rc); @@ -2275,7 +2486,7 @@ int ha_connect::index_init(uint idx, bool sorted) DBUG_ENTER("index_init"); if (xtrace) - printf("index_init: this=%p idx=%u sorted=%d\n", this, idx, sorted); + htrc("index_init: this=%p idx=%u sorted=%d\n", this, idx, sorted); if ((rc= rnd_init(0))) return rc; @@ -2291,7 +2502,7 @@ int ha_connect::index_init(uint idx, bool sorted) if (indexing <= 0) { DBUG_PRINT("index_init", ("%s", g->Message)); - printf("index_init CONNECT: %s\n", g->Message); + htrc("index_init CONNECT: %s\n", g->Message); active_index= MAX_KEY; rc= HA_ERR_INTERNAL_ERROR; } else { @@ -2307,7 +2518,7 @@ int ha_connect::index_init(uint idx, bool sorted) } // endif indexing if (xtrace) - printf("index_init: rc=%d indexing=%d active_index=%d\n", + htrc("index_init: rc=%d indexing=%d active_index=%d\n", rc, indexing, active_index); DBUG_RETURN(rc); @@ -2350,13 +2561,13 @@ int ha_connect::ReadIndexed(uchar *buf, OPVAL op, const uchar *key, uint key_len break; default: // Read error DBUG_PRINT("ReadIndexed", ("%s", xp->g->Message)); - printf("ReadIndexed: %s\n", xp->g->Message); + htrc("ReadIndexed: %s\n", xp->g->Message); rc= HA_ERR_INTERNAL_ERROR; break; } // endswitch RC if (xtrace > 1) - printf("ReadIndexed: op=%d rc=%d\n", op, rc); + htrc("ReadIndexed: op=%d rc=%d\n", op, rc); table->status= (rc == RC_OK) ? 0 : STATUS_NOT_FOUND; return rc; @@ -2399,7 +2610,7 @@ int ha_connect::index_read(uchar * buf, const uchar * key, uint key_len, } // endswitch find_flag if (xtrace > 1) - printf("%p index_read: op=%d\n", this, op); + htrc("%p index_read: op=%d\n", this, op); if (indexing > 0) rc= ReadIndexed(buf, op, key, key_len); @@ -2542,7 +2753,7 @@ int ha_connect::rnd_init(bool scan) } // endif xmod if (xtrace) - printf("rnd_init: this=%p scan=%d xmod=%d alter=%d\n", + htrc("rnd_init: this=%p scan=%d xmod=%d alter=%d\n", this, scan, xmod, alter); if (!g || !table || xmod == MODE_INSERT) @@ -2637,22 +2848,23 @@ int ha_connect::rnd_next(uchar *buf) rc= HA_ERR_RECORD_DELETED; break; default: // Read error - printf("rnd_next CONNECT: %s\n", xp->g->Message); + htrc("rnd_next CONNECT: %s\n", xp->g->Message); rc= (records()) ? HA_ERR_INTERNAL_ERROR : HA_ERR_END_OF_FILE; break; } // endswitch RC -#ifndef DBUG_OFF - if (rc || !(xp->nrd++ % 16384)) { + if (xtrace > 1 && (rc || !(xp->nrd++ % 16384))) { ulonglong tb2= my_interval_timer(); double elapsed= (double) (tb2 - xp->tb1) / 1000000000ULL; DBUG_PRINT("rnd_next", ("rc=%d nrd=%u fnd=%u nfd=%u sec=%.3lf\n", rc, (uint)xp->nrd, (uint)xp->fnd, (uint)xp->nfd, elapsed)); + htrc("rnd_next: rc=%d nrd=%u fnd=%u nfd=%u sec=%.3lf\n", + rc, (uint)xp->nrd, (uint)xp->fnd, + (uint)xp->nfd, elapsed); xp->tb1= tb2; xp->fnd= xp->nfd= 0; } // endif nrd -#endif table->status= (!rc) ? 0 : STATUS_NOT_FOUND; DBUG_RETURN(rc); @@ -2766,7 +2978,7 @@ int ha_connect::info(uint flag) DBUG_ENTER("ha_connect::info"); if (xtrace) - printf("%p In info: flag=%u valid_info=%d\n", this, flag, valid_info); + htrc("%p In info: flag=%u valid_info=%d\n", this, flag, valid_info); if (!valid_info) { // tdbp must be available to get updated info @@ -2879,7 +3091,7 @@ int ha_connect::delete_all_rows() if (!(rc= OpenTable(g))) { if (CntDeleteRow(g, tdbp, true)) { - printf("%s\n", g->Message); + htrc("%s\n", g->Message); rc= HA_ERR_INTERNAL_ERROR; } // endif @@ -2989,8 +3201,8 @@ MODE ha_connect::CheckMode(PGLOBAL g, THD *thd, { if (xtrace) { LEX_STRING *query_string= thd_query_string(thd); - printf("%p check_mode: cmdtype=%d\n", this, thd_sql_command(thd)); - printf("Cmd=%.*s\n", (int) query_string->length, query_string->str); + htrc("%p check_mode: cmdtype=%d\n", this, thd_sql_command(thd)); + htrc("Cmd=%.*s\n", (int) query_string->length, query_string->str); } // endif xtrace // Next code is temporarily replaced until sql_command is set @@ -3040,7 +3252,7 @@ MODE ha_connect::CheckMode(PGLOBAL g, THD *thd, newmode= MODE_ALTER; break; default: - printf("Unsupported sql_command=%d", thd_sql_command(thd)); + htrc("Unsupported sql_command=%d", thd_sql_command(thd)); strcpy(g->Message, "CONNECT Unsupported command"); my_message(ER_NOT_ALLOWED_COMMAND, g->Message, MYF(0)); newmode= MODE_ERROR; @@ -3085,7 +3297,7 @@ MODE ha_connect::CheckMode(PGLOBAL g, THD *thd, newmode= MODE_ALTER; break; default: - printf("Unsupported sql_command=%d", thd_sql_command(thd)); + htrc("Unsupported sql_command=%d", thd_sql_command(thd)); strcpy(g->Message, "CONNECT Unsupported command"); my_message(ER_NOT_ALLOWED_COMMAND, g->Message, MYF(0)); newmode= MODE_ERROR; @@ -3095,7 +3307,7 @@ MODE ha_connect::CheckMode(PGLOBAL g, THD *thd, } // endif's newmode if (xtrace) - printf("New mode=%d\n", newmode); + htrc("New mode=%d\n", newmode); return newmode; } // end of check_mode @@ -3170,7 +3382,7 @@ int ha_connect::external_lock(THD *thd, int lock_type) DBUG_ASSERT(thd == current_thd); if (xtrace) - printf("external_lock: this=%p thd=%p xp=%p g=%p lock_type=%d\n", + htrc("external_lock: this=%p thd=%p xp=%p g=%p lock_type=%d\n", this, thd, xp, g, lock_type); if (!g) @@ -3309,7 +3521,7 @@ int ha_connect::external_lock(THD *thd, int lock_type) if (check_privileges(thd, options, table->s->db.str)) { strcpy(g->Message, "This operation requires the FILE privilege"); - printf("%s\n", g->Message); + htrc("%s\n", g->Message); DBUG_RETURN(HA_ERR_INTERNAL_ERROR); } // endif check_privileges @@ -3343,18 +3555,18 @@ int ha_connect::external_lock(THD *thd, int lock_type) if (xtrace) { #if 0 - printf("xcheck=%d cras=%d\n", xcheck, cras); + htrc("xcheck=%d cras=%d\n", xcheck, cras); if (xcheck) - printf("oldsep=%d oldpix=%p\n", + htrc("oldsep=%d oldpix=%p\n", ((PCHK)g->Xchk)->oldsep, ((PCHK)g->Xchk)->oldpix); #endif // 0 - printf("Calling CntCheckDB db=%s cras=%d\n", GetDBName(NULL), cras); + htrc("Calling CntCheckDB db=%s cras=%d\n", GetDBName(NULL), cras); } // endif xtrace // Set or reset the good database environment if (CntCheckDB(g, this, GetDBName(NULL))) { - printf("%p external_lock: %s\n", this, g->Message); + htrc("%p external_lock: %s\n", this, g->Message); rc= HA_ERR_INTERNAL_ERROR; // This can NOT be called without open called first, but // the table can have been closed since then @@ -3375,7 +3587,7 @@ int ha_connect::external_lock(THD *thd, int lock_type) } // endif tdbp if (xtrace) - printf("external_lock: rc=%d\n", rc); + htrc("external_lock: rc=%d\n", rc); DBUG_RETURN(rc); } // end of external_lock @@ -3517,10 +3729,10 @@ int ha_connect::delete_or_rename_table(const char *name, const char *to) if (xtrace) { if (to) - printf("rename_table: this=%p thd=%p sqlcom=%d from=%s to=%s\n", + htrc("rename_table: this=%p thd=%p sqlcom=%d from=%s to=%s\n", this, thd, sqlcom, name, to); else - printf("delete_table: this=%p thd=%p sqlcom=%d name=%s\n", + htrc("delete_table: this=%p thd=%p sqlcom=%d name=%s\n", this, thd, sqlcom, name); } // endif xtrace @@ -3625,7 +3837,7 @@ ha_rows ha_connect::records_in_range(uint inx, key_range *min_key, index_init(inx, false); if (xtrace) - printf("records_in_range: inx=%d indexing=%d\n", inx, indexing); + htrc("records_in_range: inx=%d indexing=%d\n", inx, indexing); if (indexing > 0) { int nval; @@ -4626,7 +4838,7 @@ int ha_connect::create(const char *name, TABLE *table_arg, table= table_arg; // Used by called functions if (xtrace) - printf("create: this=%p thd=%p xp=%p g=%p sqlcom=%d name=%s\n", + htrc("create: this=%p thd=%p xp=%p g=%p sqlcom=%d name=%s\n", this, thd, xp, g, sqlcom, GetTableName()); // CONNECT engine specific table options: @@ -4954,7 +5166,7 @@ int ha_connect::create(const char *name, TABLE *table_arg, } // endif if (xtrace) - printf("xchk=%p createas=%d\n", g->Xchk, g->Createas); + htrc("xchk=%p createas=%d\n", g->Xchk, g->Createas); // To check whether indices have to be made or remade if (!g->Xchk) { @@ -4985,7 +5197,7 @@ int ha_connect::create(const char *name, TABLE *table_arg, cat->SetDataPath(g, table_arg->s->db.str); if ((rc= optimize(table->in_use, NULL))) { - printf("Create rc=%d %s\n", rc, g->Message); + htrc("Create rc=%d %s\n", rc, g->Message); my_message(ER_UNKNOWN_ERROR, g->Message, MYF(0)); rc= HA_ERR_INTERNAL_ERROR; } else @@ -5025,7 +5237,7 @@ int ha_connect::create(const char *name, TABLE *table_arg, g->Xchk= NULL; if (xtrace && g->Xchk) - printf("oldsep=%d newsep=%d oldpix=%p newpix=%p\n", + htrc("oldsep=%d newsep=%d oldpix=%p newpix=%p\n", xcp->oldsep, xcp->newsep, xcp->oldpix, xcp->newpix); // if (g->Xchk && @@ -5306,7 +5518,7 @@ ha_connect::check_if_supported_inplace_alter(TABLE *altered_table, tshp= NULL; if (xtrace && g->Xchk) - printf( + htrc( "oldsep=%d newsep=%d oldopn=%s newopn=%s oldpix=%p newpix=%p\n", xcp->oldsep, xcp->newsep, SVP(xcp->oldopn), SVP(xcp->newopn), diff --git a/storage/connect/ha_connect.h b/storage/connect/ha_connect.h index 1bd3991e907..a2d3c5d6801 100644 --- a/storage/connect/ha_connect.h +++ b/storage/connect/ha_connect.h @@ -334,8 +334,11 @@ public: condition stack. */ virtual const COND *cond_push(const COND *cond); -PFIL CheckCond(PGLOBAL g, PFIL filp, AMT tty, Item *cond); +PCFIL CheckCond(PGLOBAL g, PCFIL filp, AMT tty, Item *cond); const char *GetValStr(OPVAL vop, bool neg); +#if defined(BLK_INDX) +PFIL CondFilter(PGLOBAL g, Item *cond); +#endif // BLK_INDX /** Number of rows in table. It will only be called if diff --git a/storage/connect/plgdbsem.h b/storage/connect/plgdbsem.h index e251ded13df..99e9fb8fb08 100644 --- a/storage/connect/plgdbsem.h +++ b/storage/connect/plgdbsem.h @@ -29,6 +29,10 @@ enum BLKTYP {TYPE_TABLE = 50, /* Table Name/Srcdef/... Block */ // TYPE_OPVAL = 52, /* Operator value (OPVAL) */ TYPE_TDB = 53, /* Table Description Block */ TYPE_COLBLK = 54, /* Column Description Block */ +#if defined(BLK_INDX) + TYPE_FILTER = 55, /* Filter Description Block */ + TYPE_ARRAY = 63, /* General array type */ +#endif // BLK_INDX TYPE_PSZ = 64, /* Pointer to String ended by 0 */ TYPE_SQL = 65, /* Pointer to SQL block */ TYPE_XOBJECT = 69, /* Extended DB object */ @@ -144,21 +148,19 @@ enum RECFM {RECFM_NAF = -2, /* Not a file */ RECFM_PLG = 5, /* Table accessed via PLGconn */ RECFM_DBF = 6}; /* DBase formatted file */ -#if 0 enum MISC {DB_TABNO = 1, /* DB routines in Utility Table */ MAX_MULT_KEY = 10, /* Max multiple key number */ NAM_LEN = 128, /* Length of col and tab names */ ARRAY_SIZE = 50, /* Default array block size */ - MAXRES = 500, /* Default maximum result lines */ - MAXLIN = 10000, /* Default maximum data lines */ +// MAXRES = 500, /* Default maximum result lines */ +// MAXLIN = 10000, /* Default maximum data lines */ MAXBMP = 32}; /* Default XDB2 max bitmap size */ +#if 0 enum ALGMOD {AMOD_AUTO = 0, /* PLG chooses best algorithm */ AMOD_SQL = 1, /* Use SQL algorithm */ AMOD_QRY = 2}; /* Use QUERY algorithm */ -#else // !0 -#define NAM_LEN 128 -#endif // !0 +#endif // 0 enum MODE {MODE_ERROR = -1, /* Invalid mode */ MODE_ANY = 0, /* Unspecified mode */ @@ -342,7 +344,7 @@ typedef class XTAB *PTABLE; typedef class COLUMN *PCOLUMN; typedef class XOBJECT *PXOB; typedef class COLBLK *PCOL; -typedef class TBX *PTBX; +//pedef class TBX *PTBX; typedef class TDB *PTDB; typedef void *PSQL; // Not used typedef class TDBASE *PTDBASE; @@ -376,6 +378,9 @@ typedef class COLDEF *PCOLDEF; typedef class CONSTANT *PCONST; typedef class VALUE *PVAL; typedef class VALBLK *PVBLK; +#if defined(BLK_INDX) +typedef class FILTER *PFIL; +#endif // BLK_INDX typedef struct _fblock *PFBLOCK; typedef struct _mblock *PMBLOCK; @@ -431,7 +436,9 @@ typedef struct { /* User application block */ //int Maxres; /* Result Max nb of lines */ //int Maxtmp; /* Intermediate tables Maxres */ //int Maxlin; /* Query Max nb of data lines */ -//int Maxbmp; /* Maximum XDB2 bitmap size */ +#if defined(BLK_INDX) + int Maxbmp; /* Maximum XDB2 bitmap size */ +#endif // BLK_INDX int Check; /* General level of checking */ int Numlines; /* Number of lines involved */ //ALGMOD AlgChoice; /* Choice of algorithm mode */ @@ -481,6 +488,38 @@ typedef struct _tabs { PTABADR P3; } TABS; +#if defined(BLK_INDX) +/***********************************************************************/ +/* Argument of expression, function, filter etc. (Xobject) */ +/***********************************************************************/ +typedef struct _arg { /* Argument */ + PXOB To_Obj; /* To the argument object */ + PVAL Value; /* Argument value */ + bool Conv; /* TRUE if conversion is required */ + } ARGBLK, *PARG; + +typedef struct _oper { /* Operator */ + PSZ Name; /* The input/output operator name */ + OPVAL Val; /* Operator numeric value */ + int Mod; /* The modificator */ + } OPER, *POPER; + +#if 0 +/***********************************************************************/ +/* Definitions and table of Scalar Functions. */ +/***********************************************************************/ +typedef struct _sfdsc { /* Scalar function description block*/ + char Name[16]; /* Scalar function name */ + EVAL EvalType; /* Type of Init and Eval functions */ + OPVAL Op; /* Equivalent operator number */ + int R_Type; /* Result Type */ + int R_Length; /* Result Length */ + int R_Prec; /* Result Precision */ + int Numarg; /* Number of arguments */ + } SFDSC, *PSFDSC; +#endif // 0 +#endif // BLK_INDX + /***********************************************************************/ /* Following definitions are used to define table fields (columns). */ /***********************************************************************/ diff --git a/storage/connect/plgdbutl.cpp b/storage/connect/plgdbutl.cpp index c51100e0141..f52515e540b 100644 --- a/storage/connect/plgdbutl.cpp +++ b/storage/connect/plgdbutl.cpp @@ -382,7 +382,9 @@ PDBUSER PlgMakeUser(PGLOBAL g) //#endif //dbuserp->Maxres = MAXRES; //dbuserp->Maxlin = MAXLIN; -//dbuserp->Maxbmp = MAXBMP; +#if defined(BLK_INDX) + dbuserp->Maxbmp = MAXBMP; +#endif // BLK_INDX //dbuserp->AlgChoice = AMOD_AUTO; dbuserp->UseTemp = TMP_AUTO; dbuserp->Check = CHK_ALL; diff --git a/storage/connect/reldef.cpp b/storage/connect/reldef.cpp index 81c9bf9faca..a36901a5d65 100644 --- a/storage/connect/reldef.cpp +++ b/storage/connect/reldef.cpp @@ -204,6 +204,7 @@ PTABDEF OEMDEF::GetXdef(PGLOBAL g) return xdefp; } // end of GetXdef +#if 0 /***********************************************************************/ /* DeleteTableFile: Delete an OEM table file if applicable. */ /***********************************************************************/ @@ -214,6 +215,7 @@ bool OEMDEF::DeleteTableFile(PGLOBAL g) return (Pxdef) ? Pxdef->DeleteTableFile(g) : true; } // end of DeleteTableFile +#endif // 0 /***********************************************************************/ /* Define: initialize the table definition block from XDB file. */ @@ -285,8 +287,11 @@ PTDB OEMDEF::GetTable(PGLOBAL g, MODE mode) if (cmpr == 1) txfp = new(g) ZIPFAM(defp); else { +#if defined(BLK_INDX) + txfp = new(g) ZLBFAM(defp); +#else // !BLK_INDX strcpy(g->Message, "Compress 2 not supported yet"); -// txfp = new(g) ZLBFAM(defp); +#endif // !BLK_INDX return NULL; } // endelse #else // !ZIP_SUPPORT @@ -339,6 +344,7 @@ COLCRT::COLCRT(PSZ name) Offset = -1; Long = -1; Precision = -1; + Freq = -1; Key = -1; Scale = -1; Opt = -1; @@ -355,6 +361,7 @@ COLCRT::COLCRT(void) Offset = 0; Long = 0; Precision = 0; + Freq = 0; Key = 0; Scale = 0; Opt = 0; @@ -368,6 +375,16 @@ COLCRT::COLCRT(void) /***********************************************************************/ COLDEF::COLDEF(void) : COLCRT() { +#if defined(BLK_INDX) + To_Min = NULL; + To_Max = NULL; + To_Pos = NULL; + Xdb2 = FALSE; + To_Bmap = NULL; + To_Dval = NULL; + Ndv = 0; + Nbm = 0; +#endif // BLK_INDX Buf_Type = TYPE_ERROR; Clen = 0; Poff = 0; @@ -401,7 +418,7 @@ int COLDEF::Define(PGLOBAL g, void *memp, PCOLINFO cfp, int poff) Long = cfp->Length; Opt = cfp->Opt; Key = cfp->Key; -// Freq = cfp->Freq; + Freq = cfp->Freq; if (cfp->Remark && *cfp->Remark) { Desc = (PSZ)PlugSubAlloc(g, memp, strlen(cfp->Remark) + 1); diff --git a/storage/connect/reldef.h b/storage/connect/reldef.h index a877c8ee915..c54d81f30cb 100644 --- a/storage/connect/reldef.h +++ b/storage/connect/reldef.h @@ -38,7 +38,7 @@ class DllExport RELDEF : public BLOCK { // Relation definition block void SetCat(PCATLG cat) { Cat=cat; } // Methods - virtual bool DeleteTableFile(PGLOBAL g) {return true;} +//virtual bool DeleteTableFile(PGLOBAL g) {return true;} virtual bool Indexable(void) {return false;} virtual bool Define(PGLOBAL g, PCATLG cat, LPCSTR name, LPCSTR am) = 0; virtual PTDB GetTable(PGLOBAL g, MODE mode) = 0; @@ -116,7 +116,7 @@ class DllExport OEMDEF : public TABDEF { /* OEM table */ virtual AMT GetDefType(void) {return TYPE_AM_OEM;} // Methods - virtual bool DeleteTableFile(PGLOBAL g); +//virtual bool DeleteTableFile(PGLOBAL g); virtual bool DefineAM(PGLOBAL g, LPCSTR am, int poff); virtual PTDB GetTable(PGLOBAL g, MODE mode); @@ -148,6 +148,7 @@ class DllExport COLCRT : public BLOCK { /* Column description block PSZ GetDecode(void) {return Decode;} PSZ GetFmt(void) {return Fmt;} int GetOpt(void) {return Opt;} + int GetFreq(void) {return Freq;} int GetLong(void) {return Long;} int GetPrecision(void) {return Precision;} int GetOffset(void) {return Offset;} @@ -165,6 +166,7 @@ class DllExport COLCRT : public BLOCK { /* Column description block int Precision; /* Logical column length */ int Scale; /* Decimals for float/decimal values */ int Opt; /* 0:Not 1:clustered 2:sorted-asc 3:desc */ + int Freq; /* Estimated number of different values */ char DataType; /* Internal data type (C, N, F, T) */ }; // end of COLCRT @@ -188,10 +190,36 @@ class DllExport COLDEF : public COLCRT { /* Column description block int GetClen(void) {return Clen;} int GetType(void) {return Buf_Type;} int GetPoff(void) {return Poff;} +#if defined(BLK_INDX) + void *GetMin(void) {return To_Min;} + void SetMin(void *minp) {To_Min = minp;} + void *GetMax(void) {return To_Max;} + void SetMax(void *maxp) {To_Max = maxp;} + bool GetXdb2(void) {return Xdb2;} + void SetXdb2(bool b) {Xdb2 = b;} + void *GetBmap(void) {return To_Bmap;} + void SetBmap(void *bmp) {To_Bmap = bmp;} + void *GetDval(void) {return To_Dval;} + void SetDval(void *dvp) {To_Dval = dvp;} + int GetNdv(void) {return Ndv;} + void SetNdv(int ndv) {Ndv = ndv;} + int GetNbm(void) {return Nbm;} + void SetNbm(int nbm) {Nbm = nbm;} +#endif // BLK_INDX int Define(PGLOBAL g, void *memp, PCOLINFO cfp, int poff); void Define(PGLOBAL g, PCOL colp); protected: +#if defined(BLK_INDX) + void *To_Min; /* Point to array of block min values */ + void *To_Max; /* Point to array of block max values */ + int *To_Pos; /* Point to array of block positions */ + bool Xdb2; /* TRUE if to be optimized by XDB2 */ + void *To_Bmap; /* To array of block bitmap values */ + void *To_Dval; /* To array of column distinct values */ + int Ndv; /* Number of distinct values */ + int Nbm; /* Number of ULONG in bitmap (XDB2) */ +#endif // BLK_INDX int Buf_Type; /* Internal data type */ int Clen; /* Internal data size in chars (bytes) */ int Poff; /* Calculated offset for Packed tables */ diff --git a/storage/connect/tabdos.cpp b/storage/connect/tabdos.cpp index 15215dc0f01..c1da09080cb 100644 --- a/storage/connect/tabdos.cpp +++ b/storage/connect/tabdos.cpp @@ -1,11 +1,11 @@ /************* TabDos C++ Program Source Code File (.CPP) **************/ /* PROGRAM NAME: TABDOS */ /* ------------- */ -/* Version 4.8 */ +/* Version 4.9 */ /* */ /* COPYRIGHT: */ /* ---------- */ -/* (C) Copyright to the author Olivier BERTRAND 1998-2012 */ +/* (C) Copyright to the author Olivier BERTRAND 1998-2014 */ /* */ /* WHAT THIS PROGRAM DOES: */ /* ----------------------- */ @@ -57,27 +57,32 @@ #include "tabdos.h" #include "tabfix.h" #include "tabmul.h" - -#define PLGINI "plugdb.ini" // Configuration settings file - -#if defined(UNIX) -#define _fileno fileno -#define _O_RDONLY O_RDONLY -#endif +#if defined(BLK_INDX) +#include "array.h" +#include "blkfil.h" +//nclude "token.h" +//#include "scalfnc.h" +#endif // BLK_INDX /***********************************************************************/ /* DB static variables. */ /***********************************************************************/ int num_read, num_there, num_eq[2]; // Statistics -extern "C" char plgini[_MAX_PATH]; extern "C" int trace; +#if defined(BLK_INDX) /***********************************************************************/ -/* Min and Max blocks contains zero ended fields (blank = false). */ -/* No conversion of block values (check = true). */ +/* Size of optimize file header. */ +/***********************************************************************/ +#define NZ 4 + +/***********************************************************************/ +/* Min and Max blocks contains zero ended fields (blank = FALSE). */ +/* No conversion of block values (check = TRUE). */ /***********************************************************************/ PVBLK AllocValBlock(PGLOBAL, void *, int, int, int len = 0, int prec = 0, - bool check = true, bool blank = false, bool un = false); + bool check = TRUE, bool blank = FALSE, bool un = FALSE); +#endif // BLK_INDX /* --------------------------- Class DOSDEF -------------------------- */ @@ -96,7 +101,11 @@ DOSDEF::DOSDEF(void) Huge = false; Accept = false; Eof = false; +#if defined(BLK_INDX) To_Pos = NULL; + Optimized = 0; + AllocBlks = 0; +#endif // BLK_INDX Compressed = 0; Lrecl = 0; AvgLen = 0; @@ -109,6 +118,98 @@ DOSDEF::DOSDEF(void) //Mtime = 0; } // end of DOSDEF constructor +/***********************************************************************/ +/* DefineAM: define specific AM block values from XDB file. */ +/***********************************************************************/ +bool DOSDEF::DefineAM(PGLOBAL g, LPCSTR am, int poff) + { + char buf[8]; + bool map = (am && (*am == 'M' || *am == 'm')); + LPCSTR dfm = (am && (*am == 'F' || *am == 'f')) ? "F" + : (am && (*am == 'B' || *am == 'b')) ? "B" + : (am && !stricmp(am, "DBF")) ? "D" : "V"; + + Desc = Fn = Cat->GetStringCatInfo(g, "Filename", NULL); + Ofn = Cat->GetStringCatInfo(g, "Optname", Fn); + Cat->GetCharCatInfo("Recfm", (PSZ)dfm, buf, sizeof(buf)); + Recfm = (toupper(*buf) == 'F') ? RECFM_FIX : + (toupper(*buf) == 'B') ? RECFM_BIN : + (toupper(*buf) == 'D') ? RECFM_DBF : RECFM_VAR; + Lrecl = Cat->GetIntCatInfo("Lrecl", 0); + + if (Recfm != RECFM_DBF) + Compressed = Cat->GetIntCatInfo("Compressed", 0); + + Mapped = Cat->GetBoolCatInfo("Mapped", map); + Block = Cat->GetIntCatInfo("Blocks", 0); + Last = Cat->GetIntCatInfo("Last", 0); + Ending = Cat->GetIntCatInfo("Ending", CRLF); + + if (Recfm == RECFM_FIX || Recfm == RECFM_BIN) { + Huge = Cat->GetBoolCatInfo("Huge", Cat->GetDefHuge()); + Padded = Cat->GetBoolCatInfo("Padded", false); + Blksize = Cat->GetIntCatInfo("Blksize", 0); + Eof = (Cat->GetIntCatInfo("EOF", 0) != 0); + } else if (Recfm == RECFM_DBF) { + Maxerr = Cat->GetIntCatInfo("Maxerr", 0); + Accept = (Cat->GetIntCatInfo("Accept", 0) != 0); + ReadMode = Cat->GetIntCatInfo("Readmode", 0); + } else // (Recfm == RECFM_VAR) + AvgLen = Cat->GetIntCatInfo("Avglen", 0); + + // Ignore wrong Index definitions for catalog commands + return (Cat->GetIndexInfo(g, this) /*&& !Cat->GetCatFnc()*/); + } // end of DefineAM + +#if 0 +#if defined(BLK_INDX) +/***********************************************************************/ +/* DeleteTableFile: Delete DOS/UNIX table files using platform API. */ +/* If the table file is protected (declared as read/only) we still */ +/* erase the the eventual optimize and index files but return TRUE. */ +/***********************************************************************/ +bool DOSDEF::DeleteTableFile(PGLOBAL g) + { + char filename[_MAX_PATH]; + bool rc, irc, orc = FALSE; + PIXDEF pxd; + PCOLDEF cdp = NULL; + + /*********************************************************************/ + /* Check for potential optimization. These tests are done */ + /* because Optimized is set to 1 only after the first use of an */ + /* optimized table and can be 0 if it has not been used yet. */ + /*********************************************************************/ + if (!Optimized) + for (cdp = To_Cols; cdp; cdp = cdp->GetNext()) + if (cdp->GetOpt()) + break; + + if (IsOptimized() || cdp || (Recfm == RECFM_VAR && Elemt > 1 && Block)) + if (!GetOptFileName(g, filename)) +#if defined(WIN32) + orc = !DeleteFile(filename); +#else // UNIX + orc = remove(filename); +#endif // WIN32 + + // Now delete the table file itself if not protected + if (!IsReadOnly()) { + rc = Erase(filename); + } else + rc = true; + + // Delete eventual index file(s) + if ((pxd = To_Indx)) { + To_Indx = NULL; // So file can be erase + irc = DeleteIndexFile(g, pxd); + } else + irc = false; + + return rc || orc || irc; // Return TRUE if error + } // end of DeleteTableFile + +#else // !BLK_INDX /***********************************************************************/ /* DeleteTableFile: Delete DOS/UNIX table files using platform API. */ /* If the table file is protected (declared as read/only) we still */ @@ -127,6 +228,7 @@ bool DOSDEF::DeleteTableFile(PGLOBAL g) return rc; // Return true if error } // end of DeleteTableFile +#endif // !BLK_INDX /***********************************************************************/ /* Erase: This was made a separate routine because a strange thing */ @@ -147,6 +249,66 @@ bool DOSDEF::Erase(char *filename) return rc; // Return true if error } // end of Erase +#endif // 0 + +#if defined(BLK_INDX) +/***********************************************************************/ +/* Get the full path/name of the optization file. */ +/***********************************************************************/ +bool DOSDEF::GetOptFileName(PGLOBAL g, char *filename) + { + char *ftype; + + switch (Recfm) { + case RECFM_VAR: ftype = ".dop"; break; + case RECFM_FIX: ftype = ".fop"; break; + case RECFM_BIN: ftype = ".bop"; break; + case RECFM_VCT: ftype = ".vop"; break; + case RECFM_DBF: ftype = ".dbp"; break; + default: + sprintf(g->Message, MSG(INVALID_FTYPE), Recfm); + return TRUE; + } // endswitch Ftype + + PlugSetPath(filename, Ofn, GetPath()); + strcat(PlugRemoveType(filename, filename), ftype); + return FALSE; + } // end of GetOptFileName + +/***********************************************************************/ +/* After an optimize error occured, remove all set optimize values. */ +/***********************************************************************/ +void DOSDEF::RemoveOptValues(PGLOBAL g) + { + char filename[_MAX_PATH]; + PCOLDEF cdp; + + // Delete settings of optimized columns + for (cdp = To_Cols; cdp; cdp = cdp->GetNext()) + if (cdp->GetOpt()) { + cdp->SetMin(NULL); + cdp->SetMax(NULL); + cdp->SetNdv(0); + cdp->SetNbm(0); + cdp->SetDval(NULL); + cdp->SetBmap(NULL); + } // endif Opt + + // Delete block position setting for not fixed tables + To_Pos = NULL; + AllocBlks = 0; + + // Delete any eventually ill formed non matching optimization file + if (!GetOptFileName(g, filename)) +#if defined(WIN32) + DeleteFile(filename); +#else // UNIX + remove(filename); +#endif // WIN32 + + Optimized = 0; + } // end of RemoveOptValues +#endif // BLK_INDX /***********************************************************************/ /* DeleteIndexFile: Delete DOS/UNIX index file(s) using platform API. */ @@ -221,49 +383,6 @@ bool DOSDEF::DeleteIndexFile(PGLOBAL g, PIXDEF pxdf) return rc; // Return true if error } // end of DeleteIndexFile -/***********************************************************************/ -/* DefineAM: define specific AM block values from XDB file. */ -/***********************************************************************/ -bool DOSDEF::DefineAM(PGLOBAL g, LPCSTR am, int poff) - { - char buf[8]; - bool map = (am && (*am == 'M' || *am == 'm')); - LPCSTR dfm = (am && (*am == 'F' || *am == 'f')) ? "F" - : (am && (*am == 'B' || *am == 'b')) ? "B" - : (am && !stricmp(am, "DBF")) ? "D" : "V"; - - Desc = Fn = Cat->GetStringCatInfo(g, "Filename", NULL); - Ofn = Cat->GetStringCatInfo(g, "Optname", Fn); - Cat->GetCharCatInfo("Recfm", (PSZ)dfm, buf, sizeof(buf)); - Recfm = (toupper(*buf) == 'F') ? RECFM_FIX : - (toupper(*buf) == 'B') ? RECFM_BIN : - (toupper(*buf) == 'D') ? RECFM_DBF : RECFM_VAR; - Lrecl = Cat->GetIntCatInfo("Lrecl", 0); - - if (Recfm != RECFM_DBF) - Compressed = Cat->GetIntCatInfo("Compressed", 0); - - Mapped = Cat->GetBoolCatInfo("Mapped", map); - Block = Cat->GetIntCatInfo("Blocks", 0); - Last = Cat->GetIntCatInfo("Last", 0); - Ending = Cat->GetIntCatInfo("Ending", CRLF); - - if (Recfm == RECFM_FIX || Recfm == RECFM_BIN) { - Huge = Cat->GetBoolCatInfo("Huge", Cat->GetDefHuge()); - Padded = Cat->GetBoolCatInfo("Padded", false); - Blksize = Cat->GetIntCatInfo("Blksize", 0); - Eof = (Cat->GetIntCatInfo("EOF", 0) != 0); - } else if (Recfm == RECFM_DBF) { - Maxerr = Cat->GetIntCatInfo("Maxerr", 0); - Accept = (Cat->GetIntCatInfo("Accept", 0) != 0); - ReadMode = Cat->GetIntCatInfo("Readmode", 0); - } else // (Recfm == RECFM_VAR) - AvgLen = Cat->GetIntCatInfo("Avglen", 0); - - // Ignore wrong Index definitions for catalog commands - return (Cat->GetIndexInfo(g, this) /*&& !Cat->GetCatFnc()*/); - } // end of DefineAM - /***********************************************************************/ /* InvalidateIndex: mark all indexes as invalid. */ /***********************************************************************/ @@ -311,28 +430,35 @@ PTDB DOSDEF::GetTable(PGLOBAL g, MODE mode) txfp = new(g) BGXFAM(this); else if (map) txfp = new(g) MPXFAM(this); + else if (Compressed) { #if defined(ZIP_SUPPORT) - else if (Compressed) txfp = new(g) ZIXFAM(this); -#endif // ZIP_SUPPORT - else +#else // !ZIP_SUPPORT + sprintf(g->Message, MSG(NO_FEAT_SUPPORT), "ZIP"); + return NULL; +#endif // !ZIP_SUPPORT + } else txfp = new(g) FIXFAM(this); tdbp = new(g) TDBFIX(this, txfp); } else { -#if defined(ZIP_SUPPORT) if (Compressed) { +#if defined(ZIP_SUPPORT) if (Compressed == 1) txfp = new(g) ZIPFAM(this); else { +#if defined(BLK_INDX) + txfp = new(g) ZLBFAM(this); +#else // !BLK_INDX strcpy(g->Message, "Compress 2 not supported yet"); -// txfp = new(g) ZLBFAM(defp); return NULL; +#endif // !BLK_INDX } // endelse - - } else -#endif // ZIP_SUPPORT - if (map) +#else // !ZIP_SUPPORT + sprintf(g->Message, MSG(NO_FEAT_SUPPORT), "ZIP"); + return NULL; +#endif // !ZIP_SUPPORT + } else if (map) txfp = new(g) MAPFAM(this); else txfp = new(g) DOSFAM(this); @@ -344,6 +470,38 @@ PTDB DOSDEF::GetTable(PGLOBAL g, MODE mode) if (Multiple) tdbp = new(g) TDBMUL(tdbp); +#if defined(BLK_INDX) + else + /*******************************************************************/ + /* For block tables, get eventually saved optimization values. */ + /*******************************************************************/ + if (tdbp->GetBlockValues(g)) { + PushWarning(g, tdbp); +// return NULL; // causes a crash when deleting index + } else if (Recfm == RECFM_VAR || Compressed > 1) { + if (IsOptimized()) { + if (map) { + txfp = new(g) MBKFAM(this); + } else if (Compressed) { +#if defined(ZIP_SUPPORT) + if (Compressed == 1) + txfp = new(g) ZBKFAM(this); + else { + txfp->SetBlkPos(To_Pos); + ((PZLBFAM)txfp)->SetOptimized(To_Pos != NULL); + } // endelse +#else + sprintf(g->Message, MSG(NO_FEAT_SUPPORT), "ZIP"); + return NULL; +#endif + } else + txfp = new(g) BLKFAM(this); + + ((PTDBDOS)tdbp)->SetTxfp(txfp); + } // endif Optimized + + } // endif Recfm +#endif // BLK_INDX return tdbp; } // end of GetTable @@ -364,6 +522,13 @@ TDBDOS::TDBDOS(PDOSDEF tdp, PTXF txfp) : TDBASE(tdp) Ftype = tdp->Recfm; To_Line = NULL; Cardinal = -1; +#if defined(BLK_INDX) +//To_BlkIdx = NULL; + To_BlkFil = NULL; + SavFil = NULL; +//Xeval = 0; + Beval = 0; +#endif // BLK_INDX } // end of TDBDOS standard constructor TDBDOS::TDBDOS(PGLOBAL g, PTDBDOS tdbp) : TDBASE(tdbp) @@ -374,6 +539,13 @@ TDBDOS::TDBDOS(PGLOBAL g, PTDBDOS tdbp) : TDBASE(tdbp) Ftype = tdbp->Ftype; To_Line = tdbp->To_Line; Cardinal = tdbp->Cardinal; +#if defined(BLK_INDX) +//To_BlkIdx = tdbp->To_BlkIdx; + To_BlkFil = tdbp->To_BlkFil; + SavFil = tdbp->SavFil; +//Xeval = tdbp->Xeval; + Beval = tdbp->Beval; +#endif // BLK_INDX } // end of TDBDOS copy constructor // Method @@ -416,14 +588,49 @@ void TDBDOS::PrintAM(FILE *f, char *m) /***********************************************************************/ /* Remake the indexes after the table was modified. */ /***********************************************************************/ -int TDBDOS::ResetTableOpt(PGLOBAL g, bool dox) +int TDBDOS::ResetTableOpt(PGLOBAL g, bool dop, bool dox) { - int rc = RC_OK; + int prc = RC_OK, rc = RC_OK; MaxSize = -1; // Size must be recalculated Cardinal = -1; // as well as Cardinality - if (dox) { +#if defined(BLK_INDX) + PTXF xp = Txfp; + + To_Filter = NULL; // Disable filtering +//To_BlkIdx = NULL; // and index filtering + To_BlkFil = NULL; // and block filtering + + if (dop) { + Columns = NULL; // Not used anymore + + if (Txfp->Blocked) { + // MakeBlockValues must be executed in non blocked mode + // except for ZLIB access method. + if (Txfp->GetAmType() == TYPE_AM_MAP) { + Txfp = new(g) MAPFAM((PDOSDEF)To_Def); +#if defined(ZIP_SUPPORT) + } else if (Txfp->GetAmType() == TYPE_AM_ZIP) { + Txfp = new(g) ZIPFAM((PDOSDEF)To_Def); + } else if (Txfp->GetAmType() == TYPE_AM_ZLIB) { + Txfp->Reset(); + ((PZLBFAM)Txfp)->SetOptimized(FALSE); +#endif // ZIP_SUPPORT + } else // (Txfp->GetAmType() == TYPE_AM_BLK) + Txfp = new(g) DOSFAM((PDOSDEF)To_Def); + + Txfp->SetTdbp(this); + } else + Txfp->Reset(); + + Use = USE_READY; // So the table can be reopened + Mode = MODE_ANY; // Just to be clean + rc = MakeBlockValues(g); // Redo optimization + } // endif dop +#endif // BLK_INDX + + if (dox && (rc == RC_OK || rc == RC_INFO)) { // Remake eventual indexes if (Mode != MODE_UPDATE) To_SetCols = NULL; // Only used on Update @@ -432,6 +639,7 @@ int TDBDOS::ResetTableOpt(PGLOBAL g, bool dox) Txfp->Reset(); // New start Use = USE_READY; // So the table can be reopened Mode = MODE_READ; // New mode + prc = rc; if (!(PlgGetUser(g)->Check & CHK_OPT)) { // After the table was modified the indexes @@ -441,11 +649,1020 @@ int TDBDOS::ResetTableOpt(PGLOBAL g, bool dox) // ... or we should remake them. rc = MakeIndex(g, NULL, false); +#if defined(BLK_INDX) + rc = (rc == RC_INFO) ? prc : rc; +#endif // BLK_INDX } // endif dox return rc; } // end of ResetTableOpt +#if defined(BLK_INDX) +/***********************************************************************/ +/* Calculate the block sizes so block I/O can be used and also the */ +/* Min/Max values for clustered/sorted table columns. */ +/***********************************************************************/ +int TDBDOS::MakeBlockValues(PGLOBAL g) + { + int i, lg, nrec, rc; + int curnum, curblk, block, last, savndv, savnbm; + void *savmin, *savmax; + bool blocked, xdb2 = FALSE; +//POOLHEADER save; + PCOLDEF cdp; + PDOSDEF defp = (PDOSDEF)To_Def; + PDOSCOL colp = NULL; + PDBUSER dup = PlgGetUser(g); + PCATLG cat = defp->GetCat(); +//void *memp = cat->GetDescp(); + + if ((nrec = defp->GetElemt()) < 2) { + strcpy(g->Message, MSG(TABLE_NOT_OPT)); + return RC_INFO; // Not to be optimized + } else if (GetMaxSize(g) == 0 || !(dup->Check & CHK_OPT)) { + // Suppress the opt file firstly if the table is void, + // secondly when it was modified with OPTIMIZATION unchecked + // because it is no more valid. + defp->RemoveOptValues(g); // Erase opt file + return RC_OK; // void table + } else if (MaxSize < 0) + return RC_FX; + + defp->SetOptimized(0); + + // Estimate the number of needed blocks + block = (int)((MaxSize + (int)nrec - 1) / (int)nrec); + + // We have to use local variables because Txfp->CurBlk is set + // to Rows+1 by unblocked variable length table access methods. + curblk = -1; + curnum = nrec - 1; + last = 0; + Txfp->Block = block; // This is useful mainly for + Txfp->CurBlk = curblk; // blocked tables (ZLBFAM), for + Txfp->CurNum = curnum; // others it is just to be clean. + + /*********************************************************************/ + /* Allocate the array of block starting positions. */ + /*********************************************************************/ +//if (memp) +// save = *(PPOOLHEADER)memp; + + Txfp->BlkPos = (int*)PlugSubAlloc(g, NULL, (block + 1) * sizeof(int)); + + /*********************************************************************/ + /* Allocate the blocks for clustered columns. */ + /*********************************************************************/ + blocked = Txfp->Blocked; // Save + Txfp->Blocked = TRUE; // So column block can be allocated + + for (cdp = defp->GetCols(), i = 1; cdp; cdp = cdp->GetNext(), i++) + if (cdp->GetOpt()) { + lg = cdp->GetClen(); + + if (cdp->GetFreq() && cdp->GetFreq() <= dup->Maxbmp) { + cdp->SetXdb2(TRUE); + savndv = cdp->GetNdv(); + cdp->SetNdv(0); // Reset Dval number of values + xdb2 = TRUE; + savmax = cdp->GetDval(); + cdp->SetDval(PlugSubAlloc(g, NULL, cdp->GetFreq() * lg)); + savnbm = cdp->GetNbm(); + cdp->SetNbm(0); // Prevent Bmap allocation +// savmin = cdp->GetBmap(); +// cdp->SetBmap(PlugSubAlloc(g, NULL, block * sizeof(int))); + + if (trace) + htrc("Dval(%p) Bmap(%p) col(%d) %s Block=%d lg=%d\n", + cdp->GetDval(), cdp->GetBmap(), i, cdp->GetName(), block, lg); + + // colp will be initialized with proper Dval VALBLK + colp = (PDOSCOL)MakeCol(g, cdp, colp, i); + colp->InitValue(g); // Allocate column value buffer + cdp->SetNbm(savnbm); +// cdp->SetBmap(savmin); // Can be reused if the new size + cdp->SetDval(savmax); // is not greater than this one. + cdp->SetNdv(savndv); + } else { + cdp->SetXdb2(FALSE); // Maxbmp may have been reset + savmin = cdp->GetMin(); + savmax = cdp->GetMax(); + cdp->SetMin(PlugSubAlloc(g, NULL, block * lg)); + cdp->SetMax(PlugSubAlloc(g, NULL, block * lg)); + + if (trace) + htrc("min(%p) max(%p) col(%d) %s Block=%d lg=%d\n", + cdp->GetMin(), cdp->GetMax(), i, cdp->GetName(), block, lg); + + // colp will be initialized with proper opt VALBLK's + colp = (PDOSCOL)MakeCol(g, cdp, colp, i); + colp->InitValue(g); // Allocate column value buffer + cdp->SetMin(savmin); // Can be reused if the number + cdp->SetMax(savmax); // of blocks does not change. + } // endif Freq + + } // endif Clustered + +//if (!colp) +// return RC_INFO; + + Txfp->Blocked = blocked; + + /*********************************************************************/ + /* Now do calculate the optimization values. */ + /*********************************************************************/ + Mode = MODE_READ; + + if (OpenDB(g)) + return RC_FX; + + if (xdb2) { + /*********************************************************************/ + /* Retrieve the distinct values of XDB2 columns. */ + /*********************************************************************/ + if (GetDistinctColumnValues(g, nrec)) + return RC_FX; + + OpenDB(g); // Rewind the table file + } // endif xdb2 + +#if defined(PROG_INFO) + /*********************************************************************/ + /* Initialize progress information */ + /*********************************************************************/ + char *p = (char *)PlugSubAlloc(g, NULL, 24 + strlen(Name)); + + dup->Step = strcat(strcpy(p, MSG(OPTIMIZING)), Name); + dup->ProgMax = GetProgMax(g); + dup->ProgCur = 0; +#endif // SOCKET_MODE || THREAD + + /*********************************************************************/ + /* Make block starting pos and min/max values of cluster columns. */ + /*********************************************************************/ + while ((rc = ReadDB(g)) == RC_OK) { + if (blocked) { + // A blocked FAM class handles CurNum and CurBlk (ZLBFAM) + if (!Txfp->CurNum) + Txfp->BlkPos[Txfp->CurBlk] = Txfp->GetPos(); + + } else { + if (++curnum >= nrec) { + if (++curblk >= block) { + strcpy(g->Message, MSG(BAD_BLK_ESTIM)); + goto err; + } else + curnum = 0; + + // Get block starting position + Txfp->BlkPos[curblk] = Txfp->GetPos(); + } // endif CurNum + + last = curnum + 1; // curnum is zero based + Txfp->CurBlk = curblk; // Used in COLDOS::SetMinMax + Txfp->CurNum = curnum; // Used in COLDOS::SetMinMax + } // endif blocked + + /*******************************************************************/ + /* Now calculate the min and max values for the cluster columns. */ + /*******************************************************************/ + for (colp = (PDOSCOL)Columns; colp; colp = (PDOSCOL)colp->GetNext()) + if (colp->Clustered == 2) { + if (colp->SetBitMap(g)) + goto err; + + } else + if (colp->SetMinMax(g)) + goto err; // Currently: column is not sorted + +#if defined(SOCKET_MODE) || defined(THREAD) +#if defined(SOCKET_MODE) + if (SendProgress(dup)) { + strcpy(g->Message, MSG(OPT_CANCELLED)); + goto err; + } else +#elif defined(THREAD) + if (!dup->Step) { + strcpy(g->Message, MSG(OPT_CANCELLED)); + goto err; + } else +#endif // THREAD + dup->ProgCur = GetProgCur(); +#endif // SOCKET_MODE || THREAD + + } // endwhile + + if (rc == RC_EF) { + Txfp->Nrec = nrec; + + if (blocked) { + Txfp->Block = Txfp->CurBlk + 1; + Txfp->Last = (Txfp->CurNum) ? Txfp->CurNum + 1 : nrec; + } else { + Txfp->Block = curblk + 1; + Txfp->Last = last; + } // endif blocked + + // This is needed to be able to calculate the last block size + Txfp->BlkPos[Txfp->Block] = Txfp->GetNextPos(); + } else + goto err; + + /*********************************************************************/ + /* Save the optimization values for this table. */ + /*********************************************************************/ + if (!SaveBlockValues(g)) { + PCATLG cat = PlgGetCatalog(g); + + defp->Block = Txfp->Block; + defp->Last = Txfp->Last; + CloseDB(g); + cat->SetIntCatInfo("Blocks", Txfp->Block); + cat->SetIntCatInfo("Last", Txfp->Last); + return RC_OK; + } // endif SaveBlockValues + + err: + // Restore Desc memory suballocation +//if (memp) +// *(PPOOLHEADER)memp = save; + + defp->RemoveOptValues(g); + CloseDB(g); + return RC_FX; + } // end of MakeBlockValues + +/***********************************************************************/ +/* Save the block and Min/Max values for this table. */ +/* The problem here is to avoid name duplication, because more than */ +/* one data file can have the same name (but different types) and/or */ +/* the same data file can be used with different block sizes. This is */ +/* why we use Ofn that defaults to the file name but can be set to a */ +/* different name if necessary. */ +/***********************************************************************/ +bool TDBDOS::SaveBlockValues(PGLOBAL g) + { + char filename[_MAX_PATH]; + int lg, n[NZ + 2]; + size_t nbk, ndv, nbm, block = Txfp->Block; + bool rc = FALSE; + FILE *opfile; + PDOSCOL colp; + PDOSDEF defp = (PDOSDEF)To_Def; + + if (defp->GetOptFileName(g, filename)) + return TRUE; + + if (!(opfile = fopen(filename, "wb"))) { + sprintf(g->Message, MSG(OPEN_MODE_ERROR), + "wb", (int)errno, filename); + strcat(strcat(g->Message, ": "), strerror(errno)); + + if (trace) + htrc("%s\n", g->Message); + + return TRUE; + } // endif opfile + + if (Ftype == RECFM_VAR || defp->Compressed == 2) { + /*******************************************************************/ + /* Write block starting positions into the opt file. */ + /*******************************************************************/ + block++; + lg = sizeof(int); + n[0] = Txfp->Last; n[1] = lg; n[2] = Txfp->Nrec; n[3] = Txfp->Block; + + if (fwrite(n, sizeof(int), NZ, opfile) != NZ) { + sprintf(g->Message, MSG(OPT_HEAD_WR_ERR), strerror(errno)); + rc = TRUE; + } // endif size + + if (fwrite(Txfp->BlkPos, lg, block, opfile) != block) { + sprintf(g->Message, MSG(OPTBLK_WR_ERR), strerror(errno)); + rc = TRUE; + } // endif size + + block--; // = Txfp->Block; + } // endif Ftype + + /*********************************************************************/ + /* Write the Min/Max values into the opt file. */ + /*********************************************************************/ + for (colp = (PDOSCOL)Columns; colp; colp = (PDOSCOL)colp->Next) { + lg = colp->Value->GetClen(); + + // Now start the writing process + if (colp->Clustered == 2) { + // New XDB2 block optimization. Will be recognized when reading + // because the column index is negated. + ndv = colp->Ndv; nbm = colp->Nbm; + nbk = nbm * block; + n[0] = -colp->Index; n[1] = lg; n[2] = Txfp->Nrec; n[3] = block; + n[4] = ndv; n[5] = nbm; + + if (fwrite(n, sizeof(int), NZ + 2, opfile) != NZ + 2) { + sprintf(g->Message, MSG(OPT_HEAD_WR_ERR), strerror(errno)); + rc = TRUE; + } // endif size + + if (fwrite(colp->Dval->GetValPointer(), lg, ndv, opfile) != ndv) { + sprintf(g->Message, MSG(OPT_DVAL_WR_ERR), strerror(errno)); + rc = TRUE; + } // endif size + + if (fwrite(colp->Bmap->GetValPointer(), sizeof(int), nbk, opfile) != nbk) { + sprintf(g->Message, MSG(OPT_BMAP_WR_ERR), strerror(errno)); + rc = TRUE; + } // endif size + + } else { + n[0] = colp->Index; n[1] = lg; n[2] = Txfp->Nrec; n[3] = block; + + if (fwrite(n, sizeof(int), NZ, opfile) != NZ) { + sprintf(g->Message, MSG(OPT_HEAD_WR_ERR), strerror(errno)); + rc = TRUE; + } // endif size + + if (fwrite(colp->Min->GetValPointer(), lg, block, opfile) != block) { + sprintf(g->Message, MSG(OPT_MIN_WR_ERR), strerror(errno)); + rc = TRUE; + } // endif size + + if (fwrite(colp->Max->GetValPointer(), lg, block, opfile) != block) { + sprintf(g->Message, MSG(OPT_MAX_WR_ERR), strerror(errno)); + rc = TRUE; + } // endif size + + } // endif Clustered + + } // endfor colp + + fclose(opfile); + return rc; + } // end of SaveBlockValues + +/***********************************************************************/ +/* Read the Min/Max values for this table. */ +/* The problem here is to avoid name duplication, because more than */ +/* one data file can have the same name (but different types) and/or */ +/* the same data file can be used with different block sizes. This is */ +/* why we use Ofn that defaults to the file name but can be set to a */ +/* different name if necessary. */ +/***********************************************************************/ +bool TDBDOS::GetBlockValues(PGLOBAL g) + { + char filename[_MAX_PATH]; + int i, lg, n[NZ]; + int nrec, block, last = 0, allocblk = 0; + int len; + bool newblk = FALSE; + size_t ndv, nbm, nbk, blk; + FILE *opfile; + PCOLDEF cdp; + PDOSDEF defp = (PDOSDEF)To_Def; + PCATLG cat = defp->GetCat(); + + if (defp->Optimized) + return FALSE; // Already done or to be redone + + if (Ftype == RECFM_VAR || defp->Compressed == 2) { + /*******************************************************************/ + /* Variable length file that can be read by block. */ + /*******************************************************************/ + block = defp->GetBlock(); + nrec = (defp->GetElemt()) ? defp->GetElemt() : 1; + last = defp->GetLast(); + + if (nrec > 1 && block) + len = (int)((block - 1) * nrec + last); + else + len = 0; + + if (!len) { + if (nrec > 1) { + // The table can be declared optimized if it is void. + // This is useful to handle Insert in optimized mode. + char filename[_MAX_PATH]; + int h; + int flen = -1; + + PlugSetPath(filename, defp->Fn, GetPath()); + h = open(filename, _O_RDONLY); + flen = (h == -1 && errno == ENOENT) ? 0 : _filelength(h); + defp->SetOptimized((flen) ? 0 : 1); + + if (h != -1) + close(h); + + } // endif nrec + + return FALSE; // Opt file does not exist yet + } else if (len < 0) + return TRUE; // Table error + + cdp = defp->GetCols(); + i = 1; + } else { + /*******************************************************************/ + /* Fixed length file. Opt file exists only for clustered columns. */ + /*******************************************************************/ + // Check for existence of clustered columns + for (cdp = defp->GetCols(), i = 1; cdp; cdp = cdp->GetNext(), i++) + if (cdp->GetOpt()) + break; + + if (!cdp) + return FALSE; // No optimization needed + + if ((len = Cardinality(g)) < 0) + return TRUE; // Table error + else if (!len) + return FALSE; // File does not exist yet + + block = Txfp->Block; // Was set in Cardinality + nrec = Txfp->Nrec; + } // endif Ftype + + if (defp->GetOptFileName(g, filename)) + return TRUE; + + if (!(opfile = fopen(filename, "rb"))) + return FALSE; // No saved values + +//if (memp) { +// save = *(PPOOLHEADER)memp; +// allocblk = defp->GetAllocBlks(); +// } // endif memp + + if (block > allocblk) + newblk = TRUE; // Current allocation is too small + + if (Ftype == RECFM_VAR || defp->Compressed == 2) { + /*******************************************************************/ + /* Read block starting positions from the opt file. */ + /*******************************************************************/ + blk = block + 1; + lg = sizeof(int); + + if (fread(n, sizeof(int), NZ, opfile) != NZ) { + sprintf(g->Message, MSG(OPT_HEAD_RD_ERR), strerror(errno)); + goto err; + } // endif size + + if (n[0] != last || n[1] != lg || n[2] != nrec || n[3] != block) { + sprintf(g->Message, MSG(OPT_NOT_MATCH), filename); + goto err; + } // endif + + if (newblk) + defp->To_Pos = (int*)PlugSubAlloc(g, NULL, blk * lg); + + if (fread(defp->To_Pos, lg, blk, opfile) != blk) { + sprintf(g->Message, MSG(OPTBLK_RD_ERR), strerror(errno)); + goto err; + } // endif size + + } // endif Ftype + + /*********************************************************************/ + /* Read the Min/Max values from the opt file. */ + /*********************************************************************/ + for (; cdp; cdp = cdp->GetNext(), i++) + if (cdp->GetOpt()) { + lg = cdp->GetClen(); + blk = block; + + // Now start the reading process. + if (fread(n, sizeof(int), NZ, opfile) != NZ) { + sprintf(g->Message, MSG(OPT_HEAD_RD_ERR), strerror(errno)); + goto err; + } // endif size + + if (n[0] == -i) { + // Read the XDB2 opt values from the opt file + if (n[1] != lg || n[2] != nrec || n[3] != block) { + sprintf(g->Message, MSG(OPT_NOT_MATCH), filename); + goto err; + } // endif + + if (fread(n, sizeof(int), 2, opfile) != 2) { + sprintf(g->Message, MSG(OPT_HEAD_RD_ERR), strerror(errno)); + goto err; + } // endif fread + + ndv = n[0]; nbm = n[1]; nbk = nbm * blk; + + if (cdp->GetNdv() < (int)ndv || !cdp->GetDval()) + cdp->SetDval(PlugSubAlloc(g, NULL, ndv * lg)); + + cdp->SetNdv((int)ndv); + + if (fread(cdp->GetDval(), lg, ndv, opfile) != ndv) { + sprintf(g->Message, MSG(OPT_DVAL_RD_ERR), strerror(errno)); + goto err; + } // endif size + + if (newblk || cdp->GetNbm() < (int)nbm || !cdp->GetBmap()) + cdp->SetBmap(PlugSubAlloc(g, NULL, nbk * sizeof(int))); + + cdp->SetNbm((int)nbm); + + if (fread(cdp->GetBmap(), sizeof(int), nbk, opfile) != nbk) { + sprintf(g->Message, MSG(OPT_BMAP_RD_ERR), strerror(errno)); + goto err; + } // endif size + + cdp->SetXdb2(TRUE); + } else { + // Read the Min/Max values from the opt file + if (n[0] != i || n[1] != lg || n[2] != nrec || n[3] != block) { + sprintf(g->Message, MSG(OPT_NOT_MATCH), filename); + goto err; + } // endif + + if (newblk || !cdp->GetMin()) + cdp->SetMin(PlugSubAlloc(g, NULL, blk * lg)); + + if (fread(cdp->GetMin(), lg, blk, opfile) != blk) { + sprintf(g->Message, MSG(OPT_MIN_RD_ERR), strerror(errno)); + goto err; + } // endif size + + if (newblk || !cdp->GetMax()) + cdp->SetMax(PlugSubAlloc(g, NULL, blk * lg)); + + if (fread(cdp->GetMax(), lg, blk, opfile) != blk) { + sprintf(g->Message, MSG(OPT_MAX_RD_ERR), strerror(errno)); + goto err; + } // endif size + + cdp->SetXdb2(FALSE); + } // endif n[0] (XDB2) + + } // endif Clustered + + defp->SetBlock(block); + + if (newblk) + defp->SetAllocBlks(block); + + defp->SetOptimized(1); + fclose(opfile); + MaxSize = -1; // Can be refined later + return FALSE; + + err: + defp->RemoveOptValues(g); + fclose(opfile); +//return TRUE; + // Ignore error if not in mode CHK_OPT + return (PlgGetUser(g)->Check & CHK_OPT) != 0; + } // end of GetBlockValues + +/***********************************************************************/ +/* This fonction is used while making XDB2 block optimization. */ +/* It constructs for each elligible columns, the sorted list of the */ +/* distinct values existing in the column. This function uses an */ +/* algorithm that permit to get several sets of distinct values by */ +/* reading the table only once, which cannot be done using a standard */ +/* SQL query. */ +/***********************************************************************/ +bool TDBDOS::GetDistinctColumnValues(PGLOBAL g, int nrec) + { + char *p; + int rc, blk, n = 0; + PDOSCOL colp; + PDBUSER dup = PlgGetUser(g); + + /*********************************************************************/ + /* Initialize progress information */ + /*********************************************************************/ + p = (char *)PlugSubAlloc(g, NULL, 48 + strlen(Name)); + dup->Step = strcat(strcpy(p, MSG(GET_DIST_VALS)), Name); + dup->ProgMax = GetProgMax(g); + dup->ProgCur = 0; + + while ((rc = ReadDB(g)) == RC_OK) { + for (colp = (PDOSCOL)Columns; colp; colp = (PDOSCOL)colp->Next) + if (colp->Clustered == 2) + if (colp->AddDistinctValue(g)) + return TRUE; // Too many distinct values + +#if defined(SOCKET_MODE) + if (SendProgress(dup)) { + strcpy(g->Message, MSG(OPT_CANCELLED)); + return TRUE; + } else +#elif defined(THREAD) + if (!dup->Step) { + strcpy(g->Message, MSG(OPT_CANCELLED)); + return TRUE; + } else +#endif // THREAD + dup->ProgCur = GetProgCur(); + + n++; + } // endwhile + + if (rc != RC_EF) + return TRUE; + + // Reset the number of table blocks +//nrec = ((PDOSDEF)To_Def)->GetElemt(); (or default value) + blk = (n + nrec - 1) / nrec; + Txfp->Block = blk; // Useful mainly for ZLBFAM ??? + + // Set Nbm, Bmap for XDB2 columns + for (colp = (PDOSCOL)Columns; colp; colp = (PDOSCOL)colp->Next) + if (colp->Clustered == 2) { +// colp->Cdp->SetNdv(colp->Ndv); + colp->Nbm = (colp->Ndv + MAXBMP - 1) / MAXBMP; + colp->Bmap = AllocValBlock(g, NULL, TYPE_INT, colp->Nbm * blk); + } // endif Clustered + + return FALSE; + } // end of GetDistinctColumnValues + +/***********************************************************************/ +/* Analyze the filter and construct the Block Evaluation Filter. */ +/* This is possible when a filter contains predicates implying a */ +/* column marked as "clustered" or "sorted" matched to a constant */ +/* argument. It is then possible by comparison against the smallest */ +/* and largest column values in each block to determine whether the */ +/* filter condition will be always true or always false for the block.*/ +/***********************************************************************/ +PBF TDBDOS::InitBlockFilter(PGLOBAL g, PFIL filp) + { + bool blk = Txfp->Blocked; + + if (To_BlkFil) + return To_BlkFil; // Already done + else if (!filp) + return NULL; + else if (blk) { + if (Txfp->GetAmType() == TYPE_AM_DBF) + /*****************************************************************/ + /* If RowID is used in this query, block optimization cannot be */ + /* used because currently the file must be read sequentially. */ + /*****************************************************************/ + for (PCOL cp = Columns; cp; cp = cp->GetNext()) + if (cp->GetAmType() == TYPE_AM_ROWID && !((RIDBLK*)cp)->GetRnm()) + return NULL; + + } // endif blk + + int i, op = filp->GetOpc(), opm = filp->GetOpm(), n = 0; + bool cnv[2]; + PCOL colp; + PXOB arg[2] = {NULL,NULL}; + PBF *fp = NULL, bfp = NULL; + + switch (op) { + case OP_EQ: + case OP_NE: + case OP_GT: + case OP_GE: + case OP_LT: + case OP_LE: + if (! opm) { + for (i = 0; i < 2; i++) { + arg[i] = filp->Arg(i); + cnv[i] = filp->Conv(i); + } // endfor i + + bfp = CheckBlockFilari(g, arg, op, cnv); + break; + } // endif !opm + + // if opm, pass thru + case OP_IN: + if (filp->GetArgType(0) == TYPE_COLBLK && + filp->GetArgType(1) == TYPE_ARRAY) { + arg[0] = filp->Arg(0); + arg[1] = filp->Arg(1); + colp = (PCOL)arg[0]; + + if (colp->GetTo_Tdb() == this) { + // Block evaluation is possible for... + if (colp->GetAmType() == TYPE_AM_ROWID) { + // Special column ROWID and constant array, but + // currently we don't know how to retrieve a RowID + // from a DBF table that is not sequentially read. +// if (Txfp->GetAmType() != TYPE_AM_DBF || +// ((RIDBLK*)arg[0])->GetRnm()) + bfp = new(g) BLKSPCIN(g, this, op, opm, arg, Txfp->Nrec); + + } else if (blk && Txfp->Nrec > 1 && colp->IsClustered()) + // Clustered column and constant array + if (colp->GetClustered() == 2) + bfp = new(g) BLKFILIN2(g, this, op, opm, arg); + else + bfp = new(g) BLKFILIN(g, this, op, opm, arg); + + } // endif this + +#if 0 + } else if (filp->GetArgType(0) == TYPE_SCALF && + filp->GetArgType(1) == TYPE_ARRAY) { + arg[0] = filp->Arg(0); + arg[1] = filp->Arg(1); + + if (((PSCALF)arg[0])->GetOp() == OP_ROW && + arg[1]->GetResultType() == TYPE_LIST) { + PARRAY par = (PARRAY)arg[1]; + LSTVAL *vlp = (LSTVAL*)par->GetValue(); + + ((SFROW*)arg[0])->GetParms(n); + + if (n != vlp->GetN()) + return NULL; + else + n = par->GetNval(); + + arg[1] = new(g) CONSTANT(vlp); + fp = (PBF*)PlugSubAlloc(g, NULL, n * sizeof(PBF)); + cnv[0] = cnv[1] = FALSE; + + if (op == OP_IN) + op = OP_EQ; + + for (i = 0; i < n; i++) { + par->GetNthValue(vlp, i); + + if (!(fp[i] = CheckBlockFilari(g, arg, op, cnv))) + return NULL; + + } // endfor i + + bfp = new(g) BLKFILLOG(this, (opm == 2 ? OP_AND : OP_OR), fp, n); + } // endif ROW +#endif // 0 + + } // endif Type + + break; + case OP_AND: + case OP_OR: + fp = (PBF*)PlugSubAlloc(g, NULL, 2 * sizeof(PBF)); + fp[0] = InitBlockFilter(g, (PFIL)(filp->Arg(0))); + fp[1] = InitBlockFilter(g, (PFIL)(filp->Arg(1))); + + if (fp[0] || fp[1]) + bfp = new(g) BLKFILLOG(this, op, fp, 2); + + break; + case OP_NOT: + fp = (PBF*)PlugSubAlloc(g, NULL, sizeof(PBF)); + + if ((*fp = InitBlockFilter(g, (PFIL)(filp->Arg(0))))) + bfp = new(g) BLKFILLOG(this, op, fp, 1); + + break; + case OP_LIKE: + default: + break; + } // endswitch op + + return bfp; + } // end of InitBlockFilter + +/***********************************************************************/ +/* Analyze the passed arguments and construct the Block Filter. */ +/***********************************************************************/ +PBF TDBDOS::CheckBlockFilari(PGLOBAL g, PXOB *arg, int op, bool *cnv) + { +//int i, n1, n2, ctype = TYPE_ERROR, n = 0, type[2] = {0,0}; +//bool conv = FALSE, xdb2 = FALSE, ok = FALSE, b[2]; +//PXOB *xarg1, *xarg2 = NULL, xp[2]; + int i, ctype = TYPE_ERROR, n = 0, type[2] = {0,0}; + bool conv = FALSE, xdb2 = FALSE, ok = FALSE; + PXOB *xarg2 = NULL, xp[2]; + PCOL colp; +//LSTVAL *vlp = NULL; +//SFROW *sfr[2]; + PBF *fp = NULL, bfp = NULL; + + for (i = 0; i < 2; i++) { + switch (arg[i]->GetType()) { + case TYPE_CONST: + type[i] = 1; + ctype = arg[i]->GetResultType(); + break; + case TYPE_COLBLK: + conv = cnv[i]; + colp = (PCOL)arg[i]; + + if (colp->GetTo_Tdb() == this) { + if (colp->GetAmType() == TYPE_AM_ROWID) { + // Currently we don't know how to retrieve a RowID + // from a DBF table that is not sequentially read. +// if (Txfp->GetAmType() != TYPE_AM_DBF || +// ((RIDBLK*)arg[i])->GetRnm()) + type[i] = 5; + + } else if (Txfp->Blocked && Txfp->Nrec > 1 && + colp->IsClustered()) { + type[i] = 2; + xdb2 = colp->GetClustered() == 2; + } // endif Clustered + + } else if (colp->GetColUse(U_CORREL)) { + // This is a column pointing to the outer query of a + // correlated subquery, it has a constant value during + // each execution of the subquery. + type[i] = 1; + ctype = arg[i]->GetResultType(); + } // endif this + + break; +// case TYPE_SCALF: +// if (((PSCALF)arg[i])->GetOp() == OP_ROW) { +// sfr[i] = (SFROW*)arg[i]; +// type[i] = 7; +// } // endif Op + +// break; + default: + break; + } // endswitch ArgType + + if (!type[i]) + break; + + n += type[i]; + } // endfor i + + if (n == 3 || n == 6) { + if (conv) { + // The constant has not the good type and will not match + // the block min/max values. What we can do here is either + // abort with an error message or simply not do the block + // optimization (as column values can be converted when + // evaluating the filter.) Currently we prefer aborting + // because the user may count on the performance enhancing + // and silently not doing it is probably worse than just + // telling him to fix his query. + sprintf(g->Message, "Block opt: %s", MSG(VALTYPE_NOMATCH)); + longjmp(g->jumper[g->jump_level], 99); + } // endif Conv + + if (type[0] == 1) { + // Make it always as Column-op-Value + *xp = arg[0]; + arg[0] = arg[1]; + arg[1] = *xp; + + switch (op) { + case OP_GT: op = OP_LT; break; + case OP_GE: op = OP_LE; break; + case OP_LT: op = OP_GT; break; + case OP_LE: op = OP_GE; break; + } // endswitch op + + } // endif + +#if defined(_DEBUG) +// assert(arg[0]->GetResultType() == ctype); +#endif + + if (n == 3) { + if (xdb2) { + if (((PDOSCOL)arg[0])->GetNbm() == 1) + bfp = new(g) BLKFILAR2(g, this, op, arg); + else // Multiple bitmap made of several ULONG's + bfp = new(g) BLKFILMR2(g, this, op, arg); + } else + bfp = new(g) BLKFILARI(g, this, op, arg); + + } else // n = 6 + bfp = new(g) BLKSPCARI(this, op, arg, Txfp->Nrec); + +#if 0 + } else if (n == 8 || n == 14) { + if (n == 8 && ctype != TYPE_LIST) { + // Should never happen + strcpy(g->Message, "Block opt: bad constant"); + longjmp(g->jumper[g->jump_level], 99); + } // endif Conv + + if (type[0] == 1) { + // Make it always as Column-op-Value + sfr[0] = sfr[1]; + arg[1] = arg[0]; + + switch (op) { + case OP_GT: op = OP_LT; break; + case OP_GE: op = OP_LE; break; + case OP_LT: op = OP_GT; break; + case OP_LE: op = OP_GE; break; + } // endswitch op + + } // endif + + xarg1 = sfr[0]->GetParms(n1); + + if (n == 8) { + vlp = (LSTVAL*)arg[1]->GetValue(); + n2 = vlp->GetN(); + xp[1] = new(g) CONSTANT((PVAL)NULL); + } else + xarg2 = sfr[1]->GetParms(n2); + + if (n1 != n2) + return NULL; // Should we flag an error ? + + fp = (PBF*)PlugSubAlloc(g, NULL, n1 * sizeof(PBF)); + + for (i = 0; i < n1; i++) { + xp[0] = xarg1[i]; + + if (n == 8) + ((CONSTANT*)xp[1])->SetValue(vlp->GetSubVal(i)); + else + xp[1] = xarg2[i]; + + b[0] = b[1] = (xp[0]->GetResultType() != xp[1]->GetResultType()); + ok |= ((fp[i] = CheckBlockFilari(g, xp, op, b)) != NULL); + } // endfor i + + if (ok) + bfp = new(g) BLKFILLOG(this, OP_AND, fp, n1); +#endif // 0 + + } // endif n + + return bfp; + } // end of CheckBlockFilari + +/***********************************************************************/ +/* ResetBlkFil: reset the block filter and restore filtering. */ +/***********************************************************************/ +void TDBDOS::ResetBlockFilter(PGLOBAL g) + { + if (!To_BlkFil) + return; + + To_BlkFil->Reset(g); + + if (SavFil && !To_Filter) { + // Restore filter if it was disabled by optimization + To_Filter = SavFil; + SavFil = NULL; + } // endif + + Beval = 0; + } // end of ResetBlockFilter + +/***********************************************************************/ +/* Block optimization: evaluate the block index filter against */ +/* the min and max values of this block and return: */ +/* RC_OK: if some records in the block can meet filter criteria. */ +/* RC_NF: if no record in the block can meet filter criteria. */ +/* RC_EF: if no record in the remaining file can meet filter criteria.*/ +/* In addition, temporarily supress filtering if all the records in */ +/* the block meet filter criteria. */ +/***********************************************************************/ +int TDBDOS::TestBlock(PGLOBAL g) + { + int rc = RC_OK; + + if (To_BlkFil && Beval != 2) { + // Check for block filtering evaluation + if (Beval == 1) { + // Filter was removed for last block, restore it + To_Filter = SavFil; + SavFil = NULL; + } // endif Beval + + // Check for valid records in new block + switch (Beval = To_BlkFil->BlockEval(g)) { + case -2: // No more valid values in file + rc = RC_EF; + break; + case -1: // No valid values in block + rc = RC_NF; + break; + case 1: // All block values are valid + case 2: // All subsequent file values are Ok + // Before suppressing the filter for the block(s) it is + // necessary to reset the filtered columns to NOT_READ + // so their new values are retrieved by the SELECT list. + if (To_Filter) // Can be NULL when externally called (XDB) + To_Filter->Reset(); + + SavFil = To_Filter; + To_Filter = NULL; // So remove filter + } // endswitch Beval + + if (trace) + htrc("BF Eval Beval=%d\n", Beval); + + } // endif To_BlkFil + + return rc; + } // end of TestBlock +#endif // BLK_INDX + /***********************************************************************/ /* Check whether we have to create/update permanent indexes. */ /***********************************************************************/ @@ -730,11 +1947,21 @@ bool TDBDOS::OpenDB(PGLOBAL g) /*******************************************************************/ /* Table already open, just replace it at its beginning. */ /*******************************************************************/ - Txfp->Rewind(); // see comment in Work.log + if (!To_Kindex) { + Txfp->Rewind(); // see comment in Work.log - if (SkipHeader(g)) - return true; + if (SkipHeader(g)) + return TRUE; + } else + /*****************************************************************/ + /* Table is to be accessed through a sorted index table. */ + /*****************************************************************/ + To_Kindex->Reset(); + +#if defined(BLK_INDX) + ResetBlockFilter(g); +#endif // BLK_INDX return false; } // endif use @@ -770,6 +1997,13 @@ bool TDBDOS::OpenDB(PGLOBAL g) Use = USE_OPEN; // Do it now in case we are recursively called +#if defined(BLK_INDX) + /*********************************************************************/ + /* Allocate the block filter tree if evaluation is possible. */ + /*********************************************************************/ + To_BlkFil = InitBlockFilter(g, To_Filter); +#endif // BLK_INDX + /*********************************************************************/ /* Allocate the line buffer plus a null character. */ /*********************************************************************/ @@ -928,6 +2162,42 @@ DOSCOL::DOSCOL(PGLOBAL g, PCOLDEF cdp, PTDB tp, PCOL cp, int i, PSZ am) Long = cdp->GetLong(); To_Val = NULL; +#if defined(BLK_INDX) + Clustered = 0; + Sorted = 0; + Ndv = 0; // Currently used only for XDB2 + Nbm = 0; // Currently used only for XDB2 + Min = NULL; + Max = NULL; + Bmap = NULL; + Dval = NULL; + Buf = NULL; + + if (txfp->Blocked && Opt && (cdp->GetMin() || cdp->GetDval())) { + int nblk = txfp->GetBlock(); + + Clustered = (cdp->GetXdb2()) ? 2 : 1; + Sorted = (cdp->GetOpt() > 1) ? 1 : 0; // Currently ascending only + + if (Clustered == 1) { + Min = AllocValBlock(g, cdp->GetMin(), Buf_Type, nblk, Long, prec); + Max = AllocValBlock(g, cdp->GetMax(), Buf_Type, nblk, Long, prec); + } else { // Clustered == 2 + // Ndv is the number of distinct values in Dval. Ndv and Nbm + // may be 0 when optimizing because Ndval is not filled yet, + // but the size of the passed Dval memory block is Ok. + Ndv = cdp->GetNdv(); + Dval = AllocValBlock(g, cdp->GetDval(), Buf_Type, Ndv, Long, prec); + + // Bmap cannot be allocated when optimizing, we must know Nbm first + if ((Nbm = cdp->GetNbm())) + Bmap = AllocValBlock(g, cdp->GetBmap(), TYPE_INT, Nbm * nblk); + + } // endif Clustered + + } // endif Opt +#endif // BLK_INDX + OldVal = NULL; // Currently used only in MinMax Ldz = false; Nod = false; @@ -969,8 +2239,19 @@ DOSCOL::DOSCOL(DOSCOL *col1, PTDB tdbp) : COLBLK(col1, tdbp) Dcm = col1->Dcm; OldVal = col1->OldVal; Buf = col1->Buf; +#if defined(BLK_INDX) + Clustered = col1->Clustered; + Sorted = col1->Sorted; + Min = col1->Min; + Max = col1->Max; + Bmap = col1->Bmap; + Dval = col1->Dval; + Ndv = col1->Ndv; + Nbm = col1->Nbm; +#endif // BLK_INDX } // end of DOSCOL copy constructor +#if defined(BLK_INDX) /***********************************************************************/ /* VarSize: This function tells UpdateDB whether or not the block */ /* optimization file must be redone if this column is updated, even */ @@ -992,6 +2273,7 @@ bool DOSCOL::VarSize(void) return false; } // end VarSize +#endif // BLK_INDX /***********************************************************************/ /* SetBuffer: prepare a column block for write operation. */ @@ -1151,6 +2433,13 @@ void DOSCOL::WriteColumn(PGLOBAL g) htrc("Lrecl=%d deplac=%d int=%d\n", tdbp->Lrecl, Deplac, Long); field = Long; + len = (signed)strlen(tdbp->To_Line); + + if (tdbp->GetAmType() == TYPE_AM_DOS && len > tdbp->Lrecl) { + sprintf(g->Message, "Line size %d is bigger than lrecl %d", + len, tdbp->Lrecl); + longjmp(g->jumper[g->jump_level], 32); + } // endif if (tdbp->Ftype == RECFM_VAR && tdbp->Mode == MODE_UPDATE) { len = (signed)strlen(tdbp->To_Line); @@ -1269,6 +2558,146 @@ void DOSCOL::WriteColumn(PGLOBAL g) } // end of WriteColumn +#if defined(BLK_INDX) +/***********************************************************************/ +/* SetMinMax: Calculate minimum and maximum values for one block. */ +/* Note: TYPE_STRING is stored and processed with zero ended strings */ +/* to be matching the way the FILTER Eval function processes them. */ +/***********************************************************************/ +bool DOSCOL::SetMinMax(PGLOBAL g) + { + PTDBDOS tp = (PTDBDOS)To_Tdb; + + ReadColumn(g); // Extract column value from current line + + if (CheckSorted(g)) + return TRUE; + + if (!tp->Txfp->CurNum) { + Min->SetValue(Value, tp->Txfp->CurBlk); + Max->SetValue(Value, tp->Txfp->CurBlk); + } else { + Min->SetMin(Value, tp->Txfp->CurBlk); + Max->SetMax(Value, tp->Txfp->CurBlk); + } // endif CurNum + + return FALSE; + } // end of SetMinMax + +/***********************************************************************/ +/* SetBitMap: Calculate the bit map of existing values in one block. */ +/* Note: TYPE_STRING is processed with zero ended strings */ +/* to be matching the way the FILTER Eval function processes them. */ +/***********************************************************************/ +bool DOSCOL::SetBitMap(PGLOBAL g) + { + int i, m, n; + PULONG bmp; + PTDBDOS tp = (PTDBDOS)To_Tdb; + PDBUSER dup = PlgGetUser(g); + + n = tp->Txfp->CurNum; + bmp = (PULONG)Bmap->GetValPtr(Nbm * tp->Txfp->CurBlk); + + // Extract column value from current line + ReadColumn(g); + + if (CheckSorted(g)) + return TRUE; + + if (!n) // New block + for (m = 0; m < Nbm; m++) + bmp[m] = 0; // Reset the new bit map + + if ((i = Dval->Find(Value)) < 0) { + char buf[32]; + + sprintf(g->Message, MSG(DVAL_NOTIN_LIST), + Value->GetCharString(buf), Name); + return TRUE; + } else if (i >= dup->Maxbmp) { + sprintf(g->Message, MSG(OPT_LOGIC_ERR), i); + return TRUE; + } else { + m = i / MAXBMP; +#if defined(_DEBUG) + assert (m < Nbm); +#endif // _DEBUG + bmp[m] |= (1 << (i % MAXBMP)); + } // endif's i + + return FALSE; + } // end of SetBitMap + +/***********************************************************************/ +/* Checks whether a column declared as sorted is sorted indeed. */ +/***********************************************************************/ +bool DOSCOL::CheckSorted(PGLOBAL g) + { + if (Sorted) + if (OldVal) { + // Verify whether this column is sorted all right + if (OldVal->CompareValue(Value) > 0) { + // Column is no more in ascending order + sprintf(g->Message, MSG(COL_NOT_SORTED), Name, To_Tdb->GetName()); + Sorted = FALSE; + return TRUE; + } else + OldVal->SetValue_pval(Value); + + } else + OldVal = AllocateValue(g, Value); + + return FALSE; + } // end of CheckSorted + +/***********************************************************************/ +/* AddDistinctValue: Check whether this value already exist in the */ +/* list and if not add it to the distinct values list. */ +/***********************************************************************/ +bool DOSCOL::AddDistinctValue(PGLOBAL g) + { + bool found = FALSE; + int i, m, n; + + ReadColumn(g); // Extract column value from current line + + // Perhaps a better algorithm can be used when Ndv gets bigger + // Here we cannot use Find because we must get the index of where + // to insert a new value if it is not found in the array. + for (n = 0; n < Ndv; n++) { + m = Dval->CompVal(Value, n); + + if (m > 0) + continue; + else if (!m) + found = TRUE; // Already there + + break; + } // endfor n + + if (!found) { + // Check whether we have room for an additional value + if (Ndv == Freq) { + // Too many values because of wrong Freq setting + sprintf(g->Message, MSG(BAD_FREQ_SET), Name); + return TRUE; + } // endif Ndv + + // New value, add it to the list before the nth value + Dval->SetNval(Ndv + 1); + + for (i = Ndv; i > n; i--) + Dval->Move(i - 1, i); + + Dval->SetValue(Value, n); + Ndv++; + } // endif found + + return FALSE; + } // end of AddDistinctValue +#endif // BLK_INDX + /***********************************************************************/ /* Make file output of a Dos column descriptor block. */ /***********************************************************************/ diff --git a/storage/connect/tabdos.h b/storage/connect/tabdos.h index 5f67ffad92f..52bb1450c29 100644 --- a/storage/connect/tabdos.h +++ b/storage/connect/tabdos.h @@ -1,7 +1,7 @@ /*************** TabDos H Declares Source Code File (.H) ***************/ -/* Name: TABDOS.H Version 3.2 */ +/* Name: TABDOS.H Version 3.3 */ /* */ -/* (C) Copyright to the author Olivier BERTRAND 1999-2012 */ +/* (C) Copyright to the author Olivier BERTRAND 1999-2014 */ /* */ /* This file contains the DOS classes declares. */ /***********************************************************************/ @@ -12,9 +12,16 @@ #include "xtable.h" // Table base class declares #include "colblk.h" // Column base class declares #include "xindex.h" +#if defined(BLK_INDX) +#include "filter.h" +#endif // BLK_INDX //pedef struct _tabdesc *PTABD; // For friend setting typedef class TXTFAM *PTXF; +#if defined(BLK_INDX) +typedef class BLOCKFILTER *PBF; +typedef class BLOCKINDEX *PBX; +#endif // BLK_INDX /***********************************************************************/ /* DOS table. */ @@ -34,6 +41,7 @@ class DllExport DOSDEF : public TABDEF { /* Logical table description */ virtual const char *GetType(void) {return "DOS";} virtual PIXDEF GetIndx(void) {return To_Indx;} virtual void SetIndx(PIXDEF xdp) {To_Indx = xdp;} + virtual bool IsHuge(void) {return Huge;} PSZ GetFn(void) {return Fn;} PSZ GetOfn(void) {return Ofn;} void SetBlock(int block) {Block = block;} @@ -46,19 +54,28 @@ class DllExport DOSDEF : public TABDEF { /* Logical table description */ bool GetEof(void) {return Eof;} int GetBlksize(void) {return Blksize;} int GetEnding(void) {return Ending;} +#if defined(BLK_INDX) + bool IsOptimized(void) {return (Optimized == 1);} + void SetOptimized(int opt) {Optimized = opt;} + void SetAllocBlks(int blks) {AllocBlks = blks;} + int GetAllocBlks(void) {return AllocBlks;} int *GetTo_Pos(void) {return To_Pos;} - virtual bool IsHuge(void) {return Huge;} +#endif // BLK_INDX // Methods - virtual bool DeleteTableFile(PGLOBAL g); +//virtual bool DeleteTableFile(PGLOBAL g); virtual bool Indexable(void) {return Compressed != 1;} virtual bool DeleteIndexFile(PGLOBAL g, PIXDEF pxdf); virtual bool DefineAM(PGLOBAL g, LPCSTR am, int poff); virtual PTDB GetTable(PGLOBAL g, MODE mode); bool InvalidateIndex(PGLOBAL g); +#if defined(BLK_INDX) + bool GetOptFileName(PGLOBAL g, char *filename); + void RemoveOptValues(PGLOBAL g); +#endif // BLK_INDX protected: - virtual bool Erase(char *filename); +//virtual bool Erase(char *filename); // Members PSZ Fn; /* Path/Name of corresponding file */ @@ -70,7 +87,11 @@ class DllExport DOSDEF : public TABDEF { /* Logical table description */ bool Huge; /* true for files larger than 2GB */ bool Accept; /* true if wrong lines are accepted (DBF)*/ bool Eof; /* true if an EOF (0xA) character exists */ - int *To_Pos; /* To array of block starting positions */ +#if defined(BLK_INDX) + int *To_Pos; /* To array of block starting positions */ + int Optimized; /* 0: No, 1:Yes, 2:Redo optimization */ + int AllocBlks; /* Number of suballocated opt blocks */ +#endif // BLK_INDX int Compressed; /* 0: No, 1: gz, 2:zlib compressed file */ int Lrecl; /* Size of biggest record */ int AvgLen; /* Average size of records */ @@ -129,13 +150,15 @@ class DllExport TDBDOS : public TDBASE { virtual bool IsUsingTemp(PGLOBAL g); //virtual bool NeedIndexing(PGLOBAL g); virtual void ResetSize(void) {MaxSize = Cardinal = -1;} - virtual int ResetTableOpt(PGLOBAL g, bool dox); -//virtual int MakeBlockValues(PGLOBAL g); -//virtual bool SaveBlockValues(PGLOBAL g); -//virtual bool GetBlockValues(PGLOBAL g); -//virtual PBF InitBlockFilter(PGLOBAL g, PFIL filp); + virtual int ResetTableOpt(PGLOBAL g, bool dop, bool dox); +#if defined(BLK_INDX) + virtual int MakeBlockValues(PGLOBAL g); + virtual bool SaveBlockValues(PGLOBAL g); + virtual bool GetBlockValues(PGLOBAL g); + virtual PBF InitBlockFilter(PGLOBAL g, PFIL filp); //virtual PBX InitBlockIndex(PGLOBAL g); -//virtual int TestBlock(PGLOBAL g); + virtual int TestBlock(PGLOBAL g); +#endif // BLK_INDX virtual void PrintAM(FILE *f, char *m); // Database routines @@ -162,25 +185,31 @@ class DllExport TDBDOS : public TDBASE { virtual int EstimatedLength(PGLOBAL g); // Optimization routines -// void ResetBlockFilter(PGLOBAL g); - int MakeIndex(PGLOBAL g, PIXDEF pxdf, bool add); -// bool GetDistinctColumnValues(PGLOBAL g, int nrec); + int MakeIndex(PGLOBAL g, PIXDEF pxdf, bool add); +#if defined(BLK_INDX) + void ResetBlockFilter(PGLOBAL g); + bool GetDistinctColumnValues(PGLOBAL g, int nrec); protected: -// PBF CheckBlockFilari(PGLOBAL g, PXOB *arg, int op, bool *cnv); + PBF CheckBlockFilari(PGLOBAL g, PXOB *arg, int op, bool *cnv); +#endif // BLK_INDX // Members PTXF Txfp; // To the File access method class +#if defined(BLK_INDX) //PBX To_BlkIdx; // To index test block -//PBF To_BlkFil; // To evaluation block filter -//PFIL SavFil; // Saved hidden filter + PBF To_BlkFil; // To evaluation block filter + PFIL SavFil; // Saved hidden filter +#endif // BLK_INDX char *To_Line; // Points to current processed line int Cardinal; // Table Cardinality RECFM Ftype; // File type: 0-var 1-fixed 2-binary (VCT) int Lrecl; // Logical Record Length int AvgLen; // Logical Record Average Length +#if defined(BLK_INDX) //int Xeval; // BlockTest return value -//int Beval; // BlockEval return value + int Beval; // BlockEval return value +#endif // BLK_INDX }; // end of class TDBDOS /***********************************************************************/ @@ -198,50 +227,60 @@ class DllExport DOSCOL : public COLBLK { // Implementation virtual int GetAmType(void) {return TYPE_AM_DOS;} -//virtual int GetClustered(void) {return Clustered;} -//virtual int IsClustered(void) {return (Clustered && -// ((PDOSDEF)(((PTDBDOS)To_Tdb)->To_Def))->IsOptimized());} -//virtual int IsSorted(void) {return Sorted;} virtual void SetTo_Val(PVAL valp) {To_Val = valp;} -//virtual PVBLK GetMin(void) {return Min;} -//virtual PVBLK GetMax(void) {return Max;} -//virtual int GetNdv(void) {return Ndv;} -//virtual int GetNbm(void) {return Nbm;} -//virtual PVBLK GetBmap(void) {return Bmap;} -//virtual PVBLK GetDval(void) {return Dval;} +#if defined(BLK_INDX) + virtual int GetClustered(void) {return Clustered;} + virtual int IsClustered(void) {return (Clustered && + ((PDOSDEF)(((PTDBDOS)To_Tdb)->To_Def))->IsOptimized());} + virtual int IsSorted(void) {return Sorted;} + virtual PVBLK GetMin(void) {return Min;} + virtual PVBLK GetMax(void) {return Max;} + virtual int GetNdv(void) {return Ndv;} + virtual int GetNbm(void) {return Nbm;} + virtual PVBLK GetBmap(void) {return Bmap;} + virtual PVBLK GetDval(void) {return Dval;} +#endif // BLK_INDX // Methods +#if defined(BLK_INDX) virtual bool VarSize(void); +#endif // BLK_INDX virtual bool SetBuffer(PGLOBAL g, PVAL value, bool ok, bool check); virtual void ReadColumn(PGLOBAL g); virtual void WriteColumn(PGLOBAL g); virtual void Print(PGLOBAL g, FILE *, uint); protected: -//virtual bool SetMinMax(PGLOBAL g); -//virtual bool SetBitMap(PGLOBAL g); -// bool CheckSorted(PGLOBAL g); -// bool AddDistinctValue(PGLOBAL g); +#if defined(BLK_INDX) + virtual bool SetMinMax(PGLOBAL g); + virtual bool SetBitMap(PGLOBAL g); + bool CheckSorted(PGLOBAL g); + bool AddDistinctValue(PGLOBAL g); +#endif // BLK_INDX // Default constructor not to be used DOSCOL(void) {} // Members -//PVBLK Min; // Array of block min values -//PVBLK Max; // Array of block max values -//PVBLK Bmap; // Array of block bitmap values -//PVBLK Dval; // Array of column distinct values +#if defined(BLK_INDX) + PVBLK Min; // Array of block min values + PVBLK Max; // Array of block max values + PVBLK Bmap; // Array of block bitmap values + PVBLK Dval; // Array of column distinct values +#endif // BLK_INDX PVAL To_Val; // To value used for Update/Insert PVAL OldVal; // The previous value of the object. char *Buf; // Buffer used in write operations bool Ldz; // True if field contains leading zeros bool Nod; // True if no decimal point int Dcm; // Last Dcm digits are decimals -//int Clustered; // 0:No 1:Yes -//int Sorted; // 0:No 1:Asc (2:Desc - NIY) int Deplac; // Offset in dos_buf -//int Ndv; // Number of distinct values -//int Nbm; // Number of uint in bitmap +#if defined(BLK_INDX) + int Clustered; // 0:No 1:Yes + int Sorted; // 0:No 1:Asc (2:Desc - NIY) + int Ndv; // Number of distinct values + int Nbm; // Number of uint in bitmap +#endif // BLK_INDX }; // end of class DOSCOL #endif // __TABDOS_H diff --git a/storage/connect/tabfix.cpp b/storage/connect/tabfix.cpp index 99063e86b57..cb95cebe7d1 100644 --- a/storage/connect/tabfix.cpp +++ b/storage/connect/tabfix.cpp @@ -1,11 +1,11 @@ /************* TabFix C++ Program Source Code File (.CPP) **************/ /* PROGRAM NAME: TABFIX */ /* ------------- */ -/* Version 4.8 */ +/* Version 4.9 */ /* */ /* COPYRIGHT: */ /* ---------- */ -/* (C) Copyright to the author Olivier BERTRAND 1998-2012 */ +/* (C) Copyright to the author Olivier BERTRAND 1998-2014 */ /* */ /* WHAT THIS PROGRAM DOES: */ /* ----------------------- */ @@ -45,6 +45,10 @@ #include "filamfix.h" #include "filamdbf.h" #include "tabfix.h" // TDBFIX, FIXCOL classes declares +#if defined(BLK_INDX) +#include "array.h" +#include "blkfil.h" +#endif // BLK_INDX /***********************************************************************/ /* DB static variables. */ @@ -123,10 +127,52 @@ PCOL TDBFIX::MakeCol(PGLOBAL g, PCOLDEF cdp, PCOL cprec, int n) /***********************************************************************/ /* Remake the indexes after the table was modified. */ /***********************************************************************/ -int TDBFIX::ResetTableOpt(PGLOBAL g, bool dox) +int TDBFIX::ResetTableOpt(PGLOBAL g, bool dop, bool dox) { +#if defined(BLK_INDX) + int prc, rc = RC_OK; + + To_Filter = NULL; // Disable filtering +//To_BlkIdx = NULL; // and block filtering + To_BlkFil = NULL; // and index filtering RestoreNrec(); // May have been modified - return TDBDOS::ResetTableOpt(g, dox); + MaxSize = -1; // Size must be recalculated + Cardinal = -1; // as well as Cardinality + + if (dop) { + Columns = NULL; // Not used anymore + Txfp->Reset(); +// OldBlk = CurBlk = -1; +// ReadBlks = CurNum = Rbuf = Modif = 0; + Use = USE_READY; // So the table can be reopened + Mode = MODE_ANY; // Just to be clean + rc = MakeBlockValues(g); // Redo optimization + } // endif dop + + if (dox && (rc == RC_OK || rc == RC_INFO)) { + // Remake eventual indexes + Columns = NULL; // Not used anymore + Txfp->Reset(); // New start + Use = USE_READY; // So the table can be reopened + Mode = MODE_READ; // New mode + prc = rc; + + if (!(PlgGetUser(g)->Check & CHK_OPT)) { + // After the table was modified the indexes + // are invalid and we should mark them as such... + rc = ((PDOSDEF)To_Def)->InvalidateIndex(g); + } else + // ... or we should remake them. + rc = MakeIndex(g, NULL, FALSE); + + rc = (rc == RC_INFO) ? prc : rc; + } // endif dox + + return rc; +#else // !BLK_INDX + RestoreNrec(); // May have been modified + return TDBDOS::ResetTableOpt(g, dop, dox); +#endif // !BLK_INDX } // end of ResetTableOpt /***********************************************************************/ @@ -163,8 +209,17 @@ int TDBFIX::Cardinality(PGLOBAL g) /***********************************************************************/ int TDBFIX::GetMaxSize(PGLOBAL g) { - if (MaxSize < 0) + if (MaxSize < 0) { MaxSize = Cardinality(g); +#if defined(BLK_INDX) + if (MaxSize > 0 && (To_BlkFil = InitBlockFilter(g, To_Filter)) + && !To_BlkFil->Correlated()) { + // Use BlockTest to reduce the estimated size + MaxSize = Txfp->MaxBlkSize(g, MaxSize); + ResetBlockFilter(g); + } // endif To_BlkFil +#endif // BLK_INDX + } // endif MaxSize return MaxSize; } // end of GetMaxSize @@ -246,6 +301,9 @@ bool TDBFIX::OpenDB(PGLOBAL g) else Txfp->Rewind(); // see comment in Work.log +#if defined(BLK_INDX) + ResetBlockFilter(g); +#endif // BLK_INDX return false; } // endif use @@ -277,6 +335,13 @@ bool TDBFIX::OpenDB(PGLOBAL g) /*********************************************************************/ To_Line = Txfp->GetBuf(); // For WriteDB +#if defined(BLK_INDX) + /*********************************************************************/ + /* Allocate the block filter tree if evaluation is possible. */ + /*********************************************************************/ + To_BlkFil = InitBlockFilter(g, To_Filter); +#endif // BLK_INDX + if (trace) htrc("OpenDos: R%hd mode=%d\n", Tdb_No, Mode); diff --git a/storage/connect/tabfix.h b/storage/connect/tabfix.h index bcd171b37bb..5feb3589928 100644 --- a/storage/connect/tabfix.h +++ b/storage/connect/tabfix.h @@ -38,7 +38,7 @@ class DllExport TDBFIX : public TDBDOS { virtual void ResetDB(void); virtual bool IsUsingTemp(PGLOBAL g); virtual int RowNumber(PGLOBAL g, bool b = false); - virtual int ResetTableOpt(PGLOBAL g, bool dox); + virtual int ResetTableOpt(PGLOBAL g, bool dop, bool dox); virtual void ResetSize(void); virtual int GetBadLines(void) {return Txfp->GetNerr();} diff --git a/storage/connect/tabfmt.cpp b/storage/connect/tabfmt.cpp index 9a121b9ab9a..18ecaae430a 100644 --- a/storage/connect/tabfmt.cpp +++ b/storage/connect/tabfmt.cpp @@ -459,9 +459,12 @@ PTDB CSVDEF::GetTable(PGLOBAL g, MODE mode) if (Compressed == 1) txfp = new(g) ZIPFAM(this); else { +#if defined(BLK_INDX) + txfp = new(g) ZLBFAM(this); +#else // !BLK_INDX strcpy(g->Message, "Compress 2 not supported yet"); -// txfp = new(g) ZLBFAM(defp); return NULL; +#endif // !BLK_INDX } // endelse #else // !ZIP_SUPPORT strcpy(g->Message, "Compress not supported"); @@ -1272,6 +1275,7 @@ CSVCOL::CSVCOL(CSVCOL *col1, PTDB tdbp) : DOSCOL(col1, tdbp) Fldnum = col1->Fldnum; } // end of CSVCOL copy constructor +#if defined(BLK_INDX) /***********************************************************************/ /* VarSize: This function tells UpdateDB whether or not the block */ /* optimization file must be redone if this column is updated, even */ @@ -1290,6 +1294,7 @@ bool CSVCOL::VarSize(void) return false; } // end VarSize +#endif // BLK_INDX /***********************************************************************/ /* ReadColumn: call DOSCOL::ReadColumn after having set the offet */ diff --git a/storage/connect/tabfmt.h b/storage/connect/tabfmt.h index d4a7f4105d6..52b0094bfef 100644 --- a/storage/connect/tabfmt.h +++ b/storage/connect/tabfmt.h @@ -112,7 +112,9 @@ class CSVCOL : public DOSCOL { virtual int GetAmType() {return TYPE_AM_CSV;} // Methods +#if defined(BLK_INDX) virtual bool VarSize(void); +#endif // BLK_INDX virtual void ReadColumn(PGLOBAL g); virtual void WriteColumn(PGLOBAL g); // void Print(FILE *, uint); diff --git a/storage/connect/table.cpp b/storage/connect/table.cpp index 37035ed752d..d23740394b4 100644 --- a/storage/connect/table.cpp +++ b/storage/connect/table.cpp @@ -36,27 +36,6 @@ extern "C" int trace; // The general trace value void NewPointer(PTABS, void *, void *); void AddPointer(PTABS, void *); -/* ---------------------------- class TBX ---------------------------- */ - -/***********************************************************************/ -/* TBX public constructors. */ -/***********************************************************************/ -TBX::TBX(void) - { - Use = USE_NO; - To_Orig = NULL; - To_Filter = NULL; - } // end of TBX constructor - -TBX::TBX(PTBX txp) - { - Use = txp->Use; - To_Orig = txp; - To_Filter = NULL; - } // end of TBX copy constructor - -// Methods - /* ---------------------------- class TDB ---------------------------- */ /***********************************************************************/ @@ -64,6 +43,12 @@ TBX::TBX(PTBX txp) /***********************************************************************/ TDB::TDB(PTABDEF tdp) : Tdb_No(++Tnum) { + Use = USE_NO; + To_Orig = NULL; +#if defined(BLK_INDX) + To_Filter = NULL; +#endif // BLK_FILTER + To_CondFil = NULL; Next = NULL; Name = (tdp) ? tdp->GetName() : NULL; To_Table = NULL; @@ -72,8 +57,14 @@ TDB::TDB(PTABDEF tdp) : Tdb_No(++Tnum) Mode = MODE_READ; } // end of TDB standard constructor -TDB::TDB(PTDB tdbp) : TBX(tdbp), Tdb_No(++Tnum) +TDB::TDB(PTDB tdbp) : Tdb_No(++Tnum) { + Use = tdbp->Use; + To_Orig = tdbp; +#if defined(BLK_INDX) + To_Filter = NULL; +#endif // BLK_FILTER + To_CondFil = NULL; Next = NULL; Name = tdbp->Name; To_Table = tdbp->To_Table; @@ -179,7 +170,7 @@ int TDB::RowNumber(PGLOBAL g, bool b) return 0; } // end of RowNumber -PTBX TDB::Copy(PTABS t) +PTDB TDB::Copy(PTABS t) { PTDB tp, tdb1, tdb2 = NULL, outp = NULL; //PGLOBAL g = t->G; // Is this really useful ??? @@ -398,7 +389,7 @@ PCOL TDBASE::InsertSpcBlk(PGLOBAL g, PCOLDEF cdp) /***********************************************************************/ /* ResetTableOpt: Wrong for this table type. */ /***********************************************************************/ -int TDBASE::ResetTableOpt(PGLOBAL g, bool dox) +int TDBASE::ResetTableOpt(PGLOBAL g, bool dop, bool dox) { strcpy(g->Message, "This table is not indexable"); return RC_INFO; diff --git a/storage/connect/tabmac.h b/storage/connect/tabmac.h index cfdf842cdc8..eb115b18049 100644 --- a/storage/connect/tabmac.h +++ b/storage/connect/tabmac.h @@ -31,7 +31,7 @@ class DllExport MACDEF : public TABDEF { /* Logical table description */ // Methods virtual bool DefineAM(PGLOBAL g, LPCSTR am, int poff); virtual PTDB GetTable(PGLOBAL g, MODE m); - virtual bool DeleteTableFile(PGLOBAL g) {return true;} +//virtual bool DeleteTableFile(PGLOBAL g) {return true;} protected: // Members diff --git a/storage/connect/tabmul.h b/storage/connect/tabmul.h index 052b4e7d33e..379e8f88e93 100644 --- a/storage/connect/tabmul.h +++ b/storage/connect/tabmul.h @@ -38,7 +38,7 @@ class DllExport TDBMUL : public TDBASE { // Methods virtual void ResetDB(void); virtual PTDB CopyOne(PTABS t); - virtual bool IsSame(PTBX tp) {return tp == (PTBX)Tdbp;} + virtual bool IsSame(PTDB tp) {return tp == (PTDB)Tdbp;} virtual PSZ GetFile(PGLOBAL g) {return Tdbp->GetFile(g);} virtual int GetRecpos(void) {return 0;} virtual PCOL ColDB(PGLOBAL g, PSZ name, int num); diff --git a/storage/connect/tabmysql.cpp b/storage/connect/tabmysql.cpp index 9b4f0dbe3ce..ce41a8429be 100644 --- a/storage/connect/tabmysql.cpp +++ b/storage/connect/tabmysql.cpp @@ -527,8 +527,8 @@ bool TDBMYSQL::MakeSelect(PGLOBAL g) strcat(strcat(strcat(strcat(Query, " FROM "), tk), Tabname), tk); - if (To_Filter) - strcat(strcat(Query, " WHERE "), To_Filter->Body); + if (To_CondFil) + strcat(strcat(Query, " WHERE "), To_CondFil->Body); if (trace) htrc("Query=%s\n", Query); @@ -1395,11 +1395,11 @@ PCMD TDBMYEXC::MakeCMD(PGLOBAL g) { PCMD xcmd = NULL; - if (To_Filter) { + if (To_CondFil) { if (Cmdcol) { - if (!stricmp(Cmdcol, To_Filter->Body) && - (To_Filter->Op == OP_EQ || To_Filter->Op == OP_IN)) { - xcmd = To_Filter->Cmds; + if (!stricmp(Cmdcol, To_CondFil->Body) && + (To_CondFil->Op == OP_EQ || To_CondFil->Op == OP_IN)) { + xcmd = To_CondFil->Cmds; } else strcpy(g->Message, "Invalid command specification filter"); diff --git a/storage/connect/tabodbc.cpp b/storage/connect/tabodbc.cpp index 1fb71e33646..c7513601979 100644 --- a/storage/connect/tabodbc.cpp +++ b/storage/connect/tabodbc.cpp @@ -408,7 +408,7 @@ char *TDBODBC::MakeSQL(PGLOBAL g, bool cnt) // Below 14 is length of 'select ' + length of ' from ' + 1 len = (strlen(colist) + strlen(buf) + 14); - len += (To_Filter ? strlen(To_Filter->Body) + 7 : 0); + len += (To_CondFil ? strlen(To_CondFil->Body) + 7 : 0); if (Catalog && *Catalog) catp = Catalog; @@ -441,8 +441,8 @@ char *TDBODBC::MakeSQL(PGLOBAL g, bool cnt) strcat(sql, tabname); - if (To_Filter) - strcat(strcat(sql, " WHERE "), To_Filter->Body); + if (To_CondFil) + strcat(strcat(sql, " WHERE "), To_CondFil->Body); return sql; } // end of MakeSQL @@ -1229,11 +1229,11 @@ PCMD TDBXDBC::MakeCMD(PGLOBAL g) { PCMD xcmd = NULL; - if (To_Filter) { + if (To_CondFil) { if (Cmdcol) { - if (!stricmp(Cmdcol, To_Filter->Body) && - (To_Filter->Op == OP_EQ || To_Filter->Op == OP_IN)) { - xcmd = To_Filter->Cmds; + if (!stricmp(Cmdcol, To_CondFil->Body) && + (To_CondFil->Op == OP_EQ || To_CondFil->Op == OP_IN)) { + xcmd = To_CondFil->Cmds; } else strcpy(g->Message, "Invalid command specification filter"); diff --git a/storage/connect/tabsys.cpp b/storage/connect/tabsys.cpp index 2bb5532cea0..409352fdee6 100644 --- a/storage/connect/tabsys.cpp +++ b/storage/connect/tabsys.cpp @@ -113,6 +113,7 @@ PTDB INIDEF::GetTable(PGLOBAL g, MODE m) return tdbp; } // end of GetTable +#if 0 /***********************************************************************/ /* DeleteTableFile: Delete INI table files using platform API. */ /***********************************************************************/ @@ -134,6 +135,7 @@ bool INIDEF::DeleteTableFile(PGLOBAL g) return rc; // Return true if error } // end of DeleteTableFile +#endif // 0 /* ------------------------------------------------------------------- */ diff --git a/storage/connect/tabsys.h b/storage/connect/tabsys.h index 2780eb3ca98..714f2475873 100644 --- a/storage/connect/tabsys.h +++ b/storage/connect/tabsys.h @@ -32,7 +32,7 @@ class DllExport INIDEF : public TABDEF { /* INI table description */ // Methods virtual bool DefineAM(PGLOBAL g, LPCSTR am, int poff); virtual PTDB GetTable(PGLOBAL g, MODE m); - virtual bool DeleteTableFile(PGLOBAL g); +//virtual bool DeleteTableFile(PGLOBAL g); protected: // Members diff --git a/storage/connect/tabtbl.cpp b/storage/connect/tabtbl.cpp index 56305871c69..c78c62af9cc 100644 --- a/storage/connect/tabtbl.cpp +++ b/storage/connect/tabtbl.cpp @@ -1,11 +1,11 @@ /************* TabTbl C++ Program Source Code File (.CPP) **************/ /* PROGRAM NAME: TABTBL */ /* ------------- */ -/* Version 1.6 */ +/* Version 1.7 */ /* */ /* COPYRIGHT: */ /* ---------- */ -/* (C) Copyright to PlugDB Software Development 2008-2013 */ +/* (C) Copyright to PlugDB Software Development 2008-2014 */ /* Author: Olivier BERTRAND */ /* */ /* WHAT THIS PROGRAM DOES: */ @@ -66,7 +66,6 @@ #include "global.h" // global declarations #include "plgdbsem.h" // DB application declarations #include "reldef.h" // DB definition declares -//#include "filter.h" // FILTER classes dcls #include "filamtxt.h" #include "tabcol.h" #include "tabdos.h" // TDBDOS and DOSCOL class dcls @@ -245,7 +244,7 @@ bool TDBTBL::InitTableList(PGLOBAL g) // PlugSetPath(filename, Tdbp->GetFile(g), Tdbp->GetPath()); for (n = 0, tp = tdp->Tablep; tp; tp = tp->GetNext()) { - if (TestFil(g, To_Filter, tp)) { + if (TestFil(g, To_CondFil, tp)) { tabp = new(g) XTAB(tp); if (tabp->GetSrc()) { @@ -286,14 +285,14 @@ bool TDBTBL::InitTableList(PGLOBAL g) hc->get_table()->s->connect_string.length = sln; //NumTables = n; - To_Filter = NULL; // To avoid doing it several times + To_CondFil = NULL; // To avoid doing it several times return FALSE; } // end of InitTableList /***********************************************************************/ /* Test the tablename against the pseudo "local" filter. */ /***********************************************************************/ -bool TDBTBL::TestFil(PGLOBAL g, PFIL filp, PTABLE tabp) +bool TDBTBL::TestFil(PGLOBAL g, PCFIL filp, PTABLE tabp) { char *body, *fil, op[8], tn[NAME_LEN]; bool neg; @@ -421,12 +420,12 @@ bool TDBTBL::OpenDB(PGLOBAL g) } // endif use /*********************************************************************/ - /* When GetMaxsize was called, To_Filter was not set yet. */ + /* When GetMaxsize was called, To_CondFil was not set yet. */ /*********************************************************************/ - if (To_Filter && Tablist) { + if (To_CondFil && Tablist) { Tablist = NULL; Nbc = 0; - } // endif To_Filter + } // endif To_CondFil /*********************************************************************/ /* Open the first table of the list. */ @@ -661,12 +660,12 @@ bool TDBTBM::OpenDB(PGLOBAL g) #if 0 /*********************************************************************/ - /* When GetMaxsize was called, To_Filter was not set yet. */ + /* When GetMaxsize was called, To_CondFil was not set yet. */ /*********************************************************************/ - if (To_Filter && Tablist) { + if (To_CondFil && Tablist) { Tablist = NULL; Nbc = 0; - } // endif To_Filter + } // endif To_CondFil #endif // 0 /*********************************************************************/ diff --git a/storage/connect/tabtbl.h b/storage/connect/tabtbl.h index 48371e40ade..0ff6c37eaed 100644 --- a/storage/connect/tabtbl.h +++ b/storage/connect/tabtbl.h @@ -87,7 +87,7 @@ class DllExport TDBTBL : public TDBPRX { protected: // Internal functions bool InitTableList(PGLOBAL g); - bool TestFil(PGLOBAL g, PFIL filp, PTABLE tabp); + bool TestFil(PGLOBAL g, PCFIL filp, PTABLE tabp); // Members PTABLE Tablist; // Points to the table list @@ -152,7 +152,7 @@ class DllExport TDBTBM : public TDBTBL { protected: // Internal functions //bool InitTableList(PGLOBAL g); -//bool TestFil(PGLOBAL g, PFIL filp, PTABLE tabp); +//bool TestFil(PGLOBAL g, PCFIL filp, PTABLE tabp); bool OpenTables(PGLOBAL g); int ReadNextRemote(PGLOBAL g); diff --git a/storage/connect/tabvct.cpp b/storage/connect/tabvct.cpp index 8b7d8ba91eb..cb5e6f3200c 100644 --- a/storage/connect/tabvct.cpp +++ b/storage/connect/tabvct.cpp @@ -1,11 +1,11 @@ /************* TabVct C++ Program Source Code File (.CPP) **************/ /* PROGRAM NAME: TABVCT */ /* ------------- */ -/* Version 3.7 */ +/* Version 3.8 */ /* */ /* COPYRIGHT: */ /* ---------- */ -/* (C) Copyright to the author Olivier BERTRAND 1999-2012 */ +/* (C) Copyright to the author Olivier BERTRAND 1999-2014 */ /* */ /* WHAT THIS PROGRAM DOES: */ /* ----------------------- */ @@ -118,6 +118,7 @@ bool VCTDEF::DefineAM(PGLOBAL g, LPCSTR am, int poff) return false; } // end of DefineAM +#if 0 /***********************************************************************/ /* Erase: This was made a separate routine because a strange thing */ /* happened when DeleteTablefile was defined for the VCTDEF class: */ @@ -157,6 +158,7 @@ bool VCTDEF::Erase(char *filename) return rc; // Return true if error } // end of Erase +#endif // 0 /***********************************************************************/ /* Prepare the column file name pattern for a split table. */ @@ -231,7 +233,8 @@ PTDB VCTDEF::GetTable(PGLOBAL g, MODE mode) /*********************************************************************/ if (mode != MODE_INSERT) if (tdbp->GetBlockValues(g)) - return NULL; + PushWarning(g, (PTDBASE)tdbp); +// return NULL; // causes a crash when deleting index return tdbp; } // end of GetTable @@ -298,6 +301,9 @@ bool TDBVCT::OpenDB(PGLOBAL g) To_Kindex->Reset(); Txfp->Rewind(); +#if defined(BLK_INDX) + ResetBlockFilter(g); +#endif // BLK_INDX return false; } // endif Use @@ -319,6 +325,13 @@ bool TDBVCT::OpenDB(PGLOBAL g) // This was not done in previous version Use = USE_OPEN; // Do it now in case we are recursively called +#if defined(BLK_INDX) + /*********************************************************************/ + /* Allocate the block filter tree if evaluation is possible. */ + /*********************************************************************/ + To_BlkFil = InitBlockFilter(g, To_Filter); +#endif // BLK_INDX + /*********************************************************************/ /* Reset buffer access according to indexing and to mode. */ /*********************************************************************/ diff --git a/storage/connect/tabvct.h b/storage/connect/tabvct.h index 4049b4f7683..f1c0a8a3a98 100644 --- a/storage/connect/tabvct.h +++ b/storage/connect/tabvct.h @@ -37,7 +37,7 @@ class DllExport VCTDEF : public DOSDEF { /* Logical table description */ protected: // Specific file erase routine for vertical tables - virtual bool Erase(char *filename); +//virtual bool Erase(char *filename); int MakeFnPattern(char *fpat); // Members diff --git a/storage/connect/tabwmi.cpp b/storage/connect/tabwmi.cpp index 8f91d9b3ed8..5052268b9e2 100644 --- a/storage/connect/tabwmi.cpp +++ b/storage/connect/tabwmi.cpp @@ -14,7 +14,7 @@ #include "reldef.h" #include "xtable.h" #include "colblk.h" -#include "filter.h" +//#include "filter.h" //#include "xindex.h" #include "tabwmi.h" #include "valblk.h" @@ -480,8 +480,8 @@ bool TDBWMI::Initialize(PGLOBAL g) /***********************************************************************/ void TDBWMI::DoubleSlash(PGLOBAL g) { - if (To_Filter && strchr(To_Filter->Body, '\\')) { - char *body = To_Filter->Body; + if (To_CondFil && strchr(To_CondFil->Body, '\\')) { + char *body = To_CondFil->Body; char *buf = (char*)PlugSubAlloc(g, NULL, strlen(body) * 2); int i = 0, k = 0; @@ -492,8 +492,8 @@ void TDBWMI::DoubleSlash(PGLOBAL g) buf[k++] = body[i]; } while (body[i++]); - To_Filter->Body = buf; - } // endif To_Filter + To_CondFil->Body = buf; + } // endif To_CondFil } // end of DoubleSlash @@ -540,13 +540,13 @@ char *TDBWMI::MakeWQL(PGLOBAL g) // Below 14 is length of 'select ' + length of ' from ' + 1 len = (strlen(colist) + strlen(Wclass) + 14); - len += (To_Filter ? strlen(To_Filter->Body) + 7 : 0); + len += (To_CondFil ? strlen(To_CondFil->Body) + 7 : 0); wql = (char*)PlugSubAlloc(g, NULL, len); strcat(strcat(strcpy(wql, "SELECT "), colist), " FROM "); strcat(wql, Wclass); - if (To_Filter) - strcat(strcat(wql, " WHERE "), To_Filter->Body); + if (To_CondFil) + strcat(strcat(wql, " WHERE "), To_CondFil->Body); return wql; } // end of MakeWQL @@ -659,8 +659,8 @@ bool TDBWMI::OpenDB(PGLOBAL g) return true; } // endif Mode - if (!To_Filter && !stricmp(Wclass, "CIM_Datafile") - && !stricmp(Nspace, "root\\cimv2")) { + if (!To_CondFil && !stricmp(Wclass, "CIM_Datafile") + && !stricmp(Nspace, "root\\cimv2")) { strcpy(g->Message, "Would last forever when not filtered, use DIR table instead"); return true; diff --git a/storage/connect/tabwmi.h b/storage/connect/tabwmi.h index 9df57e7c579..558e527773e 100644 --- a/storage/connect/tabwmi.h +++ b/storage/connect/tabwmi.h @@ -48,7 +48,7 @@ class WMIDEF : public TABDEF { /* Logical table description */ // Methods virtual bool DefineAM(PGLOBAL g, LPCSTR am, int poff); virtual PTDB GetTable(PGLOBAL g, MODE m); - virtual bool DeleteTableFile(PGLOBAL g) {return true;} +//virtual bool DeleteTableFile(PGLOBAL g) {return true;} protected: // Members diff --git a/storage/connect/tabxml.cpp b/storage/connect/tabxml.cpp index 7e4ee3453ae..7a2c0169c2b 100644 --- a/storage/connect/tabxml.cpp +++ b/storage/connect/tabxml.cpp @@ -187,6 +187,7 @@ PTDB XMLDEF::GetTable(PGLOBAL g, MODE m) return tdbp; } // end of GetTable +#if 0 /***********************************************************************/ /* DeleteTableFile: Delete XML table files using platform API. */ /***********************************************************************/ @@ -208,6 +209,7 @@ bool XMLDEF::DeleteTableFile(PGLOBAL g) return rc; // Return true if error } // end of DeleteTableFile +#endif // 0 /* ------------------------- TDBXML Class ---------------------------- */ diff --git a/storage/connect/tabxml.h b/storage/connect/tabxml.h index 817bea45b3b..20998589967 100644 --- a/storage/connect/tabxml.h +++ b/storage/connect/tabxml.h @@ -30,7 +30,7 @@ class DllExport XMLDEF : public TABDEF { /* Logical table description */ // Methods virtual bool DefineAM(PGLOBAL g, LPCSTR am, int poff); virtual PTDB GetTable(PGLOBAL g, MODE m); - virtual bool DeleteTableFile(PGLOBAL g); +//virtual bool DeleteTableFile(PGLOBAL g); protected: // Members diff --git a/storage/connect/valblk.cpp b/storage/connect/valblk.cpp index c0127072896..e8da2044a47 100644 --- a/storage/connect/valblk.cpp +++ b/storage/connect/valblk.cpp @@ -40,7 +40,8 @@ #include "plgdbsem.h" #include "valblk.h" -#define CheckBlanks assert(!Blanks); +#define CheckBlanks assert(!Blanks); +#define CheckParms(V, N) ChkIndx(N); ChkTyp(V); /***********************************************************************/ /* AllocValBlock: allocate a VALBLK according to type. */ @@ -447,6 +448,38 @@ template <> uchar TYPBLK::GetTypedValue(PVBLK blk, int n) {return blk->GetUTinyValue(n);} +#if defined(BLK_INDX) +/***********************************************************************/ +/* Set one value in a block if val is less than the current value. */ +/***********************************************************************/ +template +void TYPBLK::SetMin(PVAL valp, int n) + { + CheckParms(valp, n) + TYPE tval = GetTypedValue(valp); + TYPE& tmin = Typp[n]; + + if (tval < tmin) + tmin = tval; + + } // end of SetMin + +/***********************************************************************/ +/* Set one value in a block if val is greater than the current value. */ +/***********************************************************************/ +template +void TYPBLK::SetMax(PVAL valp, int n) + { + CheckParms(valp, n) + TYPE tval = GetTypedValue(valp); + TYPE& tmin = Typp[n]; + + if (tval > tmin) + tmin = tval; + + } // end of SetMax +#endif // BLK_INDX + #if 0 /***********************************************************************/ /* Set many values in a block from values in another block. */ @@ -779,6 +812,38 @@ void CHRBLK::SetValue(PVBLK pv, int n1, int n2) SetNull(n1, b); } // end of SetValue +#if defined(BLK_INDX) +/***********************************************************************/ +/* Set one value in a block if val is less than the current value. */ +/***********************************************************************/ +void CHRBLK::SetMin(PVAL valp, int n) + { + CheckParms(valp, n) + CheckBlanks + char *vp = valp->GetCharValue(); + char *bp = Chrp + n * Long; + + if (((Ci) ? strnicmp(vp, bp, Long) : strncmp(vp, bp, Long)) < 0) + memcpy(bp, vp, Long); + + } // end of SetMin + +/***********************************************************************/ +/* Set one value in a block if val is greater than the current value. */ +/***********************************************************************/ +void CHRBLK::SetMax(PVAL valp, int n) + { + CheckParms(valp, n) + CheckBlanks + char *vp = valp->GetCharValue(); + char *bp = Chrp + n * Long; + + if (((Ci) ? strnicmp(vp, bp, Long) : strncmp(vp, bp, Long)) > 0) + memcpy(bp, vp, Long); + + } // end of SetMax +#endif // BLK_INDX + #if 0 /***********************************************************************/ /* Set many values in a block from values in another block. */ @@ -1101,6 +1166,36 @@ void STRBLK::SetValue(char *sp, uint len, int n) Strp[n] = p; } // end of SetValue +#if defined(BLK_INDX) +/***********************************************************************/ +/* Set one value in a block if val is less than the current value. */ +/***********************************************************************/ +void STRBLK::SetMin(PVAL valp, int n) + { + CheckParms(valp, n) + char *vp = valp->GetCharValue(); + char *bp = Strp[n]; + + if (strcmp(vp, bp) < 0) + SetValue(valp, n); + + } // end of SetMin + +/***********************************************************************/ +/* Set one value in a block if val is greater than the current value. */ +/***********************************************************************/ +void STRBLK::SetMax(PVAL valp, int n) + { + CheckParms(valp, n) + char *vp = valp->GetCharValue(); + char *bp = Strp[n]; + + if (strcmp(vp, bp) > 0) + SetValue(valp, n); + + } // end of SetMax +#endif // BLK_INDX + /***********************************************************************/ /* Move one value from i to j. */ /***********************************************************************/ @@ -1240,5 +1335,51 @@ void DATBLK::SetValue(PSZ p, int n) } // end of SetValue +#if defined(BLK_INDX) +/* -------------------------- Class MBVALS --------------------------- */ + +/***********************************************************************/ +/* Allocate a value block according to type,len, and nb of values. */ +/***********************************************************************/ +PVBLK MBVALS::Allocate(PGLOBAL g, int type, int len, int prec, + int n, bool sub) + { + Mblk.Sub = sub; + Mblk.Size = n * GetTypeSize(type, len); + + if (!PlgDBalloc(g, NULL, Mblk)) { + sprintf(g->Message, MSG(ALLOC_ERROR), "MBVALS::Allocate"); + return NULL; + } else + Vblk = AllocValBlock(g, Mblk.Memp, type, n, len, prec, + TRUE, TRUE, FALSE); + + return Vblk; + } // end of Allocate + +/***********************************************************************/ +/* Reallocate the value block according to the new size. */ +/***********************************************************************/ +bool MBVALS::ReAllocate(PGLOBAL g, int n) + { + if (!PlgDBrealloc(g, NULL, Mblk, n * Vblk->GetVlen())) { + sprintf(g->Message, MSG(ALLOC_ERROR), "MBVALS::ReAllocate"); + return TRUE; + } else + Vblk->ReAlloc(Mblk.Memp, n); + + return FALSE; + } // end of ReAllocate + +/***********************************************************************/ +/* Free the value block. */ +/***********************************************************************/ +void MBVALS::Free(void) + { + PlgDBfree(Mblk); + Vblk = NULL; + } // end of Free +#endif // BLK_INDX + /* ------------------------- End of Valblk --------------------------- */ diff --git a/storage/connect/valblk.h b/storage/connect/valblk.h index 1edfe7f76b4..6b427512332 100644 --- a/storage/connect/valblk.h +++ b/storage/connect/valblk.h @@ -22,6 +22,38 @@ DllExport PVBLK AllocValBlock(PGLOBAL, void*, int, int, int, int, bool, bool, bool); const char *GetFmt(int type, bool un = false); +#if defined(BLK_INDX) +/***********************************************************************/ +/* DB static external variables. */ +/***********************************************************************/ +extern MBLOCK Nmblk; /* Used to initialize MBLOCK's */ + +/***********************************************************************/ +/* Class MBVALS is a utility class for (re)allocating VALBLK's. */ +/***********************************************************************/ +class MBVALS : public BLOCK { +//friend class LSTBLK; + friend class ARRAY; + public: + // Constructors + MBVALS(void) {Vblk = NULL; Mblk = Nmblk;} + + // Methods + void *GetMemp(void) {return Mblk.Memp;} + PVBLK Allocate(PGLOBAL g, int type, int len, int prec, + int n, bool sub = FALSE); + bool ReAllocate(PGLOBAL g, int n); + void Free(void); + + protected: + // Members + PVBLK Vblk; // Pointer to VALBLK + MBLOCK Mblk; // The memory block + }; // end of class MBVALS + +typedef class MBVALS *PMBV; +#endif // BLK_INDX + /***********************************************************************/ /* Class VALBLK represent a base class for variable blocks. */ /***********************************************************************/ @@ -79,9 +111,11 @@ class VALBLK : public BLOCK { virtual void SetValue(char *sp, uint len, int n) {assert(false);} virtual void SetValue(PVAL valp, int n) = 0; virtual void SetValue(PVBLK pv, int n1, int n2) = 0; -#if 0 +#if defined(BLK_INDX) virtual void SetMin(PVAL valp, int n) = 0; virtual void SetMax(PVAL valp, int n) = 0; +#endif // BLK_INDX +#if 0 virtual void SetValues(PVBLK pv, int i, int n) = 0; virtual void AddMinus1(PVBLK pv, int n1, int n2) {assert(false);} #endif // 0 @@ -161,6 +195,10 @@ class TYPBLK : public VALBLK { virtual void SetValue(PVAL valp, int n); virtual void SetValue(PVBLK pv, int n1, int n2); //virtual void SetValues(PVBLK pv, int k, int n); +#if defined(BLK_INDX) + virtual void SetMin(PVAL valp, int n); + virtual void SetMax(PVAL valp, int n); +#endif // BLK_INDX virtual void Move(int i, int j); virtual int CompVal(PVAL vp, int n); virtual int CompVal(int i1, int i2); @@ -212,6 +250,10 @@ class CHRBLK : public VALBLK { virtual void SetValue(PVAL valp, int n); virtual void SetValue(PVBLK pv, int n1, int n2); //virtual void SetValues(PVBLK pv, int k, int n); +#if defined(BLK_INDX) + virtual void SetMin(PVAL valp, int n); + virtual void SetMax(PVAL valp, int n); +#endif // BLK_INDX virtual void Move(int i, int j); virtual int CompVal(PVAL vp, int n); virtual int CompVal(int i1, int i2); @@ -264,6 +306,10 @@ class STRBLK : public VALBLK { virtual void SetValue(PVAL valp, int n); virtual void SetValue(PVBLK pv, int n1, int n2); //virtual void SetValues(PVBLK pv, int k, int n); +#if defined(BLK_INDX) + virtual void SetMin(PVAL valp, int n); + virtual void SetMax(PVAL valp, int n); +#endif // BLK_INDX virtual void Move(int i, int j); virtual int CompVal(PVAL vp, int n); virtual int CompVal(int i1, int i2); diff --git a/storage/connect/value.cpp b/storage/connect/value.cpp index c818070b970..7653b222a41 100644 --- a/storage/connect/value.cpp +++ b/storage/connect/value.cpp @@ -83,6 +83,7 @@ int DTVAL::Shift = 0; /* Routines called externally. */ /***********************************************************************/ bool PlugEvalLike(PGLOBAL, LPCSTR, LPCSTR, bool); + #if !defined(WIN32) extern "C" { PSZ strupr(PSZ s); @@ -90,6 +91,34 @@ PSZ strlwr(PSZ s); } #endif // !WIN32 +#if defined(BLK_INDX) +/***********************************************************************/ +/* Returns the bitmap representing the conditions that must not be */ +/* met when returning from TestValue for a given operator. */ +/* Bit one is EQ, bit 2 is LT, and bit 3 is GT. */ +/***********************************************************************/ +BYTE OpBmp(PGLOBAL g, OPVAL opc) + { + BYTE bt; + + switch (opc) { + case OP_IN: + case OP_EQ: bt = 0x06; break; + case OP_NE: bt = 0x01; break; + case OP_GT: bt = 0x03; break; + case OP_GE: bt = 0x02; break; + case OP_LT: bt = 0x05; break; + case OP_LE: bt = 0x04; break; + case OP_EXIST: bt = 0x00; break; + default: + sprintf(g->Message, MSG(BAD_FILTER_OP), opc); + longjmp(g->jumper[g->jump_level], TYPE_ARRAY); + } // endswitch opc + + return bt; + } // end of OpBmp +#endif // BLK_INDX + /***********************************************************************/ /* Get a long long number from its character representation. */ /* IN p: Pointer to the numeric string */ @@ -277,7 +306,7 @@ const char *GetFmt(int type, bool un) return fmt; } // end of GetFmt -#if 0 +#if defined(BLK_INDX) /***********************************************************************/ /* ConvertType: what this function does is to determine the type to */ /* which should be converted a value so no precision would be lost. */ @@ -324,7 +353,7 @@ int ConvertType(int target, int type, CONV kind, bool match) } // endswitch kind } // end of ConvertType -#endif // 0 +#endif // BLK_INDX /***********************************************************************/ /* AllocateConstant: allocates a constant Value. */ @@ -422,7 +451,7 @@ PVAL AllocateValue(PGLOBAL g, int type, int len, int prec, return valp; } // end of AllocateValue -#if 0 +#if defined(BLK_INDX) /***********************************************************************/ /* Allocate a constant Value converted to newtype. */ /* Can also be used to copy a Value eventually converted. */ @@ -490,7 +519,7 @@ PVAL AllocateValue(PGLOBAL g, PVAL valp, int newtype, int uns) valp->SetGlobal(g); return valp; } // end of AllocateValue -#endif // 0 +#endif // BLK_INDX /* -------------------------- Class VALUE ---------------------------- */ @@ -527,6 +556,20 @@ const char *VALUE::GetXfmt(void) return fmt; } // end of GetFmt +#if defined(BLK_INDX) +/***********************************************************************/ +/* Returns a BYTE indicating the comparison between two values. */ +/* Bit 1 indicates equality, Bit 2 less than, and Bit3 greater than. */ +/* More than 1 bit can be set only in the case of TYPE_LIST. */ +/***********************************************************************/ +BYTE VALUE::TestValue(PVAL vp) + { + int n = CompareValue(vp); + + return (n > 0) ? 0x04 : (n < 0) ? 0x02 : 0x01; + } // end of TestValue +#endif // BLK_INDX + /* -------------------------- Class TYPVAL ---------------------------- */ /***********************************************************************/ @@ -897,6 +940,26 @@ bool TYPVAL::IsEqual(PVAL vp, bool chktype) } // end of IsEqual +#if defined(BLK_INDX) +/***********************************************************************/ +/* Compare values and returns 1, 0 or -1 according to comparison. */ +/* This function is used for evaluation of numeric filters. */ +/***********************************************************************/ +template +int TYPVAL::CompareValue(PVAL vp) + { +//assert(vp->GetType() == Type); + + // Process filtering on numeric values. + TYPE n = GetTypedValue(vp); + +//if (trace) +// htrc(" Comparing: val=%d,%d\n", Tval, n); + + return (Tval > n) ? 1 : (Tval < n) ? (-1) : 0; + } // end of CompareValue +#endif // BLK_INDX + /***********************************************************************/ /* FormatValue: This function set vp (a STRING value) to the string */ /* constructed from its own value formated using the fmt format. */ @@ -1347,6 +1410,34 @@ bool TYPVAL::IsEqual(PVAL vp, bool chktype) } // end of IsEqual +#if defined(BLK_INDX) +/***********************************************************************/ +/* Compare values and returns 1, 0 or -1 according to comparison. */ +/* This function is used for evaluation of numeric filters. */ +/***********************************************************************/ +int TYPVAL::CompareValue(PVAL vp) + { + int n; +//assert(vp->GetType() == Type); + + if (trace) + htrc(" Comparing: val='%s','%s'\n", Strp, vp->GetCharValue()); + + // Process filtering on character strings. + if (Ci || vp->IsCi()) + n = stricmp(Strp, vp->GetCharValue()); + else + n = strcmp(Strp, vp->GetCharValue()); + +#if defined(WIN32) + if (n == _NLSCMPERROR) + return n; // Here we should raise an error +#endif // WIN32 + + return (n > 0) ? 1 : (n < 0) ? -1 : 0; + } // end of CompareValue +#endif // BLK_INDX + /***********************************************************************/ /* FormatValue: This function set vp (a STRING value) to the string */ /* constructed from its own value formated using the fmt format. */ @@ -1573,6 +1664,25 @@ bool DECVAL::IsEqual(PVAL vp, bool chktype) return !strcmp(Strp, vp->GetCharString(buf)); } // end of IsEqual +#if defined(BLK_INDX) +/***********************************************************************/ +/* Compare values and returns 1, 0 or -1 according to comparison. */ +/* This function is used for evaluation of numeric filters. */ +/***********************************************************************/ +int DECVAL::CompareValue(PVAL vp) + { +//assert(vp->GetType() == Type); + + // Process filtering on numeric values. + double f = atof(Strp), n = vp->GetFloatValue(); + +//if (trace) +// htrc(" Comparing: val=%d,%d\n", f, n); + + return (f > n) ? 1 : (f < n) ? (-1) : 0; + } // end of CompareValue +#endif // BLK_INDX + #if 0 /***********************************************************************/ /* FormatValue: This function set vp (a STRING value) to the string */ diff --git a/storage/connect/value.h b/storage/connect/value.h index f8e89ba55fd..39fee7f73bb 100644 --- a/storage/connect/value.h +++ b/storage/connect/value.h @@ -46,8 +46,10 @@ DllExport char *GetFormatType(int); DllExport int GetFormatType(char); DllExport bool IsTypeChar(int type); DllExport bool IsTypeNum(int type); -//lExport int ConvertType(int, int, CONV, bool match = false); -//lExport PVAL AllocateValue(PGLOBAL, PVAL, int = TYPE_VOID, int = 0); +#if defined(BLK_INDX) +DllExport int ConvertType(int, int, CONV, bool match = false); +DllExport PVAL AllocateValue(PGLOBAL, PVAL, int = TYPE_VOID, int = 0); +#endif // BLK_INDX DllExport PVAL AllocateValue(PGLOBAL, int, int len = 0, int prec = 0, bool uns = false, PSZ fmt = NULL); DllExport ulonglong CharToNumber(char *, int, ulonglong, bool, @@ -95,6 +97,11 @@ class DllExport VALUE : public BLOCK { virtual bool SetValue_pval(PVAL valp, bool chktype = false) = 0; virtual bool SetValue_char(char *p, int n) = 0; virtual void SetValue_psz(PSZ s) = 0; +#if defined(BLK_INDX) + virtual void SetValue_bool(bool b) {assert(FALSE);} + virtual int CompareValue(PVAL vp) = 0; + virtual BYTE TestValue(PVAL vp); +#endif // BLK_INDX virtual void SetValue(char c) {assert(false);} virtual void SetValue(uchar c) {assert(false);} virtual void SetValue(short i) {assert(false);} @@ -163,6 +170,10 @@ class DllExport TYPVAL : public VALUE { virtual bool SetValue_pval(PVAL valp, bool chktype); virtual bool SetValue_char(char *p, int n); virtual void SetValue_psz(PSZ s); +#if defined(BLK_INDX) + virtual void SetValue_bool(bool b) {Tval = (b) ? 1 : 0;} + virtual int CompareValue(PVAL vp); +#endif // BLK_INDX virtual void SetValue(char c) {Tval = (TYPE)c; Null = false;} virtual void SetValue(uchar c) {Tval = (TYPE)c; Null = false;} virtual void SetValue(short i) {Tval = (TYPE)i; Null = false;} @@ -242,6 +253,9 @@ class DllExport TYPVAL: public VALUE { virtual void SetValue(ulonglong n); virtual void SetValue(double f); virtual void SetBinValue(void *p); +#if defined(BLK_INDX) + virtual int CompareValue(PVAL vp); +#endif // BLK_INDX virtual bool GetBinValue(void *buf, int buflen, bool go); virtual char *ShowValue(char *buf, int); virtual char *GetCharString(char *p); @@ -280,6 +294,9 @@ class DllExport DECVAL: public TYPVAL { virtual char *ShowValue(char *buf, int); //virtual char *GetCharString(char *p); virtual bool IsEqual(PVAL vp, bool chktype); +#if defined(BLK_INDX) + virtual int CompareValue(PVAL vp); +#endif // BLK_INDX //virtual bool FormatValue(PVAL vp, char *fmt); //virtual bool SetConstFormat(PGLOBAL, FORMAT&); diff --git a/storage/connect/xobject.cpp b/storage/connect/xobject.cpp index aa87517bd2e..4050fd520fa 100644 --- a/storage/connect/xobject.cpp +++ b/storage/connect/xobject.cpp @@ -109,7 +109,7 @@ int CONSTANT::GetLengthEx(void) return Value->GetValLen(); } // end of GetLengthEx -#if 0 +#if defined(BLK_INDX) /***********************************************************************/ /* Convert a constant to the given type. */ /***********************************************************************/ @@ -120,7 +120,7 @@ void CONSTANT::Convert(PGLOBAL g, int newtype) longjmp(g->jumper[g->jump_level], TYPE_CONST); } // end of Convert -#endif // 0 +#endif // BLK_INDX /***********************************************************************/ /* Compare: returns true if this object is equivalent to xp. */ diff --git a/storage/connect/xobject.h b/storage/connect/xobject.h index b7869be96b2..793a08ddb9e 100644 --- a/storage/connect/xobject.h +++ b/storage/connect/xobject.h @@ -68,7 +68,7 @@ class DllExport XOBJECT : public BLOCK { virtual bool CheckLocal(PTDB) {return true;} virtual int CheckSpcCol(PTDB, int) {return 2;} virtual bool CheckSort(PTDB) {return false;} - virtual bool VerifyColumn(PTBX txp) {return false;} + virtual bool VerifyColumn(PTDB txp) {return false;} virtual bool VerifyTdb(PTDB& tdbp) {return false;} virtual bool IsColInside(PCOL colp) {return false;} @@ -123,10 +123,12 @@ class DllExport CONSTANT : public XOBJECT { virtual bool SetFormat(PGLOBAL g, FORMAT& fmt) {return Value->SetConstFormat(g, fmt);} virtual int CheckSpcCol(PTDB, int) {return 1;} -// void Convert(PGLOBAL g, int newtype); +#if defined(BLK_INDX) + void Convert(PGLOBAL g, int newtype); +#endif // BLK_INDX // bool Rephrase(PGLOBAL g, PSZ work); void SetValue(PVAL vp) {Value = vp;} - virtual bool VerifyColumn(PTBX txp) {return true;} + virtual bool VerifyColumn(PTDB txp) {return true;} virtual bool VerifyTdb(PTDB& tdbp) {return true;} virtual void Print(PGLOBAL g, FILE *, uint); virtual void Print(PGLOBAL g, char *, uint); diff --git a/storage/connect/xtable.h b/storage/connect/xtable.h index 8884860cb5b..79b343e8a6d 100644 --- a/storage/connect/xtable.h +++ b/storage/connect/xtable.h @@ -33,58 +33,20 @@ class CMD : public BLOCK { char *Cmd; }; // end of class CMD -// Filter passed all tables -typedef struct _filter { +// Condition filter structure +typedef struct _cond_filter { char *Body; OPVAL Op; PCMD Cmds; -} FILTER, *PFIL; +} CONDFIL, *PCFIL; typedef class TDBCAT *PTDBCAT; typedef class CATCOL *PCATCOL; -/***********************************************************************/ -/* Definition of class TBX (pure virtual class for TDB and OPJOIN) */ -/***********************************************************************/ -class DllExport TBX: public BLOCK { // Base class for OPJOIN and TDB classes. - public: - // Constructors - TBX(void); - TBX(PTBX txp); - - // Implementation - inline PTBX GetOrig(void) {return To_Orig;} - inline TUSE GetUse(void) {return Use;} - inline void SetUse(TUSE n) {Use = n;} - inline PFIL GetFilter(void) {return To_Filter;} - inline void SetOrig(PTBX txp) {To_Orig = txp;} - inline void SetFilter(PFIL fp) {To_Filter = fp;} - - // Methods - virtual bool IsSame(PTBX tp) {return tp == this;} - virtual int GetTdb_No(void) = 0; // Convenience during conversion - virtual PTDB GetNext(void) = 0; - virtual int Cardinality(PGLOBAL) = 0; - virtual int GetMaxSize(PGLOBAL) = 0; - virtual int GetProgMax(PGLOBAL) = 0; - virtual int GetProgCur(void) = 0; - virtual int GetBadLines(void) {return 0;} - virtual PTBX Copy(PTABS t) = 0; - - protected: -//virtual void PrepareFilters(PGLOBAL g) = 0; - - protected: - // Members - PTBX To_Orig; // Pointer to original if it is a copy - PFIL To_Filter; - TUSE Use; - }; // end of class TBX - /***********************************************************************/ /* Definition of class TDB with all its method functions. */ /***********************************************************************/ -class DllExport TDB: public TBX { // Table Descriptor Block. +class DllExport TDB: public BLOCK { // Table Descriptor Block. public: // Constructors TDB(PTABDEF tdp = NULL); @@ -92,11 +54,21 @@ class DllExport TDB: public TBX { // Table Descriptor Block. // Implementation static void SetTnum(int n) {Tnum = n;} + inline PTDB GetOrig(void) {return To_Orig;} + inline TUSE GetUse(void) {return Use;} + inline PCFIL GetCondFil(void) {return To_CondFil;} inline LPCSTR GetName(void) {return Name;} inline PTABLE GetTable(void) {return To_Table;} inline PCOL GetColumns(void) {return Columns;} inline int GetDegree(void) {return Degree;} inline MODE GetMode(void) {return Mode;} +#if defined(BLK_INDX) + inline PFIL GetFilter(void) {return To_Filter;} + inline void SetFilter(PFIL fp) {To_Filter = fp;} +#endif // BLK_INDX + inline void SetOrig(PTDB txp) {To_Orig = txp;} + inline void SetUse(TUSE n) {Use = n;} + inline void SetCondFil(PCFIL cfp) {To_CondFil = cfp;} inline void SetNext(PTDB tdbp) {Next = tdbp;} inline void SetName(LPCSTR name) {Name = name;} inline void SetTable(PTABLE tablep) {To_Table = tablep;} @@ -105,25 +77,30 @@ class DllExport TDB: public TBX { // Table Descriptor Block. inline void SetMode(MODE mode) {Mode = mode;} //Properties + virtual AMT GetAmType(void) {return TYPE_AM_ERROR;} virtual int GetTdb_No(void) {return Tdb_No;} virtual PTDB GetNext(void) {return Next;} virtual PCATLG GetCat(void) {return NULL;} // Methods - virtual AMT GetAmType(void) {return TYPE_AM_ERROR;} + virtual bool IsSame(PTDB tp) {return tp == this;} virtual bool GetBlockValues(PGLOBAL g) {return false;} virtual int Cardinality(PGLOBAL g) {return (g) ? -1 : 0;} + virtual int GetMaxSize(PGLOBAL) = 0; + virtual int GetProgMax(PGLOBAL) = 0; + virtual int GetProgCur(void) = 0; virtual int RowNumber(PGLOBAL g, bool b = false); virtual bool IsReadOnly(void) {return true;} - virtual const CHARSET_INFO *data_charset() { return NULL; } + virtual const CHARSET_INFO *data_charset() {return NULL;} virtual PTDB Duplicate(PGLOBAL g) {return NULL;} virtual PTDB CopyOne(PTABS t) {return this;} - virtual PTBX Copy(PTABS t); + virtual PTDB Copy(PTABS t); virtual void PrintAM(FILE *f, char *m) {fprintf(f, "%s AM(%d)\n", m, GetAmType());} virtual void Print(PGLOBAL g, FILE *f, uint n); virtual void Print(PGLOBAL g, char *ps, uint z); virtual PSZ GetServer(void) = 0; + virtual int GetBadLines(void) {return 0;} // Database pure virtual routines virtual PCOL ColDB(PGLOBAL g, PSZ name, int num) = 0; @@ -141,6 +118,12 @@ class DllExport TDB: public TBX { // Table Descriptor Block. protected: // Members + PTDB To_Orig; // Pointer to original if it is a copy + TUSE Use; +#if defined(BLK_INDX) + PFIL To_Filter; +#endif // BLK_INDX + PCFIL To_CondFil; // To condition filter structure static int Tnum; // Used to generate Tdb_no's const int Tdb_No; // GetTdb_No() is always 0 for OPJOIN PTDB Next; // Next in linearized queries @@ -194,7 +177,7 @@ class DllExport TDBASE : public TDB { virtual void ResetDB(void) {} virtual void ResetSize(void) {MaxSize = -1;} virtual void RestoreNrec(void) {} - virtual int ResetTableOpt(PGLOBAL g, bool dox); + virtual int ResetTableOpt(PGLOBAL g, bool dop, bool dox); virtual PSZ GetServer(void) {return "Current";} // Database routines From d67ad26b33ea16a3b59215ef967bdd9b89345e04 Mon Sep 17 00:00:00 2001 From: Olivier Bertrand Date: Mon, 10 Mar 2014 18:59:36 +0100 Subject: [PATCH 002/279] - Adding files needed for block indexing added: storage/connect/array.cpp storage/connect/array.h storage/connect/blkfil.cpp storage/connect/blkfil.h storage/connect/filter.cpp storage/connect/filter.h --- storage/connect/array.cpp | 1095 +++++++++++++++++++++++ storage/connect/array.h | 122 +++ storage/connect/blkfil.cpp | 1080 ++++++++++++++++++++++ storage/connect/blkfil.h | 295 ++++++ storage/connect/filter.cpp | 1733 ++++++++++++++++++++++++++++++++++++ storage/connect/filter.h | 172 ++++ 6 files changed, 4497 insertions(+) create mode 100644 storage/connect/array.cpp create mode 100644 storage/connect/array.h create mode 100644 storage/connect/blkfil.cpp create mode 100644 storage/connect/blkfil.h create mode 100644 storage/connect/filter.cpp create mode 100644 storage/connect/filter.h diff --git a/storage/connect/array.cpp b/storage/connect/array.cpp new file mode 100644 index 00000000000..052057ad12b --- /dev/null +++ b/storage/connect/array.cpp @@ -0,0 +1,1095 @@ +/************* Array C++ Functions Source Code File (.CPP) *************/ +/* Name: ARRAY.CPP Version 2.3 */ +/* */ +/* (C) Copyright to the author Olivier BERTRAND 2005-2014 */ +/* */ +/* This file contains the XOBJECT derived class ARRAY functions. */ +/* ARRAY is used for elaborate type of processing, such as sorting */ +/* and dichotomic search (Find). This new version does not use sub */ +/* classes anymore for the different types but relies entirely on the */ +/* functionalities provided by the VALUE and VALBLK classes. */ +/* Currently the only supported types are STRING, SHORT, int, DATE, */ +/* TOKEN, DOUBLE, and Compressed Strings. */ +/***********************************************************************/ + +/***********************************************************************/ +/* Include relevant MariaDB header file. */ +/***********************************************************************/ +#include "my_global.h" +#include "sql_class.h" +//#include "sql_time.h" + +#if defined(WIN32) +//#include +#else // !WIN32 +#include +#endif // !WIN32 + +/***********************************************************************/ +/* Include required application header files */ +/* global.h is header containing all global Plug declarations. */ +/* plgdbsem.h is header containing the DB applic. declarations. */ +/* xobject.h is header containing XOBJECT derived classes declares. */ +/***********************************************************************/ +#include "global.h" +#include "plgdbsem.h" +#include "xtable.h" +#include "array.h" +//#include "select.h" +//#include "query.h" +//#include "token.h" + +/***********************************************************************/ +/* Macro definitions. */ +/***********************************************************************/ +#if defined(_DEBUG) +#define ASSERT(B) assert(B); +#else +#define ASSERT(B) +#endif + +/***********************************************************************/ +/* Static variables. */ +/***********************************************************************/ +extern "C" int trace; + +/***********************************************************************/ +/* DB static external variables. */ +/***********************************************************************/ +extern MBLOCK Nmblk; /* Used to initialize MBLOCK's */ + +/***********************************************************************/ +/* External functions. */ +/***********************************************************************/ +BYTE OpBmp(PGLOBAL g, OPVAL opc); +void EncodeValue(int *lp, char *strp, int n); + +/***********************************************************************/ +/* MakeValueArray: Makes a value array from a value list. */ +/***********************************************************************/ +PARRAY MakeValueArray(PGLOBAL g, PPARM pp) + { + int n, valtyp = 0; + size_t len = 0; + PARRAY par; + PPARM parmp; + + if (!pp) + return NULL; + + /*********************************************************************/ + /* New version with values coming in a list. */ + /*********************************************************************/ + if ((valtyp = pp->Type) != TYPE_STRING) + len = 1; + + if (trace) + htrc("valtyp=%d len=%d\n", valtyp, len); + + /*********************************************************************/ + /* Firstly check the list and count the number of values in it. */ + /*********************************************************************/ + for (n = 0, parmp = pp; parmp; n++, parmp = parmp->Next) + if (parmp->Type != valtyp) { + sprintf(g->Message, MSG(BAD_PARAM_TYPE), "MakeValueArray", parmp->Type); + return NULL; + } else if (valtyp == TYPE_STRING) + len = max(len, strlen((char*)parmp->Value)); + + /*********************************************************************/ + /* Make an array object with one block of the proper size. */ + /*********************************************************************/ + par = new(g) ARRAY(g, valtyp, n, (int)len); + + if (par->GetResultType() == TYPE_ERROR) + return NULL; // Memory allocation error in ARRAY + + /*********************************************************************/ + /* All is right now, fill the array block. */ + /*********************************************************************/ + for (parmp = pp; parmp; parmp = parmp->Next) + switch (valtyp) { + case TYPE_STRING: + par->AddValue(g, (PSZ)parmp->Value); + break; + case TYPE_SHORT: + par->AddValue(g, *(SHORT*)parmp->Value); + break; + case TYPE_INT: + par->AddValue(g, *(int*)parmp->Value); + break; + case TYPE_DOUBLE: + par->AddValue(g, *(double*)parmp->Value); + break; + } // endswitch valtyp + + /*********************************************************************/ + /* Send back resulting array. */ + /*********************************************************************/ + return par; + } // end of MakeValueArray + +/* -------------------------- Class ARRAY ---------------------------- */ + +/***********************************************************************/ +/* ARRAY public constructor. */ +/***********************************************************************/ +ARRAY::ARRAY(PGLOBAL g, int type, int size, int length, int prec) + : CSORT(FALSE) + { + Nval = 0; + Ndif = 0; + Bot = 0; + Top = 0; + Size = size; + Type = type; + Xsize = -1; + Len = 1; + + switch ((Type = type)) { + case TYPE_STRING: + Len = length; + break; + case TYPE_SHORT: + case TYPE_INT: + case TYPE_DOUBLE: + break; +#if 0 + case TYPE_TOKEN: + break; + case TYPE_LIST: + Len = 0; + prec = length; + break; +#endif // 0 + default: // This is illegal an causes an ill formed array building + sprintf(g->Message, MSG(BAD_ARRAY_TYPE), type); + Type = TYPE_ERROR; + return; + } // endswitch type + + Valblk = new(g) MBVALS; + Vblp = Valblk->Allocate(g, Type, Len, prec, Size); + + if (!Valblk->GetMemp() && Type != TYPE_LIST) + // The error message was built by PlgDBalloc + Type = TYPE_ERROR; + else + Value = AllocateValue(g, type, Len, prec, NULL); + + Constant = TRUE; + } // end of ARRAY constructor + +#if 0 +/***********************************************************************/ +/* ARRAY public constructor from a QUERY. */ +/***********************************************************************/ +ARRAY::ARRAY(PGLOBAL g, PQUERY qryp) : CSORT(FALSE) + { + Type = qryp->GetColType(0); + Nval = qryp->GetNblin(); + Ndif = 0; + Bot = 0; + Top = 0; + Size = Nval; + Xsize = -1; + Len = qryp->GetColLength(0); + X = Inf = Sup = 0; + Correlated = FALSE; + + switch (Type) { + case TYPE_STRING: + case TYPE_SHORT: + case TYPE_INT: + case TYPE_DATE: + case TYPE_DOUBLE: +// case TYPE_TOKEN: +// case TYPE_LIST: +// Valblk = qryp->GetCol(0)->Result; +// Vblp = qryp->GetColBlk(0); +// Value = qryp->GetColValue(0); +// break; + default: // This is illegal an causes an ill formed array building + sprintf(g->Message, MSG(BAD_ARRAY_TYPE), Type); + Type = TYPE_ERROR; + } // endswitch type + + if (!Valblk || (!Valblk->GetMemp() && Type != TYPE_LIST)) + // The error message was built by ??? + Type = TYPE_ERROR; + + Constant = TRUE; + } // end of ARRAY constructor + +/***********************************************************************/ +/* ARRAY constructor from a TYPE_LIST subarray. */ +/***********************************************************************/ +ARRAY::ARRAY(PGLOBAL g, PARRAY par, int k) : CSORT(FALSE) + { + int prec; + LSTBLK *lp; + + if (par->Type != TYPE_LIST) { + Type = TYPE_ERROR; + return; + } // endif Type + + lp = (LSTBLK*)par->Vblp; + + Nval = par->Nval; + Ndif = 0; + Bot = 0; + Top = 0; + Size = par->Size; + Xsize = -1; + + Valblk = lp->Mbvk[k]; + Vblp = Valblk->Vblk; + Type = Vblp->GetType(); + Len = (Type == TYPE_STRING) ? Vblp->GetVlen() : 0; + prec = (Type == TYPE_FLOAT) ? 2 : 0; + Value = AllocateValue(g, Type, Len, prec, NULL); + Constant = TRUE; + } // end of ARRAY constructor +#endif // 0 + +/***********************************************************************/ +/* Empty: reset the array for a new use (correlated queries). */ +/* Note: this is temporary as correlated queries will not use arrays */ +/* anymore with future optimized algorithms. */ +/***********************************************************************/ +void ARRAY::Empty(void) + { + assert(Correlated); + Nval = Ndif = 0; + Bot = Top = X = Inf = Sup = 0; + } // end of Empty + +/***********************************************************************/ +/* Add a string element to an array. */ +/***********************************************************************/ +bool ARRAY::AddValue(PGLOBAL g, PSZ strp) + { + if (Type != TYPE_STRING) { + sprintf(g->Message, MSG(ADD_BAD_TYPE), GetTypeName(Type), "CHAR"); + return TRUE; + } // endif Type + + if (trace) + htrc(" adding string(%d): '%s'\n", Nval, strp); + +//Value->SetValue_psz(strp); +//Vblp->SetValue(valp, Nval++); + Vblp->SetValue(strp, Nval++); + return FALSE; + } // end of AddValue + +/***********************************************************************/ +/* Add a SHORT integer element to an array. */ +/***********************************************************************/ +bool ARRAY::AddValue(PGLOBAL g, SHORT n) + { + if (Type != TYPE_SHORT) { + sprintf(g->Message, MSG(ADD_BAD_TYPE), GetTypeName(Type), "SHORT"); + return TRUE; + } // endif Type + + if (trace) + htrc(" adding SHORT(%d): %hd\n", Nval, n); + +//Value->SetValue(n); +//Vblp->SetValue(valp, Nval++); + Vblp->SetValue(n, Nval++); + return FALSE; + } // end of AddValue + +/***********************************************************************/ +/* Add a int integer element to an array. */ +/***********************************************************************/ +bool ARRAY::AddValue(PGLOBAL g, int n) + { + if (Type != TYPE_INT) { + sprintf(g->Message, MSG(ADD_BAD_TYPE), GetTypeName(Type), "INTEGER"); + return TRUE; + } // endif Type + + if (trace) + htrc(" adding int(%d): %d\n", Nval, n); + +//Value->SetValue(n); +//Vblp->SetValue(valp, Nval++); + Vblp->SetValue(n, Nval++); + return FALSE; + } // end of AddValue + +/***********************************************************************/ +/* Add a double float element to an array. */ +/***********************************************************************/ +bool ARRAY::AddValue(PGLOBAL g, double d) + { + if (Type != TYPE_DOUBLE) { + sprintf(g->Message, MSG(ADD_BAD_TYPE), GetTypeName(Type), "DOUBLE"); + return TRUE; + } // endif Type + + if (trace) + htrc(" adding float(%d): %lf\n", Nval, d); + + Value->SetValue(d); + Vblp->SetValue(Value, Nval++); + return FALSE; + } // end of AddValue + +/***********************************************************************/ +/* Add the value of a XOBJECT block to an array. */ +/***********************************************************************/ +bool ARRAY::AddValue(PGLOBAL g, PXOB xp) + { + if (Type != xp->GetResultType()) { + sprintf(g->Message, MSG(ADD_BAD_TYPE), + GetTypeName(xp->GetResultType()), GetTypeName(Type)); + return TRUE; + } // endif Type + + if (trace) + htrc(" adding (%d) from xp=%p\n", Nval, xp); + +//AddValue(xp->GetValue()); + Vblp->SetValue(xp->GetValue(), Nval++); + return FALSE; + } // end of AddValue + +/***********************************************************************/ +/* Add a value to an array. */ +/***********************************************************************/ +bool ARRAY::AddValue(PGLOBAL g, PVAL vp) + { + if (Type != vp->GetType()) { + sprintf(g->Message, MSG(ADD_BAD_TYPE), + GetTypeName(vp->GetType()), GetTypeName(Type)); + return TRUE; + } // endif Type + + if (trace) + htrc(" adding (%d) from vp=%p\n", Nval, vp); + + Vblp->SetValue(vp, Nval++); + return FALSE; + } // end of AddValue + +/***********************************************************************/ +/* Retrieve the nth value of the array. */ +/***********************************************************************/ +void ARRAY::GetNthValue(PVAL valp, int n) + { + valp->SetValue_pvblk(Vblp, n); + } // end of GetNthValue + +#if 0 +/***********************************************************************/ +/* Retrieve the nth subvalue of a list array. */ +/***********************************************************************/ +bool ARRAY::GetSubValue(PGLOBAL g, PVAL valp, int *kp) + { + PVBLK vblp; + + if (Type != TYPE_LIST) { + sprintf(g->Message, MSG(NO_SUB_VAL), Type); + return TRUE; + } // endif Type + + vblp = ((LSTBLK*)Vblp)->Mbvk[kp[0]]->Vblk; + valp->SetValue_pvblk(vblp, kp[1]); + return FALSE; + } // end of GetNthValue +#endif // 0 + +/***********************************************************************/ +/* Return the nth value of a STRING array. */ +/***********************************************************************/ +char *ARRAY::GetStringValue(int n) + { + assert (Type == TYPE_STRING); + return Vblp->GetCharValue(n); + } // end of GetStringValue + +/***********************************************************************/ +/* Find whether a value is in an array. */ +/* Provide a conversion limited to the Value limitation. */ +/***********************************************************************/ +bool ARRAY::Find(PVAL valp) + { + register int n; + PVAL vp; + + if (Type != valp->GetType()) { + Value->SetValue_pval(valp); + vp = Value; + } else + vp = valp; + + Inf = Bot, Sup = Top; + + while (Sup - Inf > 1) { + X = (Inf + Sup) >> 1; + n = Vblp->CompVal(vp, X); + + if (n < 0) + Sup = X; + else if (n > 0) + Inf = X; + else + return TRUE; + + } // endwhile + + return FALSE; + } // end of Find + +/***********************************************************************/ +/* ARRAY: Compare routine for a list of values. */ +/***********************************************************************/ +BYTE ARRAY::Vcompare(PVAL vp, int n) + { + Value->SetValue_pvblk(Vblp, n); + return vp->TestValue(Value); + } // end of Vcompare + +/***********************************************************************/ +/* Test a filter condition on an array depending on operator and mod. */ +/* Modificator values are 1: ANY (or SOME) and 2: ALL. */ +/***********************************************************************/ +bool ARRAY::FilTest(PGLOBAL g, PVAL valp, OPVAL opc, int opm) + { + int i; + PVAL vp; + BYTE bt = OpBmp(g, opc); + int top = Nval - 1; + + if (top < 0) // Array is empty + // Return TRUE for ALL because it means that there are no item that + // does not verify the condition, which is true indeed. + // Return FALSE for ANY because TRUE means that there is at least + // one item that verifies the condition, which is false. + return opm == 2; + + if (valp) { + if (Type != valp->GetType()) { + Value->SetValue_pval(valp); + vp = Value; + } else + vp = valp; + + } else if (opc != OP_EXIST) { + sprintf(g->Message, MSG(MISSING_ARG), opc); + longjmp(g->jumper[g->jump_level], TYPE_ARRAY); + } else // OP_EXIST + return Nval > 0; + + if (opc == OP_IN || (opc == OP_EQ && opm == 1)) + return Find(vp); + else if (opc == OP_NE && opm == 2) + return !Find(vp); + else if (opc == OP_EQ && opm == 2) + return (Ndif == 1) ? !(Vcompare(vp, 0) & bt) : FALSE; + else if (opc == OP_NE && opm == 1) + return (Ndif == 1) ? !(Vcompare(vp, 0) & bt) : TRUE; + + if (Type != TYPE_LIST) { + if (opc == OP_GT || opc == OP_GE) + return !(Vcompare(vp, (opm == 1) ? 0 : top) & bt); + else + return !(Vcompare(vp, (opm == 2) ? 0 : top) & bt); + + } // endif Type + + // Case of TYPE_LIST + if (opm == 2) { + for (i = 0; i < Nval; i++) + if (Vcompare(vp, i) & bt) + return FALSE; + + return TRUE; + } else { // opm == 1 + for (i = 0; i < Nval; i++) + if (!(Vcompare(vp, i) & bt)) + return TRUE; + + return FALSE; + } // endif opm + + } // end of FilTest + +/***********************************************************************/ +/* Test whether this array can be converted to TYPE_SHORT. */ +/* Must be called after the array is sorted. */ +/***********************************************************************/ +bool ARRAY::CanBeShort(void) + { + int* To_Val = (int*)Valblk->GetMemp(); + + if (Type != TYPE_INT || !Ndif) + return FALSE; + + // Because the array is sorted, this is true if all the array + // int values are in the range of SHORT values + return (To_Val[0] >= -32768 && To_Val[Nval-1] < 32768); + } // end of CanBeShort + +/***********************************************************************/ +/* Convert an array to new numeric type k. */ +/* Note: conversion is always made in ascending order from STRING to */ +/* SHORT to int to double so no precision is lost in the conversion. */ +/* One exception is converting from int to SHORT compatible arrays. */ +/***********************************************************************/ +int ARRAY::Convert(PGLOBAL g, int k, PVAL vp) + { + int i, prec = 0; + bool b = FALSE; + PMBV ovblk = Valblk; + PVBLK ovblp = Vblp; + + Type = k; // k is the new type + Valblk = new(g) MBVALS; + + switch (Type) { + case TYPE_DOUBLE: + prec = 2; + case TYPE_SHORT: + case TYPE_INT: + case TYPE_DATE: + Len = 1; + break; + default: + sprintf(g->Message, MSG(BAD_CONV_TYPE), Type); + return TYPE_ERROR; + } // endswitch k + + Size = Nval; + Nval = 0; + Vblp = Valblk->Allocate(g, Type, Len, 0, Size); + + if (!Valblk->GetMemp()) + // The error message was built by PlgDBalloc + return TYPE_ERROR; + else + Value = AllocateValue(g, Type, Len, 0, NULL); + + /*********************************************************************/ + /* Converting STRING to DATE can be done according to date format. */ + /*********************************************************************/ + if (Type == TYPE_DATE && ovblp->GetType() == TYPE_STRING && vp) + if (((DTVAL*)Value)->SetFormat(g, vp)) + return TYPE_ERROR; + else + b = TRUE; // Sort the new array on date internal values + + /*********************************************************************/ + /* Do the actual conversion. */ + /*********************************************************************/ + for (i = 0; i < Size; i++) { + Value->SetValue_pvblk(ovblp, i); + + if (AddValue(g, Value)) + return TYPE_ERROR; + + } // endfor i + + /*********************************************************************/ + /* For sorted arrays, get the initial find values. */ + /*********************************************************************/ + if (b) + Sort(g); + + ovblk->Free(); + return Type; + } // end of Convert + +/***********************************************************************/ +/* ARRAY Save: save value at i (used while rordering). */ +/***********************************************************************/ +void ARRAY::Save(int i) + { + Value->SetValue_pvblk(Vblp, i); + } // end of Save + +/***********************************************************************/ +/* ARRAY Restore: restore value to j (used while rordering). */ +/***********************************************************************/ +void ARRAY::Restore(int j) + { + Vblp->SetValue(Value, j); + } // end of Restore + +/***********************************************************************/ +/* ARRAY Move: move value from k to j (used while rordering). */ +/***********************************************************************/ +void ARRAY::Move(int j, int k) + { + Vblp->Move(k, j); // VALBLK does the opposite !!! + } // end of Move + +/***********************************************************************/ +/* ARRAY: Compare routine for one LIST value (ascending only). */ +/***********************************************************************/ +int ARRAY::Qcompare(int *i1, int *i2) + { + return Vblp->CompVal(*i1, *i2); + } // end of Qcompare + +/***********************************************************************/ +/* Mainly meant to set the character arrays case sensitiveness. */ +/***********************************************************************/ +void ARRAY::SetPrecision(PGLOBAL g, int p) + { + if (Vblp == NULL) { + strcpy(g->Message, MSG(PREC_VBLP_NULL)); + longjmp(g->jumper[g->jump_level], TYPE_ARRAY); + } // endif Vblp + + bool was = Vblp->IsCi(); + + if (was && !p) { + strcpy(g->Message, MSG(BAD_SET_CASE)); + longjmp(g->jumper[g->jump_level], TYPE_ARRAY); + } // endif Vblp + + if (was || !p) + return; + else + Vblp->SetPrec(p); + + if (!was && Type == TYPE_STRING) + // Must be resorted to eliminate duplicate strings + if (Sort(g)) + longjmp(g->jumper[g->jump_level], TYPE_ARRAY); + + } // end of SetPrecision + +/***********************************************************************/ +/* Sort and eliminate distinct values from an array. */ +/* Note: this is done by making a sorted index on distinct values. */ +/* Returns FALSE if Ok or TRUE in case of error. */ +/***********************************************************************/ +bool ARRAY::Sort(PGLOBAL g) + { + int i, j, k; + + // This is to avoid multiply allocating for correlated subqueries + if (Nval > Xsize) { + if (Xsize >= 0) { + // Was already allocated + PlgDBfree(Index); + PlgDBfree(Offset); + } // endif Xsize + + // Prepare non conservative sort with offet values + Index.Size = Nval * sizeof(int); + + if (!PlgDBalloc(g, NULL, Index)) + goto error; + + Offset.Size = (Nval + 1) * sizeof(int); + + if (!PlgDBalloc(g, NULL, Offset)) + goto error; + + Xsize = Nval; + } // endif Nval + + // Call the sort program, it returns the number of distinct values + Ndif = Qsort(g, Nval); + + if (Ndif < 0) + goto error; + + // Use the sort index to reorder the data in storage so it will + // be physically sorted and Index can be removed. + for (i = 0; i < Nval; i++) { + if (Pex[i] == i || Pex[i] == Nval) + // Already placed or already moved + continue; + + Save(i); + + for (j = i;; j = k) { + k = Pex[j]; + Pex[j] = Nval; // Mark position as set + + if (k == i) { + Restore(j); + break; // end of loop + } else + Move(j, k); + + } // endfor j + + } // endfor i + + // Reduce the size of the To_Val array if Ndif < Nval + if (Ndif < Nval) { + for (i = 1; i < Ndif; i++) + if (i != Pof[i]) + break; + + for (; i < Ndif; i++) + Move(i, Pof[i]); + + Nval = Ndif; + } // endif ndif + + if (!Correlated) { + if (Size > Nval) { + Size = Nval; + Valblk->ReAllocate(g, Size); + } // endif Size + + // Index and Offset are not used anymore + PlgDBfree(Index); + PlgDBfree(Offset); + Xsize = -1; + } // endif Correlated + + Bot = -1; // For non optimized search + Top = Ndif; // Find searches the whole array. + return FALSE; + + error: + Nval = Ndif = 0; + Valblk->Free(); + PlgDBfree(Index); + PlgDBfree(Offset); + return TRUE; + } // end of Sort + +/***********************************************************************/ +/* Block filter testing for IN operator on Column/Array operands. */ +/* Here we call Find that returns TRUE if the value is in the array */ +/* with X equal to the index of the found value in the array, or */ +/* FALSE if the value is not in the array with Inf and Sup being the */ +/* indexes of the array values that are immediately below and over */ +/* the not found value. This enables to restrict the array to the */ +/* values that are between the min and max block values and to return */ +/* the indication of whether the Find will be always true, always not */ +/* true or other. */ +/***********************************************************************/ +int ARRAY::BlockTest(PGLOBAL g, int opc, int opm, + void *minp, void *maxp, bool s) + { + bool bin, bax, pin, pax, veq, all = (opm == 2); + + if (Ndif == 0) // Array is empty + // Return TRUE for ALL because it means that there are no item that + // does not verify the condition, which is true indeed. + // Return FALSE for ANY because TRUE means that there is at least + // one item that verifies the condition, which is false. + return (all) ? 2 : -2; + else if (opc == OP_EQ && all && Ndif > 1) + return -2; + else if (opc == OP_NE && !all && Ndif > 1) + return 2; +// else if (Ndif == 1) +// all = FALSE; + + // veq is true when all values in the block are equal + switch (Type) { + case TYPE_STRING: veq = (Vblp->IsCi()) + ? !stricmp((char*)minp, (char*)maxp) + : !strcmp((char*)minp, (char*)maxp); break; + case TYPE_SHORT: veq = *(SHORT*)minp == *(SHORT*)maxp; break; + case TYPE_INT: veq = *(PINT)minp == *(PINT)maxp; break; + case TYPE_DOUBLE: veq = *(double*)minp == *(double*)maxp; break; + default: veq = FALSE; // Error ? + } // endswitch type + + if (!s) + Bot = -1; + + Top = Ndif; // Reset Top at top of list + Value->SetBinValue(maxp); + Top = (bax = Find(Value)) ? X + 1 : Sup; + + if (bax) { + if (opc == OP_EQ) + return (veq) ? 1 : 0; + else if (opc == OP_NE) + return (veq) ? -1 : 0; + + if (X == 0) switch (opc) { + // Max value is equal to min list value + case OP_LE: return 1; break; + case OP_LT: return (veq) ? -1 : 0; break; + case OP_GE: return (veq) ? 1 : 0; break; + case OP_GT: return -1; break; + } // endswitch opc + + pax = (opc == OP_GE) ? (X < Ndif - 1) : TRUE; + } else if (Inf == Bot) { + // Max value is smaller than min list value + return (opc == OP_LT || opc == OP_LE || opc == OP_NE) ? 1 : -1; + } else + pax = (Sup < Ndif); // True if max value is inside the list value + + if (!veq) { + Value->SetBinValue(minp); + bin = Find(Value); + } else + bin = bax; + + Bot = (bin) ? X - 1 : Inf; + + if (bin) { + if (opc == OP_EQ || opc == OP_NE) + return 0; + + if (X == Ndif - 1) switch (opc) { + case OP_GE: return (s) ? 2 : 1; break; + case OP_GT: return (veq) ? -1 : 0; break; + case OP_LE: return (veq) ? 1 : 0; break; + case OP_LT: return (s) ? -2 : -1; break; + } // endswitch opc + + pin = (opc == OP_LE) ? (X > 0) : TRUE; + } else if (Sup == Ndif) { + // Min value is greater than max list value + if (opc == OP_GT || opc == OP_GE || opc == OP_NE) + return (s) ? 2 : 1; + else + return (s) ? -2 : -1; + + } else + pin = (Inf >= 0); // True if min value is inside the list value + + if (Top - Bot <= 1) { + // No list item between min and max value +#if defined(_DEBUG) + assert (!bin && !bax); +#endif + switch (opc) { + case OP_EQ: return -1; break; + case OP_NE: return 1; break; + default: return (all) ? -1 : 1; break; + } // endswitch opc + + } // endif + +#if defined(_DEBUG) + assert (Ndif > 1); // if Ndif = 1 we should have returned already +#endif + + // At this point, if there are no logical errors in the algorithm, + // the only possible overlaps between the array and the block are: + // Array: +-------+ +-------+ +-------+ +-----+ + // Block: +-----+ +---+ +------+ +--------+ + // TRUE: pax pin pax pin + if (all) switch (opc) { + case OP_GT: + case OP_GE: return (pax) ? -1 : 0; break; + case OP_LT: + case OP_LE: return (pin) ? -1 : 0; break; + } // endswitch opc + + return 0; + } // end of BlockTest + +/***********************************************************************/ +/* MakeArrayList: Makes a value list from an SQL IN array (in work). */ +/***********************************************************************/ +PSZ ARRAY::MakeArrayList(PGLOBAL g) + { + char *p, *tp; + int i; + size_t z, len = 2; + + if (Type == TYPE_LIST) + return "(???)"; // To be implemented + + z = max(24, GetTypeSize(Type, Len) + 4); + tp = (char*)PlugSubAlloc(g, NULL, z); + + for (i = 0; i < Nval; i++) { + Value->SetValue_pvblk(Vblp, i); + Value->Print(g, tp, z); + len += strlen(tp); + } // enfor i + + if (trace) + htrc("Arraylist: len=%d\n", len); + + p = (char *)PlugSubAlloc(g, NULL, len); + strcpy(p, "("); + + for (i = 0; i < Nval;) { + Value->SetValue_pvblk(Vblp, i); + Value->Print(g, tp, z); + strcat(p, tp); + strcat(p, (++i == Nval) ? ")" : ","); + } // enfor i + + if (trace) + htrc("Arraylist: newlen=%d\n", strlen(p)); + + return p; + } // end of MakeArrayList + +/***********************************************************************/ +/* Make file output of ARRAY contents. */ +/***********************************************************************/ +void ARRAY::Print(PGLOBAL g, FILE *f, UINT n) + { + char m[64]; + int lim = min(Nval,10); + + memset(m, ' ', n); // Make margin string + m[n] = '\0'; + fprintf(f, "%sARRAY: type=%d\n", m, Type); + memset(m, ' ', n + 2); // Make margin string + m[n] = '\0'; + + if (Type != TYPE_LIST) { + fprintf(f, "%sblock=%p numval=%d\n", m, Valblk->GetMemp(), Nval); + + if (Vblp) + for (int i = 0; i < lim; i++) { + Value->SetValue_pvblk(Vblp, i); + Value->Print(g, f, n+4); + } // endfor i + + } else + fprintf(f, "%sVALLST: numval=%d\n", m, Nval); + + } // end of Print + +/***********************************************************************/ +/* Make string output of ARRAY contents. */ +/***********************************************************************/ +void ARRAY::Print(PGLOBAL g, char *ps, UINT z) + { + if (z < 16) + return; + + sprintf(ps, "ARRAY: type=%d\n", Type); + // More to be implemented later + } // end of Print + +/* -------------------------- Class MULAR ---------------------------- */ + +/***********************************************************************/ +/* MULAR public constructor. */ +/***********************************************************************/ +MULAR::MULAR(PGLOBAL g, int n) : CSORT(FALSE) + { + Narray = n; + Pars = (PARRAY*)PlugSubAlloc(g, NULL, n * sizeof(PARRAY)); + } // end of MULAR constructor + +/***********************************************************************/ +/* MULAR: Compare routine multiple arrays. */ +/***********************************************************************/ +int MULAR::Qcompare(int *i1, int *i2) + { + register int i, n = 0; + + for (i = 0; i < Narray; i++) + if ((n = Pars[i]->Qcompare(i1, i2))) + break; + + return n; + } // end of Qcompare + +/***********************************************************************/ +/* Sort and eliminate distinct values from multiple arrays. */ +/* Note: this is done by making a sorted index on distinct values. */ +/* Returns FALSE if Ok or TRUE in case of error. */ +/***********************************************************************/ +bool MULAR::Sort(PGLOBAL g) + { + int i, j, k, n, nval, ndif; + + // All arrays must have the same number of values + nval = Pars[0]->Nval; + + for (n = 1; n < Narray; n++) + if (Pars[n]->Nval != nval) { + strcpy(g->Message, MSG(BAD_ARRAY_VAL)); + return TRUE; + } // endif nval + + // Prepare non conservative sort with offet values + Index.Size = nval * sizeof(int); + + if (!PlgDBalloc(g, NULL, Index)) + goto error; + + Offset.Size = (nval + 1) * sizeof(int); + + if (!PlgDBalloc(g, NULL, Offset)) + goto error; + + // Call the sort program, it returns the number of distinct values + ndif = Qsort(g, nval); + + if (ndif < 0) + goto error; + + // Use the sort index to reorder the data in storage so it will + // be physically sorted and Index can be removed. + for (i = 0; i < nval; i++) { + if (Pex[i] == i || Pex[i] == nval) + // Already placed or already moved + continue; + + for (n = 0; n < Narray; n++) + Pars[n]->Save(i); + + for (j = i;; j = k) { + k = Pex[j]; + Pex[j] = nval; // Mark position as set + + if (k == i) { + for (n = 0; n < Narray; n++) + Pars[n]->Restore(j); + + break; // end of loop + } else + for (n = 0; n < Narray; n++) + Pars[n]->Move(j, k); + + } // endfor j + + } // endfor i + + // Reduce the size of the To_Val array if ndif < nval + if (ndif < nval) { + for (i = 1; i < ndif; i++) + if (i != Pof[i]) + break; + + for (; i < ndif; i++) + for (n = 0; n < Narray; n++) + Pars[n]->Move(i, Pof[i]); + + for (n = 0; n < Narray; n++) { + Pars[n]->Nval = ndif; + Pars[n]->Size = ndif; + Pars[n]->Valblk->ReAllocate(g, ndif); + } // endfor n + + } // endif ndif + + // Index and Offset are not used anymore + PlgDBfree(Index); + PlgDBfree(Offset); + + for (n = 0; n < Narray; n++) { + Pars[n]->Bot = -1; // For non optimized search + Pars[n]->Top = ndif; // Find searches the whole array. + } // endfor n + + return FALSE; + + error: + PlgDBfree(Index); + PlgDBfree(Offset); + return TRUE; + } // end of Sort diff --git a/storage/connect/array.h b/storage/connect/array.h new file mode 100644 index 00000000000..489a26ac0fd --- /dev/null +++ b/storage/connect/array.h @@ -0,0 +1,122 @@ +/**************** Array H Declares Source Code File (.H) ***************/ +/* Name: ARRAY.H Version 3.1 */ +/* */ +/* (C) Copyright to the author Olivier BERTRAND 2005-2014 */ +/* */ +/* This file contains the ARRAY and VALBASE derived classes declares. */ +/***********************************************************************/ + +/***********************************************************************/ +/* Include required application header files */ +/***********************************************************************/ +#include "xobject.h" +#include "valblk.h" +#include "csort.h" + +typedef class ARRAY *PARRAY; + +/***********************************************************************/ +/* Definition of class ARRAY with all its method functions. */ +/* Note: This is not a general array class that could be defined as */ +/* a template class, but rather a specific object containing a list */ +/* of values to be processed by the filter IN operator. */ +/* In addition it must act as a metaclass by being able to give back */ +/* the type of values it contains. */ +/* It must also be able to convert itself from some type to another. */ +/***********************************************************************/ +class DllExport ARRAY : public XOBJECT, public CSORT { // Array descblock + friend class MULAR; +//friend class VALLST; +//friend class SFROW; + public: + // Constructors + ARRAY(PGLOBAL g, int type, int size, int len = 1, int prec = 0); +//ARRAY(PGLOBAL g, PQUERY qryp); +//ARRAY(PGLOBAL g, PARRAY par, int k); + + // Implementation + virtual int GetType(void) {return TYPE_ARRAY;} + virtual int GetResultType(void) {return Type;} + virtual int GetLength(void) {return Len;} + virtual int GetLengthEx(void) {return Len;} + virtual int GetScale() {return 0;} + int GetNval(void) {return Nval;} + int GetSize(void) {return Size;} +// PVAL GetValp(void) {return Valp;} + void SetType(int atype) {Type = atype;} + void SetCorrel(bool b) {Correlated = b;} + + // Methods + virtual void Reset(void) {Bot = -1;} + virtual int Qcompare(int *, int *); + virtual bool Compare(PXOB) {assert(FALSE); return FALSE;} + virtual bool SetFormat(PGLOBAL, FORMAT&) {assert(FALSE); return FALSE;} + virtual int CheckSpcCol(PTDB, int) {return 0;} + virtual void Print(PGLOBAL g, FILE *f, UINT n); + virtual void Print(PGLOBAL g, char *ps, UINT z); + void Empty(void); + void SetPrecision(PGLOBAL g, int p); + bool AddValue(PGLOBAL g, PSZ sp); + bool AddValue(PGLOBAL g, SHORT n); + bool AddValue(PGLOBAL g, int n); + bool AddValue(PGLOBAL g, double f); + bool AddValue(PGLOBAL g, PXOB xp); + bool AddValue(PGLOBAL g, PVAL vp); + void GetNthValue(PVAL valp, int n); + char *GetStringValue(int n); + BYTE Vcompare(PVAL vp, int n); + void Save(int); + void Restore(int); + void Move(int, int); + bool Sort(PGLOBAL g); + bool Find(PVAL valp); + bool FilTest(PGLOBAL g, PVAL valp, OPVAL opc, int opm); + int Convert(PGLOBAL g, int k, PVAL vp = NULL); + int BlockTest(PGLOBAL g, int opc, int opm, + void *minp, void *maxp, bool s); + PSZ MakeArrayList(PGLOBAL g); + bool CanBeShort(void); + bool GetSubValue(PGLOBAL g, PVAL valp, int *kp); + + protected: + // Members + PMBV Valblk; // To the MBVALS class + PVBLK Vblp; // To Valblock of the data array +//PVAL Valp; // The value used for Save and Restore is Value + int Size; // Size of value array + int Nval; // Total number of items in array + int Ndif; // Total number of distinct items in array + int Xsize; // Size of Index (used for correlated arrays) + int Type; // Type of individual values in the array + int Len; // Length of character string + int Bot; // Bottom of research index + int Top; // Top of research index + int X, Inf, Sup; // Used for block optimization + bool Correlated; // -----------> Temporary + }; // end of class ARRAY + +/***********************************************************************/ +/* Definition of class MULAR with all its method functions. */ +/* This class is used when constructing the arrays of constants used */ +/* for indexing. Its only purpose is to provide a way to sort, reduce */ +/* and reorder the arrays of multicolumn indexes as one block. Indeed */ +/* sorting the arrays independantly would break the correspondance of */ +/* column values. */ +/***********************************************************************/ +class MULAR : public CSORT, public BLOCK { // No need to be an XOBJECT + public: + // Constructor + MULAR(PGLOBAL g, int n); + + // Implementation + void SetPars(PARRAY par, int i) {Pars[i] = par;} + + // Methods + virtual int Qcompare(int *i1, int *i2); // Sort compare routine + bool Sort(PGLOBAL g); + + protected: + // Members + int Narray; // The number of sub-arrays + PARRAY *Pars; // To the block of real arrays + }; // end of class ARRAY diff --git a/storage/connect/blkfil.cpp b/storage/connect/blkfil.cpp new file mode 100644 index 00000000000..d6da60166e8 --- /dev/null +++ b/storage/connect/blkfil.cpp @@ -0,0 +1,1080 @@ +/************* BlkFil C++ Program Source Code File (.CPP) **************/ +/* PROGRAM NAME: BLKFIL */ +/* ------------- */ +/* Version 2.5 */ +/* */ +/* COPYRIGHT: */ +/* ---------- */ +/* (C) Copyright to the author Olivier BERTRAND 2004-2014 */ +/* */ +/* WHAT THIS PROGRAM DOES: */ +/* ----------------------- */ +/* This program is the implementation of block indexing classes. */ +/* */ +/***********************************************************************/ + +/***********************************************************************/ +/* Include relevant MariaDB header file. */ +/***********************************************************************/ +#include "my_global.h" +#include "sql_class.h" +//#include "sql_time.h" + +#if defined(WIN32) +//#include +#else // !WIN32 +#include +#include +#include +#endif // !WIN32 + +/***********************************************************************/ +/* Include application header files: */ +/***********************************************************************/ +#include "global.h" // global declarations +#include "plgdbsem.h" // DB application declarations +#include "xindex.h" // Key Index class declarations +#include "filamtxt.h" // File access method dcls +#include "tabdos.h" // TDBDOS and DOSCOL class dcls +#include "array.h" // ARRAY classes dcls +#include "blkfil.h" // Block Filter classes dcls + +/***********************************************************************/ +/* Static variables. */ +/***********************************************************************/ +extern "C" int trace; + +/* ------------------------ Class BLOCKFILTER ------------------------ */ + +/***********************************************************************/ +/* BLOCKFILTER constructor. */ +/***********************************************************************/ +BLOCKFILTER::BLOCKFILTER(PTDBDOS tdbp, int op) + { + Tdbp = tdbp; + Correl = FALSE; + Opc = op; + Opm = 0; + Result = 0; + } // end of BLOCKFILTER constructor + +/***********************************************************************/ +/* Make file output of BLOCKFILTER contents. */ +/***********************************************************************/ +void BLOCKFILTER::Print(PGLOBAL g, FILE *f, UINT n) + { + char m[64]; + + memset(m, ' ', n); // Make margin string + m[n] = '\0'; + + fprintf(f, "%sBLOCKFILTER: at %p opc=%d opm=%d result=%d\n", + m, this, Opc, Opm, Result); + } // end of Print + +/***********************************************************************/ +/* Make string output of BLOCKFILTER contents. */ +/***********************************************************************/ +void BLOCKFILTER::Print(PGLOBAL g, char *ps, UINT z) + { + strncat(ps, "BlockFilter(s)", z); + } // end of Print + + +/* ---------------------- Class BLKFILLOG ---------------------------- */ + +/***********************************************************************/ +/* BLKFILLOG constructor. */ +/***********************************************************************/ +BLKFILLOG::BLKFILLOG(PTDBDOS tdbp, int op, PBF *bfp, int n) + : BLOCKFILTER(tdbp, op) + { + N = n; + Fil = bfp; + + for (int i = 0; i < N; i++) + if (Fil[i]) + Correl |= Fil[i]->Correl; + + } // end of BLKFILLOG constructor + +/***********************************************************************/ +/* Reset: this function is used only to check the existence of a */ +/* BLKFILIN block and have it reset its Bot value for sorted columns. */ +/***********************************************************************/ +void BLKFILLOG::Reset(PGLOBAL g) + { + for (int i = 0; i < N; i++) + if (Fil[i]) + Fil[i]->Reset(g); + + } // end of Reset + +/***********************************************************************/ +/* This function is used for block filter evaluation. We use here a */ +/* fuzzy logic between the values returned by evaluation blocks: */ +/* -2: the condition will be always false for the rest of the file. */ +/* -1: the condition will be false for the whole group. */ +/* 0: the condition may be true for some of the group values. */ +/* 1: the condition will be true for the whole group. */ +/* 2: the condition will be always true for the rest of the file. */ +/***********************************************************************/ +int BLKFILLOG::BlockEval(PGLOBAL g) + { + int i, rc; + + for (i = 0; i < N; i++) { + // 0: Means some block filter value may be True + rc = (Fil[i]) ? Fil[i]->BlockEval(g) : 0; + + if (!i) + Result = (Opc == OP_NOT) ? -rc : rc; + else switch (Opc) { + case OP_AND: + Result = min(Result, rc); + break; + case OP_OR: + Result = max(Result, rc); + break; + default: + // Should never happen + Result = 0; + return Result; + } // endswitch Opc + + } // endfor i + + return Result; + } // end of BlockEval + +/* ---------------------- Class BLKFILARI----------------------------- */ + +/***********************************************************************/ +/* BLKFILARI constructor. */ +/***********************************************************************/ +BLKFILARI::BLKFILARI(PGLOBAL g, PTDBDOS tdbp, int op, PXOB *xp) + : BLOCKFILTER(tdbp, op) + { + Colp = (PDOSCOL)xp[0]; + + if (xp[1]->GetType() == TYPE_COLBLK) { + Cpx = (PCOL)xp[1]; // Subquery pseudo constant column + Correl = TRUE; + } else + Cpx = NULL; + + Sorted = Colp->IsSorted() > 0; + + // Don't remember why this was changed. Anyway it is no good for + // correlated subqueries because the Value must reflect changes + if (Cpx) + Valp = xp[1]->GetValue(); + else + Valp = AllocateValue(g, xp[1]->GetValue()); + + } // end of BLKFILARI constructor + +/***********************************************************************/ +/* Reset: re-eval the constant value in the case of pseudo constant */ +/* column use in a correlated subquery. */ +/***********************************************************************/ +void BLKFILARI::Reset(PGLOBAL g) + { + if (Cpx) { + Cpx->Reset(); + Cpx->Eval(g); + MakeValueBitmap(); // Does nothing for class BLKFILARI + } // endif Cpx + + } // end of Reset + +/***********************************************************************/ +/* Evaluate block filter for arithmetic operators. */ +/***********************************************************************/ +int BLKFILARI::BlockEval(PGLOBAL g) + { + int mincmp, maxcmp, n; + +#if defined(_DEBUG) + assert (Colp->IsClustered()); +#endif + + n = ((PTDBDOS)Colp->GetTo_Tdb())->GetCurBlk(); + mincmp = Colp->GetMin()->CompVal(Valp, n); + maxcmp = Colp->GetMax()->CompVal(Valp, n); + + switch (Opc) { + case OP_EQ: + case OP_NE: + if (mincmp < 0) // Means minval > Val + Result = (Sorted) ? -2 : -1; + else if (maxcmp > 0) // Means maxval < Val + Result = -1; + else if (!mincmp && !maxcmp) // minval = maxval = val + Result = 1; + else + Result = 0; + + break; + case OP_GT: + case OP_LE: + if (mincmp < 0) // minval > Val + Result = (Sorted) ? 2 : 1; + else if (maxcmp < 0) // maxval > Val + Result = 0; + else // maxval <= Val + Result = -1; + + break; + case OP_GE: + case OP_LT: + if (mincmp <= 0) // minval >= Val + Result = (Sorted) ? 2 : 1; + else if (maxcmp <= 0) // Maxval >= Val + Result = 0; + else // Maxval < Val + Result = -1; + + break; + } // endswitch Opc + + switch (Opc) { + case OP_NE: + case OP_LE: + case OP_LT: + Result = -Result; + break; + } // endswitch Opc + + if (trace) + htrc("BlockEval: op=%d n=%d rc=%d\n", Opc, n, Result); + + return Result; + } // end of BlockEval + +/* ---------------------- Class BLKFILAR2----------------------------- */ + +/***********************************************************************/ +/* BLKFILAR2 constructor. */ +/***********************************************************************/ +BLKFILAR2::BLKFILAR2(PGLOBAL g, PTDBDOS tdbp, int op, PXOB *xp) + : BLKFILARI(g, tdbp, op, xp) + { + MakeValueBitmap(); + } // end of BLKFILAR2 constructor + +/***********************************************************************/ +/* MakeValueBitmap: Set the constant value bit map. It can be void */ +/* if the constant value is not in the column distinct values list. */ +/***********************************************************************/ +void BLKFILAR2::MakeValueBitmap(void) + { + int i; // ndv = Colp->GetNdv(); + bool found = FALSE; + PVBLK dval = Colp->GetDval(); + + assert(dval); + + /*********************************************************************/ + /* Here we cannot use Find because we must get the index */ + /* of where to put the value if it is not found in the array. */ + /* This is needed by operators other than OP_EQ or OP_NE. */ + /*********************************************************************/ + found = dval->Locate(Valp, i); + + /*********************************************************************/ + /* Set the constant value bitmap. The bitmaps are really matching */ + /* the OP_EQ, OP_LE, and OP_LT operator but are also used for the */ + /* other operators for which the Result will be inverted. */ + /* The reason the bitmaps are not directly complemented for them is */ + /* to be able to test easily the cases of sorted columns with Bxp, */ + /* and the case of a void bitmap, which happens if the constant */ + /* value is not in the column distinct values list. */ + /*********************************************************************/ + if (found) { + Bmp = 1 << i; // Bit of the found value + Bxp = Bmp - 1; // All smaller values + + if (Opc != OP_LT && Opc != OP_GE) + Bxp |= Bmp; // Found value must be included + + } else { + Bmp = 0; + Bxp = (1 << i) - 1; + } // endif found + + if (!(Opc == OP_EQ || Opc == OP_NE)) + Bmp = Bxp; + + } // end of MakeValueBitmap + +/***********************************************************************/ +/* Evaluate XDB2 block filter for arithmetic operators. */ +/***********************************************************************/ +int BLKFILAR2::BlockEval(PGLOBAL g) + { +#if defined(_DEBUG) + assert (Colp->IsClustered()); +#endif + + int n = ((PTDBDOS)Colp->GetTo_Tdb())->GetCurBlk(); + ULONG bkmp = *(PULONG)Colp->GetBmap()->GetValPtr(n); + ULONG bres = Bmp & bkmp; + + // Set result as if Opc were OP_EQ, OP_LT, or OP_LE + if (!bres) { + if (!Bmp) + Result = -2; // No good block in the table file + else if (!Sorted) + Result = -1; // No good values in this block + else // Sorted column, test for no more good blocks in file + Result = (Bxp & bkmp) ? -1 : -2; + + } else + // Test whether all block values are good or only some ones + Result = (bres == bkmp) ? 1 : 0; + + // For OP_NE, OP_GE, and OP_GT the result must be inverted. + switch (Opc) { + case OP_NE: + case OP_GE: + case OP_GT: + Result = -Result; + break; + } // endswitch Opc + + if (trace) + htrc("BlockEval2: op=%d n=%d rc=%d\n", Opc, n, Result); + + return Result; + } // end of BlockEval + +/* ---------------------- Class BLKFILMR2----------------------------- */ + +/***********************************************************************/ +/* BLKFILMR2 constructor. */ +/***********************************************************************/ +BLKFILMR2::BLKFILMR2(PGLOBAL g, PTDBDOS tdbp, int op, PXOB *xp) + : BLKFILARI(g, tdbp, op, xp) + { + Nbm = Colp->GetNbm(); + Bmp = (PULONG)PlugSubAlloc(g, NULL, Nbm * sizeof(ULONG)); + Bxp = (PULONG)PlugSubAlloc(g, NULL, Nbm * sizeof(ULONG)); + MakeValueBitmap(); + } // end of BLKFILMR2 constructor + +/***********************************************************************/ +/* MakeValueBitmap: Set the constant value bit map. It can be void */ +/* if the constant value is not in the column distinct values list. */ +/***********************************************************************/ +void BLKFILMR2::MakeValueBitmap(void) + { + int i; // ndv = Colp->GetNdv(); + bool found = FALSE, noteq = !(Opc == OP_EQ || Opc == OP_NE); + PVBLK dval = Colp->GetDval(); + + assert(dval); + + for (i = 0; i < Nbm; i++) + Bmp[i] = Bxp[i] = 0; + + /*********************************************************************/ + /* Here we cannot use Find because we must get the index */ + /* of where to put the value if it is not found in the array. */ + /* This is needed by operators other than OP_EQ or OP_NE. */ + /*********************************************************************/ + found = dval->Locate(Valp, i); + + /*********************************************************************/ + /* For bitmaps larger than a ULONG, we must know where Bmp and Bxp */ + /* are positioned in the ULONG bit map block array. */ + /*********************************************************************/ + N = i / MAXBMP; + i %= MAXBMP; + + /*********************************************************************/ + /* Set the constant value bitmaps. The bitmaps are really matching */ + /* the OP_EQ, OP_LE, and OP_LT operator but are also used for the */ + /* other operators for which the Result will be inverted. */ + /* The reason the bitmaps are not directly complemented for them is */ + /* to be able to easily test the cases of sorted columns with Bxp, */ + /* and the case of a void bitmap, which happens if the constant */ + /* value is not in the column distinct values list. */ + /*********************************************************************/ + if (found) { + Bmp[N] = 1 << i; + Bxp[N] = Bmp[N] - 1; + + if (Opc != OP_LT && Opc != OP_GE) + Bxp[N] |= Bmp[N]; // Found value must be included + + } else + Bxp[N] = (1 << i) - 1; + + if (noteq) + Bmp[N] = Bxp[N]; + + Void = !Bmp[N]; // There are no good values in the file + + for (i = 0; i < N; i++) { + Bxp[i] = ~0; + + if (noteq) + Bmp[i] = Bxp[i]; + + Void = Void && !Bmp[i]; + } // endfor i + + if (!Bmp[N] && !Bxp[N]) + N--; + + } // end of MakeValueBitmap + +/***********************************************************************/ +/* Evaluate XDB2 block filter for arithmetic operators. */ +/***********************************************************************/ +int BLKFILMR2::BlockEval(PGLOBAL g) + { +#if defined(_DEBUG) + assert (Colp->IsClustered()); +#endif + + int i, n = ((PTDBDOS)Colp->GetTo_Tdb())->GetCurBlk(); + bool fnd = FALSE, all = TRUE, gt = TRUE; + ULONG bres; + PULONG bkmp = (PULONG)Colp->GetBmap()->GetValPtr(n * Nbm); + + // Set result as if Opc were OP_EQ, OP_LT, or OP_LE + for (i = 0; i < Nbm; i++) + if (i <= N) { + if ((bres = Bmp[i] & bkmp[i])) + fnd = TRUE; // Some good value(s) found in the block + + if (bres != bkmp[i]) + all = FALSE; // Not all block values are good + + if (Bxp[i] & bkmp[i]) + gt = FALSE; // Not all block values are > good value(s) + + } else if (bkmp[i]) { + all = FALSE; + break; + } // endif's + + if (!fnd) { + if (Void || (gt && Sorted)) + Result = -2; // No (more) good block in file + else + Result = -1; // No good values in this block + + } else + Result = (all) ? 1 : 0; // All block values are good + + // For OP_NE, OP_GE, and OP_GT the result must be inverted. + switch (Opc) { + case OP_NE: + case OP_GE: + case OP_GT: + Result = -Result; + break; + } // endswitch Opc + + if (trace) + htrc("BlockEval2: op=%d n=%d rc=%d\n", Opc, n, Result); + + return Result; + } // end of BlockEval + +/***********************************************************************/ +/* BLKSPCARI constructor. */ +/***********************************************************************/ +BLKSPCARI::BLKSPCARI(PTDBDOS tdbp, int op, PXOB *xp, int bsize) + : BLOCKFILTER(tdbp, op) + { + if (xp[1]->GetType() == TYPE_COLBLK) { + Cpx = (PCOL)xp[1]; // Subquery pseudo constant column + Correl = TRUE; + } else + Cpx = NULL; + + Valp = xp[1]->GetValue(); + Val = (int)xp[1]->GetValue()->GetIntValue(); + Bsize = bsize; + } // end of BLKFILARI constructor + +/***********************************************************************/ +/* Reset: re-eval the constant value in the case of pseudo constant */ +/* column use in a correlated subquery. */ +/***********************************************************************/ +void BLKSPCARI::Reset(PGLOBAL g) + { + if (Cpx) { + Cpx->Reset(); + Cpx->Eval(g); + Val = (int)Valp->GetIntValue(); + } // endif Cpx + + } // end of Reset + +/***********************************************************************/ +/* Evaluate block filter for arithmetic operators (ROWID) */ +/***********************************************************************/ +int BLKSPCARI::BlockEval(PGLOBAL g) + { + int mincmp, maxcmp, n, m; + + n = Tdbp->GetCurBlk(); + m = n * Bsize + 1; // Minimum Rowid value for this block + mincmp = (Val > m) ? 1 : (Val < m) ? (-1) : 0; + m = (n + 1) * Bsize; // Maximum Rowid value for this block + maxcmp = (Val > m) ? 1 : (Val < m) ? (-1) : 0; + + switch (Opc) { + case OP_EQ: + case OP_NE: + if (mincmp < 0) // Means minval > Val + Result = -2; // Always sorted + else if (maxcmp > 0) // Means maxval < Val + Result = -1; + else if (!mincmp && !maxcmp) // minval = maxval = val + Result = 1; + else + Result = 0; + + break; + case OP_GT: + case OP_LE: + if (mincmp < 0) // minval > Val + Result = 2; // Always sorted + else if (maxcmp < 0) // maxval > Val + Result = 0; + else // maxval <= Val + Result = -1; + + break; + case OP_GE: + case OP_LT: + if (mincmp <= 0) // minval >= Val + Result = 2; // Always sorted + else if (maxcmp <= 0) // Maxval >= Val + Result = 0; + else // Maxval < Val + Result = -1; + + break; + } // endswitch Opc + + switch (Opc) { + case OP_NE: + case OP_LE: + case OP_LT: + Result = -Result; + break; + } // endswitch Opc + + if (trace) + htrc("BlockEval: op=%d n=%d rc=%d\n", Opc, n, Result); + + return Result; + } // end of BlockEval + +/* ------------------------ Class BLKFILIN --------------------------- */ + +/***********************************************************************/ +/* BLKFILIN constructor. */ +/***********************************************************************/ +BLKFILIN::BLKFILIN(PGLOBAL g, PTDBDOS tdbp, int op, int opm, PXOB *xp) + : BLOCKFILTER(tdbp, op) + { + if (op == OP_IN) { + Opc = OP_EQ; + Opm = 1; + } else { + Opc = op; + Opm = opm; + } // endif op + + Colp = (PDOSCOL)xp[0]; + Arap = (PARRAY)xp[1]; + Type = Arap->GetResultType(); + + if (Colp->GetResultType() != Type) { + sprintf(g->Message, "BLKFILIN: %s", MSG(VALTYPE_NOMATCH)); + longjmp(g->jumper[g->jump_level], 99); + } else if (Colp->GetValue()->IsCi()) + Arap->SetPrecision(g, 1); // Case insensitive + + Sorted = Colp->IsSorted() > 0; + } // end of BLKFILIN constructor + +/***********************************************************************/ +/* Reset: have the sorted array reset its Bot value to -1 (bottom). */ +/***********************************************************************/ +void BLKFILIN::Reset(PGLOBAL g) + { + Arap->Reset(); +// MakeValueBitmap(); // Does nothing for class BLKFILIN + } // end of Reset + +/***********************************************************************/ +/* Evaluate block filter for a IN operator on a constant array. */ +/* Note: here we need to use the GetValPtrEx function to get a zero */ +/* ended string in case of string argument. This is because the ARRAY */ +/* can have a different width than the char column. */ +/***********************************************************************/ +int BLKFILIN::BlockEval(PGLOBAL g) + { + int n = ((PTDBDOS)Colp->GetTo_Tdb())->GetCurBlk(); + void *minp = Colp->GetMin()->GetValPtrEx(n); + void *maxp = Colp->GetMax()->GetValPtrEx(n); + + Result = Arap->BlockTest(g, Opc, Opm, minp, maxp, Sorted); + return Result; + } // end of BlockEval + +/* ------------------------ Class BLKFILIN2 -------------------------- */ + +/***********************************************************************/ +/* BLKFILIN2 constructor. */ +/* New version that takes care of all operators and modificators. */ +/* It is also ready to handle the case of correlated sub-selects. */ +/***********************************************************************/ +BLKFILIN2::BLKFILIN2(PGLOBAL g, PTDBDOS tdbp, int op, int opm, PXOB *xp) + : BLKFILIN(g, tdbp, op, opm, xp) + { + Nbm = Colp->GetNbm(); + Valp = AllocateValue(g, Colp->GetValue()); + Invert = (Opc == OP_NE || Opc == OP_GE || Opc ==OP_GT); + Bmp = (PULONG)PlugSubAlloc(g, NULL, Nbm * sizeof(ULONG)); + Bxp = (PULONG)PlugSubAlloc(g, NULL, Nbm * sizeof(ULONG)); + MakeValueBitmap(); + } // end of BLKFILIN2 constructor + +/***********************************************************************/ +/* MakeValueBitmap: Set the constant values bit map. It can be void */ +/* if the constant values are not in the column distinct values list. */ +/* The bitmaps are prepared for the EQ, LE, and LT operators and */ +/* takes care of the ALL and ANY modificators. If the operators are */ +/* NE, GE, or GT the modificator is inverted and the result will be. */ +/***********************************************************************/ +void BLKFILIN2::MakeValueBitmap(void) + { + int i, k, n, ndv = Colp->GetNdv(); + bool found, noteq = !(Opc == OP_EQ || Opc == OP_NE); + bool all = (!Invert) ? (Opm == 2) : (Opm != 2); + ULONG btp; + PVBLK dval = Colp->GetDval(); + + N = -1; + + // Take care of special cases + if (!(n = Arap->GetNval())) { + // Return TRUE for ALL because it means that there are no item that + // does not verify the condition, which is true indeed. + // Return FALSE for ANY because TRUE means that there is at least + // one item that verifies the condition, which is false. + Result = (Opm == 2) ? 2 : -2; + return; + } else if (!noteq && all && n > 1) { + // An item cannot be equal to all different values + // or an item is always unequal to any different values + Result = (Opc == OP_EQ) ? -2 : 2; + return; + } // endif's + + for (i = 0; i < Nbm; i++) + Bmp[i] = Bxp[i] = 0; + + for (k = 0; k < n; k++) { + Arap->GetNthValue(Valp, k); + found = dval->Locate(Valp, i); + N = i / MAXBMP; + btp = 1 << (i % MAXBMP); + + if (found) + Bmp[N] |= btp; + + // For LT and LE if ALL the condition applies to the smallest item + // if ANY it applies to the largest item. In the case of EQ we come + // here only if ANY or if n == 1, so it does applies to the largest. + if ((!k && all) || (k == n - 1 && !all)) { + Bxp[N] = btp - 1; + + if (found && Opc != OP_LT && Opc != OP_GE) + Bxp[N] |= btp; // Found value must be included + + } // endif k, opm + + } // endfor k + + if (noteq) + Bmp[N] = Bxp[N]; + + Void = !Bmp[N]; // There are no good values in the file + + for (i = 0; i < N; i++) { + Bxp[i] = ~0; + + if (noteq) { + Bmp[i] = Bxp[i]; + Void = FALSE; + } // endif noteq + + } // endfor i + + if (!Bmp[N] && !Bxp[N]) { + if (--N < 0) + // All array values are smaller than block values + Result = (Invert) ? 2 : -2; + + } else if (N == Nbm - 1 && (signed)Bmp[N] == (1 << (ndv % MAXBMP)) - 1) { + // Condition will be always TRUE or FALSE for the whole file + Result = (Invert) ? -2 : 2; + N = -1; + } // endif's + + } // end of MakeValueBitmap + +/***********************************************************************/ +/* Evaluate block filter for set operators on a constant array. */ +/* Note: here we need to use the GetValPtrEx function to get a zero */ +/* ended string in case of string argument. This is because the ARRAY */ +/* can have a different width than the char column. */ +/***********************************************************************/ +int BLKFILIN2::BlockEval(PGLOBAL g) + { + if (N < 0) + return Result; // Was set in MakeValueBitmap + + int i, n = ((PTDBDOS)Colp->GetTo_Tdb())->GetCurBlk(); + bool fnd = FALSE, all = TRUE, gt = TRUE; + ULONG bres; + PULONG bkmp = (PULONG)Colp->GetBmap()->GetValPtr(n * Nbm); + + // Set result as if Opc were OP_EQ, OP_LT, or OP_LE + // The difference between ALL or ANY was handled in MakeValueBitmap + for (i = 0; i < Nbm; i++) + if (i <= N) { + if ((bres = Bmp[i] & bkmp[i])) + fnd = TRUE; + + if (bres != bkmp[i]) + all = FALSE; + + if (Bxp[i] & bkmp[i]) + gt = FALSE; + + } else if (bkmp[i]) { + all = FALSE; + break; + } // endif's + + if (!fnd) { + if (Void || (Sorted && gt)) + Result = -2; // No more good block in file + else + Result = -1; // No good values in this block + + } else if (all) + Result = 1; // All block values are good + else + Result = 0; // Block contains some good values + + // For OP_NE, OP_GE, and OP_GT the result must be inverted. + switch (Opc) { + case OP_NE: + case OP_GE: + case OP_GT: + Result = -Result; + break; + } // endswitch Opc + + return Result; + } // end of BlockEval + +#if 0 +/***********************************************************************/ +/* BLKFILIN2 constructor. */ +/***********************************************************************/ +BLKFILIN2::BLKFILIN2(PGLOBAL g, PTDBDOS tdbp, int op, int opm, PXOB *xp) + : BLKFILIN(g, tdbp, op, opm, xp) + { + // Currently, bitmap matching is only implemented for the IN operator + if (!(Bitmap = (op == OP_IN || (op == OP_EQ && opm != 2)))) { + Nbm = Colp->GetNbm(); + N = 0; + return; // Revert to standard minmax method + } // endif minmax + + int i, n; + ULONG btp; + PVAL valp = AllocateValue(g, Colp->GetValue()); + PVBLK dval = Colp->GetDval(); + + Nbm = Colp->GetNbm(); + N = -1; + Bmp = (PULONG)PlugSubAlloc(g, NULL, Nbm * sizeof(ULONG)); + Bxp = (PULONG)PlugSubAlloc(g, NULL, Nbm * sizeof(ULONG)); + + for (i = 0; i < Nbm; i++) + Bmp[i] = Bxp[i] = 0; + + for (n = 0; n < Arap->GetNval(); n++) { + Arap->GetNthValue(valp, n); + + if ((i = dval->Find(valp)) >= 0) + Bmp[i / MAXBMP] |= 1 << (i % MAXBMP); + + } // endfor n + + for (i = Nbm - 1; i >= 0; i--) + if (Bmp[i]) { + for (btp = Bmp[i]; btp; btp >>= 1) + Bxp[i] |= btp; + + for (N = i--; i >= 0; i--) + Bxp[i] = ~0; + + break; + } // endif Bmp + + } // end of BLKFILIN2 constructor + +/***********************************************************************/ +/* Evaluate block filter for a IN operator on a constant array. */ +/* Note: here we need to use the GetValPtrEx function to get a zero */ +/* ended string in case of string argument. This is because the ARRAY */ +/* can have a different width than the char column. */ +/***********************************************************************/ +int BLKFILIN2::BlockEval(PGLOBAL g) + { + if (N < 0) + return -2; // IN list contains no good values + + int i, n = ((PTDBDOS)Colp->GetTo_Tdb())->GetCurBlk(); + bool fnd = FALSE, all = TRUE, gt = TRUE; + ULONG bres; + PULONG bkmp = (PULONG)Colp->GetBmap()->GetValPtr(n * Nbm); + + if (Bitmap) { + // For IN operator use the bitmap method + for (i = 0; i < Nbm; i++) + if (i <= N) { + if ((bres = Bmp[i] & bkmp[i])) + fnd = TRUE; + + if (bres != bkmp[i]) + all = FALSE; + + if (Bxp[i] & bkmp[i]) + gt = FALSE; + + } else if (bkmp[i]) { + all = FALSE; + break; + } // endif's + + if (!fnd) { + if (Sorted && gt) + Result = -2; // No more good block in file + else + Result = -1; // No good values in this block + + } else if (all) + Result = 1; // All block values are good + else + Result = 0; // Block contains some good values + + } else { + // For other than IN operators, revert to standard minmax method + int n = 0, ndv = Colp->GetNdv(); + void *minp = NULL; + void *maxp = NULL; + ULONG btp; + PVBLK dval = Colp->GetDval(); + + for (i = 0; i < Nbm; i++) + for (btp = 1; btp && n < ndv; btp <<= 1, n++) + if (btp & bkmp[i]) { + if (!minp) + minp = dval->GetValPtrEx(n); + + maxp = dval->GetValPtrEx(n); + } // endif btp + + Result = Arap->BlockTest(g, Opc, Opm, minp, maxp, Colp->IsSorted()); + } // endif Bitmap + + return Result; + } // end of BlockEval +#endif // 0 + +/* ------------------------ Class BLKSPCIN --------------------------- */ + +/***********************************************************************/ +/* BLKSPCIN constructor. */ +/***********************************************************************/ +BLKSPCIN::BLKSPCIN(PGLOBAL g, PTDBDOS tdbp, int op, int opm, + PXOB *xp, int bsize) + : BLOCKFILTER(tdbp, op) + { + if (op == OP_IN) { + Opc = OP_EQ; + Opm = 1; + } else + Opm = opm; + + Arap = (PARRAY)xp[1]; +#if defined(_DEBUG) + assert (Opm); + assert (Arap->GetResultType() == TYPE_INT); +#endif + Bsize = bsize; + } // end of BLKSPCIN constructor + +/***********************************************************************/ +/* Reset: have the sorted array reset its Bot value to -1 (bottom). */ +/***********************************************************************/ +void BLKSPCIN::Reset(PGLOBAL g) + { + Arap->Reset(); + } // end of Reset + +/***********************************************************************/ +/* Evaluate block filter for a IN operator on a constant array. */ +/***********************************************************************/ +int BLKSPCIN::BlockEval(PGLOBAL g) + { + int n = Tdbp->GetCurBlk(); + int minrow = n * Bsize + 1; // Minimum Rowid value for this block + int maxrow = (n + 1) * Bsize; // Maximum Rowid value for this block + + Result = Arap->BlockTest(g, Opc, Opm, &minrow, &maxrow, TRUE); + return Result; + } // end of BlockEval + +/* ------------------------------------------------------------------- */ + +#if 0 +/***********************************************************************/ +/* Implementation of the BLOCKINDEX class. */ +/***********************************************************************/ +BLOCKINDEX::BLOCKINDEX(PBX nx, PDOSCOL cp, PKXBASE kp) + { + Next = nx; + Tdbp = (cp) ? (PTDBDOS)cp->GetTo_Tdb() : NULL; + Colp = cp; + Kxp = kp; + Type = (cp) ? cp->GetResultType() : TYPE_ERROR; + Sorted = (cp) ? cp->IsSorted() > 0 : FALSE; + Result = 0; + } // end of BLOCKINDEX constructor + +/***********************************************************************/ +/* Reset Bot and Top values of optimized Kindex blocks. */ +/***********************************************************************/ +void BLOCKINDEX::Reset(void) + { + if (Next) + Next->Reset(); + + Kxp->Reset(); + } // end of Reset + +/***********************************************************************/ +/* Evaluate block indexing test. */ +/***********************************************************************/ +int BLOCKINDEX::BlockEval(PGLOBAL g) + { +#if defined(_DEBUG) + assert (Tdbp && Colp); +#endif + int n = Tdbp->GetCurBlk(); + void *minp = Colp->GetMin()->GetValPtr(n); + void *maxp = Colp->GetMax()->GetValPtr(n); + + Result = Kxp->BlockTest(g, minp, maxp, Type, Sorted); + return Result; + } // end of BlockEval + +/***********************************************************************/ +/* Make file output of BLOCKINDEX contents. */ +/***********************************************************************/ +void BLOCKINDEX::Print(PGLOBAL g, FILE *f, UINT n) + { + char m[64]; + + memset(m, ' ', n); // Make margin string + m[n] = '\0'; + + fprintf(f, "%sBLOCKINDEX: at %p next=%p col=%s kxp=%p result=%d\n", + m, this, Next, (Colp) ? Colp->GetName() : "Rowid", Kxp, Result); + + if (Next) + Next->Print(g, f, n); + + } // end of Print + +/***********************************************************************/ +/* Make string output of BLOCKINDEX contents. */ +/***********************************************************************/ +void BLOCKINDEX::Print(PGLOBAL g, char *ps, UINT z) + { + strncat(ps, "BlockIndex(es)", z); + } // end of Print + +/* ------------------------------------------------------------------- */ + +/***********************************************************************/ +/* Implementation of the BLOCKINDX2 class. */ +/***********************************************************************/ +BLOCKINDX2::BLOCKINDX2(PBX nx, PDOSCOL cp, PKXBASE kp) + : BLOCKINDEX(nx, cp, kp) + { + Nbm = Colp->GetNbm(); + Dval = Colp->GetDval(); + Bmap = Colp->GetBmap(); +#if defined(_DEBUG) + assert(Dval && Bmap); +#endif + } // end of BLOCKINDX2 constructor + +/***********************************************************************/ +/* Evaluate block indexing test. */ +/***********************************************************************/ +int BLOCKINDX2::BlockEval(PGLOBAL g) + { + int n = Tdbp->GetCurBlk(); + PUINT bmp = (PUINT)Bmap->GetValPtr(n * Nbm); + + Result = Kxp->BlockTst2(g, Dval, bmp, Nbm, Type, Sorted); + return Result; + } // end of BlockEval + +/* ------------------------------------------------------------------- */ + +/***********************************************************************/ +/* Implementation of the BLKSPCINDX class. */ +/***********************************************************************/ +BLKSPCINDX::BLKSPCINDX(PBX nx, PTDBDOS tp, PKXBASE kp, int bsize) + : BLOCKINDEX(nx, NULL, kp) + { + Tdbp = tp; + Bsize = bsize; + Type = TYPE_INT; + Sorted = TRUE; + } // end of BLKSPCINDX constructor + +/***********************************************************************/ +/* Evaluate block indexing test. */ +/***********************************************************************/ +int BLKSPCINDX::BlockEval(PGLOBAL g) + { + int n = Tdbp->GetCurBlk(); + int minrow = n * Bsize + 1; // Minimum Rowid value for this block + int maxrow = (n + 1) * Bsize; // Maximum Rowid value for this block + + Result = Kxp->BlockTest(g, &minrow, &maxrow, TYPE_INT, TRUE); + return Result; + } // end of BlockEval +#endif // 0 diff --git a/storage/connect/blkfil.h b/storage/connect/blkfil.h new file mode 100644 index 00000000000..a648fc2a422 --- /dev/null +++ b/storage/connect/blkfil.h @@ -0,0 +1,295 @@ +/*************** BlkFil H Declares Source Code File (.H) ***************/ +/* Name: BLKFIL.H Version 2.1 */ +/* */ +/* (C) Copyright to the author Olivier BERTRAND 2004-2010 */ +/* */ +/* This file contains the block optimization related classes declares */ +/***********************************************************************/ +#ifndef __BLKFIL__ +#define __BLKFIL__ + +typedef class BLOCKFILTER *PBF; +typedef class BLOCKINDEX *PBX; + +/***********************************************************************/ +/* Definition of class BLOCKFILTER. */ +/***********************************************************************/ +class DllExport BLOCKFILTER : public BLOCK { /* Block Filter */ + friend class BLKFILLOG; + public: + // Constructors + BLOCKFILTER(PTDBDOS tdbp, int op); + + // Implementation + int GetResult(void) {return Result;} + bool Correlated(void) {return Correl;} + + // Methods + virtual void Reset(PGLOBAL) = 0; + virtual int BlockEval(PGLOBAL) = 0; + virtual void Print(PGLOBAL g, FILE *f, UINT n); + virtual void Print(PGLOBAL g, char *ps, UINT z); + + protected: + BLOCKFILTER(void) {} // Standard constructor not to be used + + // Members + PTDBDOS Tdbp; // Owner TDB + bool Correl; // TRUE for correlated subqueries + int Opc; // Comparison operator + int Opm; // Operator modificator + int Result; // Result from evaluation + }; // end of class BLOCKFILTER + +/***********************************************************************/ +/* Definition of class BLKFILLOG (with Op=OP_AND,OP_OR, or OP_NOT) */ +/***********************************************************************/ +class DllExport BLKFILLOG : public BLOCKFILTER { /* Logical Op Block Filter */ + public: + // Constructors + BLKFILLOG(PTDBDOS tdbp, int op, PBF *bfp, int n); + + // Methods + virtual void Reset(PGLOBAL g); + virtual int BlockEval(PGLOBAL g); + + protected: + BLKFILLOG(void) {} // Standard constructor not to be used + + // Members + PBF *Fil; // Points to Block filter args + int N; + }; // end of class BLKFILLOG + +/***********************************************************************/ +/* Definition of class BLKFILARI (with Op=OP_EQ,NE,GT,GE,LT, or LE) */ +/***********************************************************************/ +class DllExport BLKFILARI : public BLOCKFILTER { /* Arithm. Op Block Filter */ + public: + // Constructors + BLKFILARI(PGLOBAL g, PTDBDOS tdbp, int op, PXOB *xp); + + // Methods + virtual void Reset(PGLOBAL g); + virtual int BlockEval(PGLOBAL g); + virtual void MakeValueBitmap(void) {} + + protected: + BLKFILARI(void) {} // Standard constructor not to be used + + // Members + PDOSCOL Colp; // Points to column argument + PCOL Cpx; // Point to subquery "constant" column + PVAL Valp; // Points to constant argument Value + bool Sorted; // True if the column is sorted + }; // end of class BLKFILARI + +/***********************************************************************/ +/* Definition of class BLKFILAR2 (with Op=OP_EQ,NE,GT,GE,LT, or LE) */ +/***********************************************************************/ +class DllExport BLKFILAR2 : public BLKFILARI { /* Arithm. Op Block Filter */ + public: + // Constructors + BLKFILAR2(PGLOBAL g, PTDBDOS tdbp, int op, PXOB *xp); + + // Methods + virtual int BlockEval(PGLOBAL g); + virtual void MakeValueBitmap(void); + + protected: + BLKFILAR2(void) {} // Standard constructor not to be used + + // Members + ULONG Bmp; // The value bitmap used to test blocks + ULONG Bxp; // Bitmap used when Opc = OP_EQ + }; // end of class BLKFILAR2 + +/***********************************************************************/ +/* Definition of class BLKFILAR2 (with Op=OP_EQ,NE,GT,GE,LT, or LE) */ +/* To be used when the bitmap is an array of ULONG bitmaps; */ +/***********************************************************************/ +class DllExport BLKFILMR2 : public BLKFILARI { /* Arithm. Op Block Filter */ + public: + // Constructors + BLKFILMR2(PGLOBAL g, PTDBDOS tdbp, int op, PXOB *xp); + + // Methods + virtual int BlockEval(PGLOBAL g); + virtual void MakeValueBitmap(void); + + protected: + BLKFILMR2(void) {} // Standard constructor not to be used + + // Members + int Nbm; // The number of ULONG bitmaps + int N; // The position of the leftmost ULONG + bool Void; // True if all file blocks can be skipped + PULONG Bmp; // The values bitmaps used to test blocks + PULONG Bxp; // Bit of values <= max value + }; // end of class BLKFILMR2 + +/***********************************************************************/ +/* Definition of class BLKSPCARI (with Op=OP_EQ,NE,GT,GE,LT, or LE) */ +/***********************************************************************/ +class DllExport BLKSPCARI : public BLOCKFILTER { /* Arithm. Op Block Filter */ + public: + // Constructors + BLKSPCARI(PTDBDOS tdbp, int op, PXOB *xp, int bsize); + + // Methods + virtual void Reset(PGLOBAL g); + virtual int BlockEval(PGLOBAL g); + + protected: + BLKSPCARI(void) {} // Standard constructor not to be used + + // Members + PCOL Cpx; // Point to subquery "constant" column + PVAL Valp; // Points to constant argument Value + int Val; // Constant argument Value + int Bsize; // Table block size + }; // end of class BLKSPCARI + +/***********************************************************************/ +/* Definition of class BLKFILIN (with Op=OP_IN) */ +/***********************************************************************/ +class DllExport BLKFILIN : public BLOCKFILTER { // With array arguments. + public: + // Constructors + BLKFILIN(PGLOBAL g, PTDBDOS tdbp, int op, int opm, PXOB *xp); + + // Methods + virtual void Reset(PGLOBAL g); + virtual int BlockEval(PGLOBAL g); + virtual void MakeValueBitmap(void) {} + + protected: + // Member + PDOSCOL Colp; // Points to column argument + PARRAY Arap; // Points to array argument + bool Sorted; // True if the column is sorted + int Type; // Type of array elements + }; // end of class BLKFILIN + +/***********************************************************************/ +/* Definition of class BLKFILIN2 (with Op=OP_IN) */ +/***********************************************************************/ +class DllExport BLKFILIN2 : public BLKFILIN { // With array arguments. + public: + // Constructors + BLKFILIN2(PGLOBAL g, PTDBDOS tdbp, int op, int opm, PXOB *xp); + + // Methods +//virtual void Reset(PGLOBAL g); + virtual int BlockEval(PGLOBAL g); + virtual void MakeValueBitmap(void); + + protected: + // Member + int Nbm; // The number of ULONG bitmaps + int N; // The position of the leftmost ULONG +//bool Bitmap; // True for IN operator (temporary) + bool Void; // True if all file blocks can be skipped + bool Invert; // True when Result must be inverted + PULONG Bmp; // The values bitmaps used to test blocks + PULONG Bxp; // Bit of values <= max value + PVAL Valp; // Used while building the bitmaps + }; // end of class BLKFILIN2 + +/***********************************************************************/ +/* Definition of class BLKSPCIN (with Op=OP_IN) Special column */ +/***********************************************************************/ +class DllExport BLKSPCIN : public BLOCKFILTER { // With array arguments. + public: + // Constructors + BLKSPCIN(PGLOBAL g, PTDBDOS tdbp, int op, int opm, PXOB *xp, int bsize); + + // Methods + virtual void Reset(PGLOBAL g); + virtual int BlockEval(PGLOBAL g); + + protected: + // Member + PARRAY Arap; // Points to array argument + int Bsize; // Table block size + }; // end of class BLKSPCIN + +// ---------------- Class used in block indexing testing ---------------- + +#if 0 +/***********************************************************************/ +/* Definition of class BLOCKINDEX. */ +/* Used to test the indexing to joined tables when the foreign key is */ +/* a clustered or sorted column. If the table is joined to several */ +/* tables, blocks will be chained together. */ +/***********************************************************************/ +class DllExport BLOCKINDEX : public BLOCK { /* Indexing Test Block */ + public: + // Constructors + BLOCKINDEX(PBX nx, PDOSCOL cp, PKXBASE kp); + + // Implementation + PBX GetNext(void) {return Next;} + + // Methods + void Reset(void); + virtual int BlockEval(PGLOBAL); + virtual void Print(PGLOBAL g, FILE *f, UINT n); + virtual void Print(PGLOBAL g, char *ps, UINT z); + + protected: + BLOCKINDEX(void) {} // Standard constructor not to be used + + // Members + PBX Next; // To next Index Block + PTDBDOS Tdbp; // To table description block + PDOSCOL Colp; // Clustered foreign key + PKXBASE Kxp; // To Kindex of joined table + bool Sorted; // TRUE if column is sorted + int Type; // Col/Index type + int Result; // Result from evaluation + }; // end of class BLOCKINDEX + +/***********************************************************************/ +/* Definition of class BLOCKINDX2. (XDB2) */ +/***********************************************************************/ +class DllExport BLOCKINDX2 : public BLOCKINDEX { /* Indexing Test Block */ + public: + // Constructors + BLOCKINDX2(PBX nx, PDOSCOL cp, PKXBASE kp); + + // Methods + virtual int BlockEval(PGLOBAL); + + protected: + BLOCKINDX2(void) {} // Standard constructor not to be used + + // Members + int Nbm; // The number of ULONG bitmaps + PVBLK Dval; // Array of column distinct values + PVBLK Bmap; // Array of block bitmap values + }; // end of class BLOCKINDX2 + +/***********************************************************************/ +/* Definition of class BLKSPCINDX. */ +/* Used to test the indexing to joined tables when the foreign key is */ +/* the ROWID special column. If the table is joined to several */ +/* tables, blocks will be chained together. */ +/***********************************************************************/ +class DllExport BLKSPCINDX : public BLOCKINDEX { /* Indexing Test Block */ + public: + // Constructors + BLKSPCINDX(PBX nx, PTDBDOS tp, PKXBASE kp, int bsize); + + // Methods + virtual int BlockEval(PGLOBAL); + + protected: + BLKSPCINDX(void) {} // Standard constructor not to be used + + // Members + int Bsize; // Table block size + }; // end of class BLOCKINDEX +#endif // 0 + +#endif // __BLKFIL__ diff --git a/storage/connect/filter.cpp b/storage/connect/filter.cpp new file mode 100644 index 00000000000..c2747d00948 --- /dev/null +++ b/storage/connect/filter.cpp @@ -0,0 +1,1733 @@ +/***************** Filter C++ Class Filter Code (.CPP) *****************/ +/* Name: FILTER.CPP Version 3.9 */ +/* */ +/* (C) Copyright to the author Olivier BERTRAND 1998-2014 */ +/* */ +/* This file contains the class FILTER function code. */ +/***********************************************************************/ + +/***********************************************************************/ +/* Include relevant MariaDB header file. */ +/***********************************************************************/ +#include "my_global.h" +#include "sql_class.h" +//#include "sql_time.h" + +#if defined(WIN32) +//#include +#else // !WIN32 +#include +#include +#include +#endif // !WIN32 + + +/***********************************************************************/ +/* Include required application header files */ +/* global.h is header containing all global Plug declarations. */ +/* plgdbsem.h is header containing the DB applic. declarations. */ +/* xobject.h is header containing the XOBJECT derived classes dcls. */ +/***********************************************************************/ +#include "global.h" +#include "plgdbsem.h" +#include "tabcol.h" +#include "xtable.h" +#include "array.h" +//#include "subquery.h" +#include "filter.h" +//#include "token.h" +//#include "select.h" +#include "xindex.h" + +/***********************************************************************/ +/* Static variables. */ +/***********************************************************************/ +extern "C" int trace; + +/***********************************************************************/ +/* Utility routines. */ +/***********************************************************************/ +void PlugConvertConstant(PGLOBAL, PVOID&, SHORT&); +PVOID PlugCopyDB(PTABS, PVOID, INT); +void NewPointer(PTABS, PVOID, PVOID); +void AddPointer(PTABS, PVOID); + +PPARM MakeParm(PGLOBAL g, PXOB xp) + { + PPARM pp = (PPARM)PlugSubAlloc(g, NULL, sizeof(PARM)); + pp->Type = TYPE_XOBJECT; + pp->Value = xp; + pp->Domain = 0; + pp->Next = NULL; + return pp; + } // end of MakeParm + +/***********************************************************************/ +/* Routines called externally by FILTER function. */ +/***********************************************************************/ +bool PlugEvalLike(PGLOBAL, LPCSTR, LPCSTR, bool); +//bool ReadSubQuery(PGLOBAL, PSUBQ); +//PSUBQ OpenSubQuery(PGLOBAL, PSQL); +void PlugCloseDB(PGLOBAL, PSQL); +BYTE OpBmp(PGLOBAL g, OPVAL opc); +PARRAY MakeValueArray(PGLOBAL g, PPARM pp); + +/***********************************************************************/ +/* Routines called externally by CondFilter. */ +/***********************************************************************/ +PFIL MakeFilter(PGLOBAL g, PFIL fp1, OPVAL vop, PFIL fp2) + { + PFIL filp = new(g) FILTER(g, vop); + + filp->Arg(0) = fp1; + filp->Arg(1) = fp2; + + if (filp->Convert(g, false)) + return NULL; + + return filp; + } // end of MakeFilter + +PFIL MakeFilter(PGLOBAL g, PCOL *colp, POPER pop, PPARM pfirst, bool neg) +{ + PPARM parmp, pp[2]; + PFIL fp1, fp2, filp = NULL; + + if (pop->Val == OP_IN) { + PARRAY par = MakeValueArray(g, pfirst); + + if (par) { + pp[0] = MakeParm(g, colp[0]); + pp[1] = MakeParm(g, par); + fp1 = new(g) FILTER(g, pop, pp); + + if (fp1->Convert(g, false)) + return NULL; + + filp = (neg) ? MakeFilter(g, fp1, OP_NOT, NULL) : fp1; + } // endif par + + } else if (pop->Val == OP_XX) { // BETWEEN + if (pfirst && pfirst->Next) { + pp[0] = MakeParm(g, colp[0]); + pp[1] = pfirst; + fp1 = new(g) FILTER(g, neg ? OP_LT : OP_GE, pp); + + if (fp1->Convert(g, false)) + return NULL; + + pp[1] = pfirst->Next; + fp2 = new(g) FILTER(g, neg ? OP_GT : OP_LE, pp); + + if (fp2->Convert(g, false)) + return NULL; + + filp = MakeFilter(g, fp1, neg ? OP_OR : OP_AND, fp2); + } // endif parmp + + } else { + parmp = pfirst; + + for (int i = 0; i < 2; i++) + if (colp[i]) { + pp[i] = MakeParm(g, colp[i]); + } else { + if (!parmp || parmp->Domain != i) + return NULL; // Logical error, should never happen + + pp[i] = parmp; + parmp = parmp->Next; + } // endif colp + + filp = new(g) FILTER(g, pop, pp); + + if (filp->Convert(g, false)) + return NULL; + + } // endif's Val + + return filp; +} // end of MakeFilter + +/* --------------------------- Class FILTER -------------------------- */ + +/***********************************************************************/ +/* FILTER public constructors. */ +/***********************************************************************/ +FILTER::FILTER(PGLOBAL g, POPER pop, PPARM *tp) + { + Constr(g, pop->Val, pop->Mod, tp); + } // end of FILTER constructor + +FILTER::FILTER(PGLOBAL g, OPVAL opc, PPARM *tp) + { + Constr(g, opc, 0, tp); + } // end of FILTER constructor + +void FILTER::Constr(PGLOBAL g, OPVAL opc, int opm, PPARM *tp) + { + Next = NULL; + Opc = opc; + Opm = opm; + Bt = 0x00; + + for (int i = 0; i < 2; i++) { + Test[i].B_T = TYPE_VOID; + + if (tp && tp[i]) { + PlugConvertConstant(g, tp[i]->Value, tp[i]->Type); +#if defined(_DEBUG) + assert(tp[i]->Type == TYPE_XOBJECT); +#endif + Arg(i) = (PXOB)tp[i]->Value; + } else + Arg(i) = pXVOID; + + Val(i) = NULL; + Test[i].Conv = FALSE; + } // endfor i + + } // end of Constr + +/***********************************************************************/ +/* FILTER copy constructor. */ +/***********************************************************************/ +FILTER::FILTER(PFIL fil1) + { + Next = NULL; + Opc = fil1->Opc; + Opm = fil1->Opm; + Test[0] = fil1->Test[0]; + Test[1] = fil1->Test[1]; + } // end of FILTER copy constructor + +/***********************************************************************/ +/* Linearize: Does the linearization of the filter tree: */ +/* Independent filters (not implied in OR/NOT) will be separated */ +/* from others and filtering operations will be automated by */ +/* making a list of filter operations in polish operation style. */ +/* Returned value points to the first filter of the list, which ends */ +/* with the filter that was pointed by the first call argument, */ +/* except for separators, in which case a loop is needed to find it. */ +/* Note: a loop is used now in all cases (was not for OP_NOT) to be */ +/* able to handle the case of filters whose arguments are already */ +/* linearized, as it is done in LNA semantic routines. Indeed for */ +/* already linearized chains, the first filter is never an OP_AND, */ +/* OP_OR or OP_NOT filter, so this function just returns 'this'. */ +/***********************************************************************/ +PFIL FILTER::Linearize(bool nosep) + { + int i; + PFIL lfp[2], ffp[2] = {NULL,NULL}; + + switch (Opc) { + case OP_NOT: + if (GetArgType(0) == TYPE_FILTER) { + lfp[0] = (PFIL)Arg(0); + ffp[0] = lfp[0]->Linearize(TRUE); + } /* endif */ + + if (!ffp[0]) + return NULL; + + while (lfp[0]->Next) // See Note above + lfp[0] = lfp[0]->Next; + + Arg(0) = lfp[0]; + lfp[0]->Next = this; + break; + case OP_OR: + nosep = TRUE; + case OP_AND: + for (i = 0; i < 2; i++) { + if (GetArgType(i) == TYPE_FILTER) { + lfp[i] = (PFIL)Arg(i); + ffp[i] = lfp[i]->Linearize(nosep); + } /* endif */ + + if (!ffp[i]) + return NULL; + + while (lfp[i]->Next) + lfp[i] = lfp[i]->Next; + + Arg(i) = lfp[i]; + } /* endfor i */ + + if (nosep) { + lfp[0]->Next = ffp[1]; + lfp[1]->Next = this; + } else { + lfp[0]->Next = this; + Opc = OP_SEP; + Arg(1) = pXVOID; + Next = ffp[1]; + } /* endif */ + + break; + default: + ffp[0] = this; + } /* endswitch */ + + return (ffp[0]); + } // end of Linearize + +/***********************************************************************/ +/* Link the fil2 filter chain to the fil1(this) filter chain. */ +/***********************************************************************/ +PFIL FILTER::Link(PGLOBAL g, PFIL fil2) + { + PFIL fil1; + + if (trace) + htrc("Linking filter %p with op=%d... to filter %p with op=%d\n", + this, Opc, fil2, (fil2) ? fil2->Opc : 0); + + for (fil1 = this; fil1->Next; fil1 = fil1->Next) ; + + if (fil1->Opc == OP_SEP) + fil1->Next = fil2; // Separator already exists + else { + // Create a filter separator and insert it between the chains + PFIL filp = new(g) FILTER(g, OP_SEP); + + filp->Arg(0) = fil1; + filp->Next = fil2; + fil1->Next = filp; + } // endelse + + return (this); + } // end of Link + +/***********************************************************************/ +/* Remove eventual last separator from a filter chain. */ +/***********************************************************************/ +PFIL FILTER::RemoveLastSep(void) + { + PFIL filp, gfp = NULL; + + // Find last filter block (filp) and previous one (gfp). + for (filp = this; filp->Next; filp = filp->Next) + gfp = filp; + + // If last filter is a separator, remove it + if (filp->Opc == OP_SEP) + if (gfp) + gfp->Next = NULL; + else + return NULL; // chain is now empty + + return this; + } // end of RemoveLastSep + +/***********************************************************************/ +/* CheckColumn: Checks references to Columns in the filter and change */ +/* them into references to Col Blocks. */ +/* Returns the number of column references or -1 in case of column */ +/* not found and -2 in case of unrecoverable error. */ +/* WHERE filters are called with *aggreg == AGG_NO. */ +/* HAVING filters are called with *aggreg == AGG_ANY. */ +/***********************************************************************/ +int FILTER::CheckColumn(PGLOBAL g, PSQL sqlp, PXOB &p, int &ag) + { + char errmsg[MAX_STR] = ""; + int agg, k, n = 0; + + if (trace) + htrc("FILTER CheckColumn: sqlp=%p ag=%d\n", sqlp, ag); + + switch (Opc) { + case OP_SEP: + case OP_AND: + case OP_OR: + case OP_NOT: + return 0; // This because we are called for a linearized filter + default: + break; + } // endswitch Opc + + // Check all arguments even in case of error for when we are called + // from CheckHaving, where references to an alias raise an error but + // we must have all other arguments to be set. + for (int i = 0; i < 2; i++) { + if (GetArgType(i) == TYPE_FILTER) // Should never happen in + return 0; // current implementation + + agg = ag; + + if ((k = Arg(i)->CheckColumn(g, sqlp, Arg(i), agg)) < -1) { + return k; + } else if (k < 0) { + if (!*errmsg) // Keep first error message + strcpy(errmsg, g->Message); + + } else + n += k; + + } // endfor i + + if (*errmsg) { + strcpy(g->Message, errmsg); + return -1; + } else + return n; + + } // end of CheckColumn + +/***********************************************************************/ +/* RefNum: Find the number of references correlated sub-queries make */ +/* to the columns of the outer query (pointed by sqlp). */ +/***********************************************************************/ +int FILTER::RefNum(PSQL sqlp) + { + int n = 0; + + for (int i = 0; i < 2; i++) + n += Arg(i)->RefNum(sqlp); + + return n; + } // end of RefNum + +#if 0 +/***********************************************************************/ +/* CheckSubQuery: see SUBQUERY::CheckSubQuery for comment. */ +/***********************************************************************/ +PXOB FILTER::CheckSubQuery(PGLOBAL g, PSQL sqlp) + { + switch (Opc) { + case OP_SEP: + case OP_AND: + case OP_OR: + case OP_NOT: + break; + default: + for (int i = 0; i < 2; i++) + if (!(Arg(i) = (PXOB)Arg(i)->CheckSubQuery(g, sqlp))) + return NULL; + + break; + } // endswitch Opc + + return this; + } // end of CheckSubQuery + +/***********************************************************************/ +/* SortJoin: function that places ahead of the list the 'good' groups */ +/* for join filtering. These are groups with only one filter that */ +/* specify equality between two different table columns, at least */ +/* one is a table key column. Doing so the join filter will be in */ +/* general compatible with linearization of the joined table tree. */ +/* This function has been added a further sorting on column indexing. */ +/***********************************************************************/ +PFIL FILTER::SortJoin(PGLOBAL g) + { + int k; + PCOL cp1, cp2; + PTDBASE tp1, tp2; + PFIL fp, filp, gfp, filstart = this, filjoin = NULL, lfp = NULL; + bool join = TRUE, key = TRUE; + + // This routine requires that the chain ends with a separator + // So check for it and eventually add one if necessary + for (filp = this; filp->Next; filp = filp->Next) ; + + if (filp->Opc != OP_SEP) + filp->Next = new(g) FILTER(g, OP_SEP); + + again: + for (k = (key) ? 0 : MAX_MULT_KEY; k <= MAX_MULT_KEY; k++) + for (gfp = NULL, fp = filp = filstart; filp; filp = filp->Next) + switch (filp->Opc) { + case OP_SEP: + if (join) { + // Put this filter group into the join filter group list. + if (!lfp) + filjoin = fp; + else + lfp->Next = fp; + + if (!gfp) + filstart = filp->Next; + else + gfp->Next = filp->Next; + + lfp = filp; // last block of join filter list + } else + gfp = filp; // last block of bad filter list + + join = TRUE; + fp = filp->Next; + break; + case OP_LOJ: + case OP_ROJ: + case OP_DTJ: + join &= TRUE; + break; + case OP_EQ: + if (join && k > 0 // So specific join operators come first + && filp->GetArgType(0) == TYPE_COLBLK + && filp->GetArgType(1) == TYPE_COLBLK) { + cp1 = (PCOL)filp->Arg(0); + cp2 = (PCOL)filp->Arg(1); + tp1 = (PTDBASE)cp1->GetTo_Tdb(); + tp2 = (PTDBASE)cp2->GetTo_Tdb(); + + if (tp1->GetTdb_No() != tp2->GetTdb_No()) { + if (key) + join &= (cp1->GetKey() == k || cp2->GetKey() == k); + else + join &= (tp1->GetColIndex(cp1) || tp2->GetColIndex(cp2)); + + } else + join = FALSE; + + } else + join = FALSE; + + break; + default: + join = FALSE; + } // endswitch filp->Opc + + if (key) { + key = FALSE; + goto again; + } // endif key + + if (filjoin) { + lfp->Next = filstart; + filstart = filjoin; + } // endif filjoin + + // Removing last separator is perhaps unuseful, but it was so + return filstart->RemoveLastSep(); + } // end of SortJoin + +/***********************************************************************/ +/* Check that this filter is a good join filter. */ +/* If so the opj block will be set accordingly. */ +/* opj points to the join block, fprec to the filter block to which */ +/* the rest of the chain must be linked in case of success. */ +/* teq, tek and tk2 indicates the severity of the tests: */ +/* tk2 == TRUE means both columns must be primary keys. */ +/* tc2 == TRUE means both args must be columns (not expression). */ +/* tek == TRUE means at least one column must be a primary key. */ +/* teq == TRUE means the filter operator must be OP_EQ. */ +/* tix == TRUE means at least one column must be a simple index key. */ +/* thx == TRUE means at least one column must be a leading index key. */ +/***********************************************************************/ +bool FILTER::FindJoinFilter(POPJOIN opj, PFIL fprec, bool teq, bool tek, + bool tk2, bool tc2, bool tix, bool thx) + { + if (trace) + htrc("FindJoinFilter: opj=%p fprec=%p tests=(%d,%d,%d,%d)\n", + opj, fprec, teq, tek, tk2, tc2); + + // Firstly check that this filter is an independent filter + // meaning that it is the only one in its own group. + if (Next && Next->Opc != OP_SEP) + return (Opc < 0); + + // Keep only equi-joins and specific joins (Outer and Distinct) + // Normally specific join operators comme first because they have + // been placed first by SortJoin. + if (teq && Opc > OP_EQ) + return FALSE; + + // We have a candidate for join filter, now check that it + // fulfil the requirement about its operands, to point to + // columns of respectively the two TDB's of that join. + int col1 = 0, col2 = 0; + bool key = tk2; + bool idx = FALSE, ihx = FALSE; + PIXDEF pdx; + + for (int i = 0; i < 2; i++) + if (GetArgType(i) == TYPE_COLBLK) { + PCOL colp = (PCOL)Arg(i); + + if (tk2) + key &= (colp->IsKey()); + else + key |= (colp->IsKey()); + + pdx = ((PTDBASE)colp->GetTo_Tdb())->GetColIndex(colp); + idx |= (pdx && pdx->GetNparts() == 1); + ihx |= (pdx != NULL); + + if (colp->VerifyColumn(opj->GetTbx1())) + col1 = i + 1; + else if (colp->VerifyColumn(opj->GetTbx2())) + col2 = i + 1; + + } else if (!tc2 && GetArgType(i) != TYPE_CONST) { + PXOB xp = Arg(i); + + if (xp->VerifyColumn(opj->GetTbx1())) + col1 = i + 1; + else if (xp->VerifyColumn(opj->GetTbx2())) + col2 = i + 1; + + } else + return (Opc < 0); + + if (col1 == 0 || col2 == 0) + return (Opc < 0); + + if (((tek && !key) || (tix && !idx) || (thx && !ihx)) && Opc != OP_DTJ) + return FALSE; + + // This is the join filter, set the join block. + if (col1 == 1) { + opj->SetCol1(Arg(0)); + opj->SetCol2(Arg(1)); + } else { + opj->SetCol1(Arg(1)); + opj->SetCol2(Arg(0)); + + switch (Opc) { +// case OP_GT: Opc = OP_LT; break; +// case OP_LT: Opc = OP_GT; break; +// case OP_GE: Opc = OP_LE; break; +// case OP_LE: Opc = OP_GE; break; + case OP_LOJ: + case OP_ROJ: + case OP_DTJ: + // For expended join operators, the filter must indicate + // the way the join should be done, and not the order of + // appearance of tables in the table list (which is kept + // because tables are sorted in AddTdb). Therefore the + // join is inversed, not the filter. + opj->InverseJoin(); + default: break; + } // endswitch Opc + + } // endif col1 + + if (Opc < 0) { + // For join operators, special processing is needed + int knum = 0; + PFIL fp; + + switch (Opc) { + case OP_LOJ: + opj->SetJtype(JT_LEFT); + knum = opj->GetCol2()->GetKey(); + break; + case OP_ROJ: + opj->SetJtype(JT_RIGHT); + knum = opj->GetCol1()->GetKey(); + break; + case OP_DTJ: + for (knum = 1, fp = this->Next; fp; fp = fp->Next) + if (fp->Opc == OP_DTJ) + knum++; + else if (fp->Opc != OP_SEP) + break; + + opj->SetJtype(JT_DISTINCT); + opj->GetCol2()->SetKey(knum); + break; + default: + break; + } // endswitch Opc + + if (knum > 1) { + // Lets take care of a multiple key join + // We do a minimum of checking here as it will done later + int k = 1; + OPVAL op; + BYTE tmp[sizeof(Test[0])]; + + for (fp = this->Next; k < knum && fp; fp = fp->Next) { + switch (op = fp->Opc) { + case OP_SEP: + continue; + case OP_LOJ: + if (Opc == OP_ROJ) { + op = Opc; + memcpy(tmp, &fp->Test[0], sizeof(Test[0])); + fp->Test[0] = fp->Test[1]; + memcpy(&fp->Test[1], tmp, sizeof(Test[0])); + } // endif Opc + + k++; + break; + case OP_ROJ: + if (Opc == OP_LOJ) { + op = Opc; + memcpy(tmp, &fp->Test[0], sizeof(Test[0])); + fp->Test[0] = fp->Test[1]; + memcpy(&fp->Test[1], tmp, sizeof(Test[0])); + } // endif Opc + + k++; + break; + case OP_DTJ: + if (op == Opc && fp->GetArgType(1) == TYPE_COLBLK) + ((PCOL)fp->Arg(1))->SetKey(knum); + + k++; + break; + default: + break; + } // endswitch op + + if (op != Opc) + return TRUE; + + fp->Opc = OP_EQ; + } // endfor fp + + } // endif k + + Opc = OP_EQ; + } // endif Opc + + // Set the join filter operator + opj->SetOpc(Opc); + + // Now mark the columns involved in the join filter because + // this information will be used by the linearize program. + // Note: this should be replaced in the future by something + // enabling to mark tables as Parent or Child. + opj->GetCol1()->MarkCol(U_J_EXT); + opj->GetCol2()->MarkCol(U_J_EXT); + + // Remove the filter from the filter chain. If the filter is + // not last in the chain, also remove the SEP filter after it. + if (Next) // Next->Opc == OP_SEP + Next = Next->Next; + + if (!fprec) + opj->SetFilter(Next); + else + fprec->Next = Next; + + return FALSE; + } // end of FindJoinFilter + +/***********************************************************************/ +/* CheckHaving: check and process a filter of an HAVING clause. */ +/* Check references to Columns and Functions in the filter. */ +/* All these references can correspond to items existing in the */ +/* SELECT list, else if it is a function, allocate a SELECT block */ +/* to be added to the To_Sel list (non projected blocks). */ +/***********************************************************************/ +bool FILTER::CheckHaving(PGLOBAL g, PSQL sqlp) + { + int agg = AGG_ANY; + PXOB xp; + +//sqlp->SetOk(TRUE); // Ok to look into outer queries for filters + + switch (Opc) { + case OP_SEP: + case OP_AND: + case OP_OR: + case OP_NOT: + return FALSE; + default: + if (CheckColumn(g, sqlp, xp, agg) < -1) + return TRUE; // Unrecovable error + + break; + } // endswitch Opc + + sqlp->SetOk(TRUE); // Ok to look into outer queries for filters + + for (int i = 0; i < 2; i++) + if (!(xp = Arg(i)->SetSelect(g, sqlp, TRUE))) + return TRUE; + else if (xp != Arg(i)) { + Arg(i) = xp; + Val(i) = Arg(i)->GetValue(); + } // endif + + sqlp->SetOk(FALSE); + return FALSE; + } // end of CheckHaving +#endif // 0 + +/***********************************************************************/ +/* Used while building a table index. This function split the filter */ +/* attached to the tdbp table into the local and not local part. */ +/* The local filter is used to restrict the size of the index and the */ +/* not local part remains to be executed later. This has been added */ +/* recently and not only to improve the performance but chiefly to */ +/* avoid loosing rows when processing distinct joins. */ +/* Returns: */ +/* 0: the whole filter is local (both arguments are) */ +/* 1: the whole filter is not local */ +/* 2: the filter was split in local (attached to fp[0]) and */ +/* not local (attached to fp[1]). */ +/***********************************************************************/ +int FILTER::SplitFilter(PFIL *fp) + { + int i, rc[2]; + + if (Opc == OP_AND) { + for (i = 0; i < 2; i++) + rc[i] = ((PFIL)Arg(i))->SplitFilter(fp); + + // Filter first argument should never be split because of the + // algorithm used to de-linearize the filter. + assert(rc[0] != 2); + + if (rc[0] != rc[1]) { + // Splitting to be done + if (rc[1] == 2) { + // 2nd argument already split, add 1st to the proper filter + assert(fp[*rc]); + Arg(1) = fp[*rc]; + Val(1) = fp[*rc]->GetValue(); + fp[*rc] = this; + } else for (i = 0; i < 2; i++) { + // Split the filter arguments + assert(!fp[rc[i]]); + fp[rc[i]] = (PFIL)Arg(i); + } // endfor i + + *rc = 2; + } // endif rc + + } else + *rc = (CheckLocal(NULL)) ? 0 : 1; + + return *rc; + } // end of SplitFilter + +/***********************************************************************/ +/* This function is called when making a Kindex after the filter was */ +/* split in local and nolocal part in the case of many to many joins. */ +/* Indeed the whole filter must be reconstructed to take care of next */ +/* same values when doing the explosive join. In addition, the link */ +/* must be done respecting the way filters are de-linearized, no AND */ +/* filter in the first argument of an AND filter, because this is */ +/* expected to be true if SplitFilter is used again on this filter. */ +/***********************************************************************/ +PFIL FILTER::LinkFilter(PGLOBAL g, PFIL fp2) + { + PFIL fp1, filp, filand = NULL; + + assert(fp2); // Test must be made by caller + + // Find where the new AND filter must be attached + for (fp1 = this; fp1->Opc == OP_AND; fp1 = (PFIL)fp1->Arg(1)) + filand = fp1; + + filp = new(g) FILTER(g, OP_AND); + filp->Arg(0) = fp1; + filp->Val(0) = fp1->GetValue(); + filp->Test[0].B_T = TYPE_INT; + filp->Test[0].Conv = FALSE; + filp->Arg(1) = fp2; + filp->Val(1) = fp2->GetValue(); + filp->Test[1].B_T = TYPE_INT; + filp->Test[1].Conv = FALSE; + filp->Value = AllocateValue(g, TYPE_INT); + + if (filand) { + // filp must be inserted here + filand->Arg(1) = filp; + filand->Val(1) = filp->GetValue(); + filp = this; + } // endif filand + + return filp; + } // end of LinkFilter + +/***********************************************************************/ +/* Checks whether filter contains reference to a previous table that */ +/* is not logically joined to the currently openned table, or whether */ +/* it is a Sub-Select filter. In any case, local is set to FALSE. */ +/* Note: This function is now applied to de-linearized filters. */ +/***********************************************************************/ +bool FILTER::CheckLocal(PTDB tdbp) + { + bool local = TRUE; + + if (trace) { + if (tdbp) + htrc("CheckLocal: filp=%p R%d\n", this, tdbp->GetTdb_No()); + else + htrc("CheckLocal: filp=%p\n", this); + } // endif trace + + for (int i = 0; local && i < 2; i++) + local = Arg(i)->CheckLocal(tdbp); + + if (trace) + htrc("FCL: returning %d\n", local); + + return (local); + } // end of CheckLocal + +/***********************************************************************/ +/* This routine is used to split the filter attached to the tdbp */ +/* table into the local and not local part where "local" means that */ +/* it applies "locally" to the FILEID special column with crit = 2 */ +/* and to the SERVID and/or TABID special columns with crit = 3. */ +/* Returns: */ +/* 0: the whole filter is local (both arguments are) */ +/* 1: the whole filter is not local */ +/* 2: the filter was split in local (attached to fp[0]) and */ +/* not local (attached to fp[1]). */ +/* Note: "Locally" means that the "local" filter can be evaluated */ +/* before opening the table. This implies that the special column be */ +/* compared only with constants and that this filter not to be or'ed */ +/* with a non "local" filter. */ +/***********************************************************************/ +int FILTER::SplitFilter(PFIL *fp, PTDB tp, int crit) + { + int i, rc[2]; + + if (Opc == OP_AND) { + for (i = 0; i < 2; i++) + rc[i] = ((PFIL)Arg(i))->SplitFilter(fp, tp, crit); + + // Filter first argument should never be split because of the + // algorithm used to de-linearize the filter. + assert(rc[0] != 2); + + if (rc[0] != rc[1]) { + // Splitting to be done + if (rc[1] == 2) { + // 2nd argument already split, add 1st to the proper filter + assert(fp[*rc]); + Arg(1) = fp[*rc]; + Val(1) = fp[*rc]->GetValue(); + fp[*rc] = this; + } else for (i = 0; i < 2; i++) { + // Split the filter arguments + assert(!fp[rc[i]]); + fp[rc[i]] = (PFIL)Arg(i); + } // endfor i + + *rc = 2; + } // endif rc + + } else + *rc = (CheckSpcCol(tp, crit) == 1) ? 0 : 1; + + return *rc; + } // end of SplitFilter + +/***********************************************************************/ +/* Checks whether filter contains only references to FILEID, SERVID, */ +/* or TABID with constants or pseudo constants. */ +/***********************************************************************/ +int FILTER::CheckSpcCol(PTDB tdbp, int n) + { + int n1 = Arg(0)->CheckSpcCol(tdbp, n); + int n2 = Arg(1)->CheckSpcCol(tdbp, n); + + return max(n1, n2); + } // end of CheckSpcCol + +/***********************************************************************/ +/* Reset the filter arguments to non evaluated yet. */ +/***********************************************************************/ +void FILTER::Reset(void) + { + for (int i = 0; i < 2; i++) + Arg(i)->Reset(); + + } // end of Reset + +/***********************************************************************/ +/* Init: called when reinitializing a query (Correlated subqueries) */ +/***********************************************************************/ +bool FILTER::Init(PGLOBAL g) + { + for (int i = 0; i < 2; i++) + Arg(i)->Init(g); + + return FALSE; + } // end of Init + +/***********************************************************************/ +/* Convert: does all filter setting and conversions. */ +/* (having = TRUE for Having Clauses, FALSE for Where Clauses) */ +/* Note: hierarchy of types is implied by the ConvertType */ +/* function, currently FLOAT, int, STRING and TOKEN. */ +/* Returns FALSE if successful or TRUE in case of error. */ +/* Note on result type for filters: */ +/* Currently the result type is of TYPE_INT (should be TYPE_BOOL). */ +/* This avoids to introduce a new type and perhaps will permit */ +/* conversions. However the boolean operators will result in a */ +/* boolean int result, meaning that result shall be only 0 or 1 . */ +/***********************************************************************/ +bool FILTER::Convert(PGLOBAL g, bool having) + { + int i, comtype = TYPE_ERROR; + + if (trace) + htrc("converting(?) %s %p opc=%d\n", + (having) ? "having" : "filter", this, Opc); + + for (i = 0; i < 2; i++) { + switch (GetArgType(i)) { + case TYPE_COLBLK: + if (((PCOL)Arg(i))->InitValue(g)) + return TRUE; + + break; + case TYPE_ARRAY: + if ((Opc != OP_IN && !Opm) || i == 0) { + strcpy(g->Message, MSG(BAD_ARRAY_OPER)); + return TRUE; + } // endif + + if (((PARRAY)Arg(i))->Sort(g)) // Sort the array + return TRUE; // Error + + break; + case TYPE_VOID: + if (i == 1) { + Val(0) = Arg(0)->GetValue(); + goto TEST; // Filter has only one argument + } // endif i + + strcpy(g->Message, MSG(VOID_FIRST_ARG)); + return TRUE; + } // endswitch + + if (trace) + htrc("Filter(%d): Arg type=%d\n", i, GetArgType(i)); + + // Set default values + Test[i].B_T = Arg(i)->GetResultType(); + Test[i].Conv = FALSE; + + // Special case of the LIKE operator. + if (Opc == OP_LIKE) { + if (!IsTypeChar((int)Test[i].B_T)) { + sprintf(g->Message, MSG(BAD_TYPE_LIKE), i, Test[i].B_T); + return TRUE; + } // endif + + comtype = TYPE_STRING; + } else { + // Set the common type for both (eventually converted) arguments + int argtyp = Test[i].B_T; + + if (GetArgType(i) == TYPE_CONST && argtyp == TYPE_INT) { + // If possible, downcast the type to smaller types to avoid + // convertion as much as possible. + int n = Arg(i)->GetValue()->GetIntValue(); + + if (n >= INT_MIN8 && n <= INT_MAX8) + argtyp = TYPE_TINY; + else if (n >= INT_MIN16 && n <= INT_MAX16) + argtyp = TYPE_SHORT; + + } else if (GetArgType(i) == TYPE_ARRAY) { + // If possible, downcast int arrays target type to TYPE_SHORT + // to take care of filters written like shortcol in (34,35,36). + if (((PARRAY)Arg(i))->CanBeShort()) + argtyp = TYPE_SHORT; + + } // endif TYPE_CONST + + comtype = ConvertType(comtype, argtyp, CNV_ANY); + } // endif Opc + + if (comtype == TYPE_ERROR) { + strcpy(g->Message, MSG(ILL_FILTER_CONV)); + return TRUE; + } // endif + + if (trace) + htrc(" comtype=%d, B_T(%d)=%d Val(%d)=%p\n", + comtype, i, Test[i].B_T, i, Val(i)); + + } // endfor i + + // Set or allocate the filter argument values and buffers + for (i = 0; i < 2; i++) { + if (trace) + htrc(" conv type %d ? i=%d B_T=%d comtype=%d\n", + GetArgType(i), i, Test[i].B_T, comtype); + + if (Test[i].B_T == comtype) { + // No conversion, set Value to argument Value + Val(i) = Arg(i)->GetValue(); +#if defined(_DEBUG) + assert (Val(i) && Val(i)->GetType() == Test[i].B_T); +#endif + } else { + // Conversion between filter arguments to be done. + // Note that the argument must be converted, not only the + // buffer and buffer type, so GetArgType() returns the new type. + switch (GetArgType(i)) { + case TYPE_CONST: + if (comtype == TYPE_DATE && Test[i].B_T == TYPE_STRING) { + // Convert according to the format of the other argument + Val(i) = AllocateValue(g, comtype, Arg(i)->GetLength()); + + if (((DTVAL*)Val(i))->SetFormat(g, Val(1-i))) + return TRUE; + + Val(i)->SetValue_psz(Arg(i)->GetValue()->GetCharValue()); + } else { + ((PCONST)Arg(i))->Convert(g, comtype); + Val(i) = Arg(i)->GetValue(); + } // endif comtype + + break; + case TYPE_ARRAY: + // Conversion PSZ or int array to int or double FLOAT. + if (((PARRAY)Arg(i))->Convert(g, comtype, Val(i-1)) == TYPE_ERROR) + return TRUE; + + break; + case TYPE_FILTER: + strcpy(g->Message, MSG(UNMATCH_FIL_ARG)); + return TRUE; + default: + // Conversion from Column, Select/Func, Expr, Scalfnc... + // The argument requires conversion during Eval + // A separate Value block must be allocated. + // Note: the test on comtype is to prevent unnecessary + // domain initialization and get the correct length in + // case of Token -> numeric conversion. + Val(i) = AllocateValue(g, comtype, (comtype == TYPE_STRING) + ? Arg(i)->GetLengthEx() : Arg(i)->GetLength()); + + if (comtype == TYPE_DATE && Test[i].B_T == TYPE_STRING) + // Convert according to the format of the other argument + if (((DTVAL*)Val(i))->SetFormat(g, Val(1 - i))) + return TRUE; + + Test[i].Conv = TRUE; + break; + } // endswitch GetType + + Test[i].B_T = comtype; + } // endif comtype + + } // endfor i + + // Last check to be sure all is correct. + if (Test[0].B_T != Test[1].B_T) { + sprintf(g->Message, MSG(BAD_FILTER_CONV), Test[0].B_T, Test[1].B_T); + return TRUE; +//} else if (Test[0].B_T == TYPE_LIST && +// ((LSTVAL*)Val(0))->GetN() != ((LSTVAL*)Val(1))->GetN()) { +// sprintf(g->Message, MSG(ROW_ARGNB_ERR), +// ((LSTVAL*)Val(0))->GetN(), ((LSTVAL*)Val(1))->GetN()); +// return TRUE; + } // endif's B_T + + + TEST: // Test for possible Eval optimization + + if (trace) + htrc("Filp %p op=%d argtypes=(%d,%d)\n", + this, Opc, GetArgType(0), GetArgType(1)); + + // Check whether we have a "simple" filter and in that case + // change its class so an optimized Eval function will be used + if (!Test[0].Conv && !Test[1].Conv) { + if (Opm) switch (Opc) { + case OP_EQ: + case OP_NE: + case OP_GT: + case OP_GE: + case OP_LT: + case OP_LE: + if (GetArgType(1) != TYPE_ARRAY) + break; // On subquery, do standard processing + + // Change the FILTER class to FILTERIN + new(this) FILTERIN; + break; + default: + break; + } // endswitch Opc + + else switch (Opc) { +#if 0 + case OP_EQ: new(this) FILTEREQ; break; + case OP_NE: new(this) FILTERNE; break; + case OP_GT: new(this) FILTERGT; break; + case OP_GE: new(this) FILTERGE; break; + case OP_LT: new(this) FILTERLT; break; + case OP_LE: new(this) FILTERLE; break; +#endif // 0 + case OP_EQ: + case OP_NE: + case OP_GT: + case OP_GE: + case OP_LT: + case OP_LE: new(this) FILTERCMP(g); break; + case OP_AND: new(this) FILTERAND; break; + case OP_OR: new(this) FILTEROR; break; + case OP_NOT: new(this) FILTERNOT; break; + case OP_EXIST: + if (GetArgType(1) == TYPE_VOID) { + // For EXISTS it is the first argument that should be null + Arg(1) = Arg(0); + Arg(0) = pXVOID; + } // endif void + + // pass thru + case OP_IN: + // For IN operator do optimize if operand is an array + if (GetArgType(1) != TYPE_ARRAY) + break; // IN on subquery, do standard processing + + // Change the FILTER class to FILTERIN + new(this) FILTERIN; + break; + default: + break; + } // endswitch Opc + + } // endif Conv + + // The result value (should be TYPE_BOOL ???) + Value = AllocateValue(g, TYPE_INT); + return FALSE; + } // end of Convert + +/***********************************************************************/ +/* Eval: Compute filter result value. */ +/* New algorithm: evaluation is now done from the root for each group */ +/* so Eval is now a recursive process for FILTER operands. */ +/***********************************************************************/ +bool FILTER::Eval(PGLOBAL g) + { + int i; // n = 0; +//PSUBQ subp = NULL; + PARRAY ap = NULL; + PDBUSER dup = PlgGetUser(g); + + if (Opc <= OP_XX) + for (i = 0; i < 2; i++) + // Evaluate the object and eventually convert it. + if (Arg(i)->Eval(g)) + return TRUE; + else if (Test[i].Conv) + Val(i)->SetValue_pval(Arg(i)->GetValue()); + + if (trace) + htrc(" Filter: op=%d type=%d %d B_T=%d %d val=%p %p\n", + Opc, GetArgType(0), GetArgType(1), Test[0].B_T, Test[1].B_T, + Val(0), Val(1)); + + // Main switch on filtering according to operator type. + switch (Opc) { + case OP_EQ: + case OP_NE: + case OP_GT: + case OP_GE: + case OP_LT: + case OP_LE: + if (!Opm) { + // Comparison boolean operators. +#if defined(_DEBUG) + if (Val(0)->GetType() != Val(1)->GetType()) + goto FilterError; +#endif + // Compare the two arguments + // New algorithm to take care of TYPE_LIST + Bt = OpBmp(g, Opc); + Value->SetValue_bool(!(Val(0)->TestValue(Val(1)) & Bt)); + break; + } // endif Opm + + // For modified operators, pass thru + case OP_IN: + case OP_EXIST: + // For IN operations, special processing is done here + switch (GetArgType(1)) { + case TYPE_ARRAY: + ap = (PARRAY)Arg(1); + break; + default: + strcpy(g->Message, MSG(IN_WITHOUT_SUB)); + goto FilterError; + } // endswitch Type + + if (trace) { + htrc(" IN filtering: ap=%p\n", ap); + + if (ap) + htrc(" Array: type=%d size=%d other_type=%d\n", + ap->GetType(), ap->GetSize(), Test[0].B_T); + + } // endif trace + + /*****************************************************************/ + /* Implementation note: The Find function is now able to do a */ + /* conversion but limited to SHORT, int, and FLOAT arrays. */ + /*****************************************************************/ +// Value->SetValue_bool(ap->Find(g, Val(0))); + + if (ap) + Value->SetValue_bool(ap->FilTest(g, Val(0), Opc, Opm)); + + break; + + case OP_LIKE: +#if defined(_DEBUG) + if (!IsTypeChar((int)Test[0].B_T) || !IsTypeChar((int)Test[1].B_T)) + goto FilterError; +#endif + if (Arg(0)->Eval(g)) + return TRUE; + + Value->SetValue_bool(PlugEvalLike(g, Val(0)->GetCharValue(), + Val(1)->GetCharValue(), + Val(0)->IsCi())); + break; + + case OP_AND: +#if defined(_DEBUG) + if (Test[0].B_T != TYPE_INT || Test[1].B_T != TYPE_INT) + goto FilterError; +#endif + + if (Arg(0)->Eval(g)) + return TRUE; + + Value->SetValue(Val(0)->GetIntValue()); + + if (!Value->GetIntValue()) + return FALSE; // No need to evaluate 2nd argument + + if (Arg(1)->Eval(g)) + return TRUE; + + Value->SetValue(Val(1)->GetIntValue()); + break; + + case OP_OR: +#if defined(_DEBUG) + if (Test[0].B_T != TYPE_INT || Test[1].B_T != TYPE_INT) + goto FilterError; +#endif + + if (Arg(0)->Eval(g)) + return TRUE; + + Value->SetValue(Val(0)->GetIntValue()); + + if (Value->GetIntValue()) + return FALSE; // No need to evaluate 2nd argument + + if (Arg(1)->Eval(g)) + return TRUE; + + Value->SetValue(Val(1)->GetIntValue()); + break; + + case OP_NOT: +#if defined(_DEBUG) + if (Test[0].B_T != TYPE_INT) // Should be type bool ??? + goto FilterError; +#endif + + if (Arg(0)->Eval(g)) + return TRUE; + + Value->SetValue_bool(!Val(0)->GetIntValue()); + break; + + case OP_SEP: // No more used while evaluating + default: + goto FilterError; + } // endswitch Opc + + if (trace) + htrc("Eval: filter %p Opc=%d result=%d\n", + this, Opc, Value->GetIntValue()); + + return FALSE; + + FilterError: + sprintf(g->Message, MSG(BAD_FILTER), + Opc, Test[0].B_T, Test[1].B_T, GetArgType(0), GetArgType(1)); + return TRUE; + } // end of Eval + +#if 0 +/***********************************************************************/ +/* Called by PlugCopyDB to make a copy of a (linearized) filter chain.*/ +/***********************************************************************/ +PFIL FILTER::Copy(PTABS t) + { + int i; + PFIL fil1, fil2, newfilchain = NULL, fprec = NULL; + + for (fil1 = this; fil1; fil1 = fil1->Next) { + fil2 = new(t->G) FILTER(fil1); + + if (!fprec) + newfilchain = fil2; + else + fprec->Next = fil2; + + NewPointer(t, fil1, fil2); + + for (i = 0; i < 2; i++) + if (fil1->GetArgType(i) == TYPE_COLBLK || + fil1->GetArgType(i) == TYPE_FILTER) + AddPointer(t, &fil2->Arg(i)); + + fprec = fil2; + } /* endfor fil1 */ + + return newfilchain; + } // end of Copy +#endif // 0 + +/*********************************************************************/ +/* Make file output of FILTER contents. */ +/*********************************************************************/ +void FILTER::Print(PGLOBAL g, FILE *f, UINT n) + { + char m[64]; + + memset(m, ' ', n); // Make margin string + m[n] = '\0'; + + bool lin = (Next != NULL); // lin == TRUE if linearized + + for (PFIL fp = this; fp; fp = fp->Next) { + fprintf(f, "%sFILTER: at %p opc=%d lin=%d result=%d\n", + m, fp, fp->Opc, lin, + (Value) ? Value->GetIntValue() : 0); + + for (int i = 0; i < 2; i++) { + fprintf(f, "%s Arg(%d) type=%d value=%p B_T=%d val=%p\n", + m, i, fp->GetArgType(i), fp->Arg(i), + fp->Test[i].B_T, fp->Val(i)); + + if (lin && fp->GetArgType(i) == TYPE_FILTER) + fprintf(f, "%s Filter at %p\n", m, fp->Arg(i)); + else + fp->Arg(i)->Print(g, f, n + 2); + + } // endfor i + + } // endfor fp + + } // end of Print + +/***********************************************************************/ +/* Make string output of TABLE contents (z should be checked). */ +/***********************************************************************/ +void FILTER::Print(PGLOBAL g, char *ps, UINT z) + { + #define FLEN 100 + + typedef struct _bc { + struct _bc *Next; + char Cold[FLEN+1]; + } BC, *PBC; + + char *p; + int n; + PFIL fp; + PBC bxp, bcp = NULL; + + *ps = '\0'; + + for (fp = this; fp && z > 0; fp = fp->Next) { + if (fp->Opc < OP_CNC || fp->Opc == OP_IN || fp->Opc == OP_NULL + || fp->Opc == OP_LIKE || fp->Opc == OP_EXIST) { + if (!(bxp = new BC)) { + strncat(ps, "Filter(s)", z); + return; + } /* endif */ + + bxp->Next = bcp; + bcp = bxp; + p = bcp->Cold; + n = FLEN; + fp->Arg(0)->Print(g, p, n); + n = FLEN - strlen(p); + + switch (fp->Opc) { + case OP_EQ: + strncat(bcp->Cold, "=", n); + break; + case OP_NE: + strncat(bcp->Cold, "!=", n); + break; + case OP_GT: + strncat(bcp->Cold, ">", n); + break; + case OP_GE: + strncat(bcp->Cold, ">=", n); + break; + case OP_LT: + strncat(bcp->Cold, "<", n); + break; + case OP_LE: + strncat(bcp->Cold, "<=", n); + break; + case OP_IN: + strncat(bcp->Cold, " in ", n); + break; + case OP_NULL: + strncat(bcp->Cold, " is null", n); + break; + case OP_LIKE: + strncat(bcp->Cold, " like ", n); + break; + case OP_EXIST: + strncat(bcp->Cold, " exists ", n); + break; + case OP_AND: + strncat(bcp->Cold, " and ", n); + break; + case OP_OR: + strncat(bcp->Cold, " or ", n); + break; + default: + strncat(bcp->Cold, "?", n); + } // endswitch Opc + + n = FLEN - strlen(p); + p += strlen(p); + fp->Arg(1)->Print(g, p, n); + } else + if (!bcp) { + strncat(ps, "???", z); + z -= 3; + } else + switch (fp->Opc) { + case OP_SEP: // Filter list separator + strncat(ps, bcp->Cold, z); + z -= strlen(bcp->Cold); + strncat(ps, ";", z--); + bxp = bcp->Next; + delete bcp; + bcp = bxp; + break; + case OP_NOT: // Filter NOT operator + for (n = min((int)strlen(bcp->Cold), FLEN-3); n >= 0; n--) + bcp->Cold[n+2] = bcp->Cold[n]; + bcp->Cold[0] = '^'; + bcp->Cold[1] = '('; + strcat(bcp->Cold, ")"); + break; + default: + for (n = min((int)strlen(bcp->Cold), FLEN-4); n >= 0; n--) + bcp->Cold[n+3] = bcp->Cold[n]; + bcp->Cold[0] = ')'; + switch (fp->Opc) { + case OP_AND: bcp->Cold[1] = '&'; break; + case OP_OR: bcp->Cold[1] = '|'; break; + default: bcp->Cold[1] = '?'; + } // endswitch + bcp->Cold[2] = '('; + strcat(bcp->Cold, ")"); + bxp = bcp->Next; + for (n = min((int)strlen(bxp->Cold), FLEN-1); n >= 0; n--) + bxp->Cold[n+1] = bxp->Cold[n]; + bxp->Cold[0] = '('; + strncat(bxp->Cold, bcp->Cold, FLEN-strlen(bxp->Cold)); + delete bcp; + bcp = bxp; + } // endswitch + + } // endfor fp + + n = 0; + + if (!bcp) + strncat(ps, "Null-Filter", z); + else do { + if (z > 0) { + if (n++ > 0) { + strncat(ps, "*?*", z); + z = max(0, (int)z-3); + } // endif + strncat(ps, bcp->Cold, z); + z -= strlen(bcp->Cold); + } // endif + + bxp = bcp->Next; + delete bcp; + bcp = bxp; + } while (bcp); // enddo + + } // end of Print + + +/* -------------------- Derived Classes Functions -------------------- */ + +/***********************************************************************/ +/* FILTERCMP constructor. */ +/***********************************************************************/ +FILTERCMP::FILTERCMP(PGLOBAL g) + { + Bt = OpBmp(g, Opc); + } // end of FILTERCMP constructor + +/***********************************************************************/ +/* Eval: Compute result value for comparison operators. */ +/***********************************************************************/ +bool FILTERCMP::Eval(PGLOBAL g) + { + if (Arg(0)->Eval(g) || Arg(1)->Eval(g)) + return TRUE; + + Value->SetValue_bool(!(Val(0)->TestValue(Val(1)) & Bt)); + return FALSE; + } // end of Eval + +/***********************************************************************/ +/* Eval: Compute result value for AND filters. */ +/***********************************************************************/ +bool FILTERAND::Eval(PGLOBAL g) + { + if (Arg(0)->Eval(g)) + return TRUE; + + Value->SetValue(Val(0)->GetIntValue()); + + if (!Value->GetIntValue()) + return FALSE; // No need to evaluate 2nd argument + + if (Arg(1)->Eval(g)) + return TRUE; + + Value->SetValue(Val(1)->GetIntValue()); + return FALSE; + } // end of Eval + +/***********************************************************************/ +/* Eval: Compute result value for OR filters. */ +/***********************************************************************/ +bool FILTEROR::Eval(PGLOBAL g) + { + if (Arg(0)->Eval(g)) + return TRUE; + + Value->SetValue(Val(0)->GetIntValue()); + + if (Value->GetIntValue()) + return FALSE; // No need to evaluate 2nd argument + + if (Arg(1)->Eval(g)) + return TRUE; + + Value->SetValue(Val(1)->GetIntValue()); + return FALSE; + } // end of Eval + +/***********************************************************************/ +/* Eval: Compute result value for NOT filters. */ +/***********************************************************************/ +bool FILTERNOT::Eval(PGLOBAL g) + { + if (Arg(0)->Eval(g)) + return TRUE; + + Value->SetValue_bool(!Val(0)->GetIntValue()); + return FALSE; + } // end of Eval + +/***********************************************************************/ +/* Eval: Compute result value for IN filters. */ +/***********************************************************************/ +bool FILTERIN::Eval(PGLOBAL g) + { + if (Arg(0)->Eval(g)) + return TRUE; + + Value->SetValue_bool(((PARRAY)Arg(1))->FilTest(g, Val(0), Opc, Opm)); + return FALSE; + } // end of Eval + +/***********************************************************************/ +/* FILTERTRUE does nothing and returns TRUE. */ +/***********************************************************************/ +void FILTERTRUE::Reset(void) + { + } // end of Reset + +bool FILTERTRUE::Eval(PGLOBAL) + { + return FALSE; + } // end of Eval + +/* ------------------------- Friend Functions ------------------------ */ + +#if 0 +/***********************************************************************/ +/* Prepare: prepare a filter for execution. This implies two things: */ +/* 1) de-linearize the filter to be able to evaluate it recursively. */ +/* This permit to conditionally evaluate only the first argument */ +/* of OP_OR and OP_AND filters without having to pass by an */ +/* intermediate Apply function (as this has a performance cost). */ +/* 2) do all the necessary conversion for all filter block arguments. */ +/***********************************************************************/ +PFIL PrepareFilter(PGLOBAL g, PFIL fp, bool having) + { + PFIL filp = NULL; + + if (trace) + htrc("PrepareFilter: fp=%p having=%d\n", fp, having); +//if (fp) +// fp->Print(g, debug, 0); + + while (fp) { + if (fp->Opc == OP_SEP) + // If separator is not last transform it into an AND filter + if (fp->Next) { + filp = PrepareFilter(g, fp->Next, having); + fp->Arg(1) = filp; + fp->Opc = OP_AND; + fp->Next = NULL; // This will end the loop + } else + break; // Remove eventual ending separator(s) + +// if (fp->Convert(g, having)) +// longjmp(g->jumper[g->jump_level], TYPE_FILTER); + + filp = fp; + fp = fp->Next; + filp->Next = NULL; + } // endwhile + + if (trace) + htrc(" returning filp=%p\n", filp); +//if (filp) +// filp->Print(g, debug, 0); + + return filp; + } // end of PrepareFilter +#endif // 0 + +/***********************************************************************/ +/* ApplyFilter: Apply filtering for a table (where or having clause). */ +/* New algorithm: evaluate from the root a de-linearized filter so */ +/* AND/OR clauses can be optimized throughout the whole tree. */ +/***********************************************************************/ +DllExport bool ApplyFilter(PGLOBAL g, PFIL filp, PTDB tdbp) + { + if (!filp) + return TRUE; + + // Must be done for null tables + filp->Reset(); + +//if (tdbp && tdbp->IsNull()) +// return TRUE; + + if (filp->Eval(g)) + longjmp(g->jumper[g->jump_level], TYPE_FILTER); + + if (trace) + htrc("PlugFilter filp=%p result=%d\n", + filp, filp->GetResult()); + + return filp->GetResult(); + } // end of ApplyFilter diff --git a/storage/connect/filter.h b/storage/connect/filter.h new file mode 100644 index 00000000000..a24ca18dc59 --- /dev/null +++ b/storage/connect/filter.h @@ -0,0 +1,172 @@ +/*************** Filter H Declares Source Code File (.H) ***************/ +/* Name: FILTER.H Version 1.2 */ +/* */ +/* (C) Copyright to the author Olivier BERTRAND 2010-2012 */ +/* */ +/* This file contains the FILTER and derived classes declares. */ +/***********************************************************************/ + +/***********************************************************************/ +/* Include required application header files */ +/***********************************************************************/ +#include "xobject.h" + +/***********************************************************************/ +/* Utilities for WHERE condition building. */ +/***********************************************************************/ +PFIL MakeFilter(PGLOBAL g, PFIL filp, OPVAL vop, PFIL fp); +PFIL MakeFilter(PGLOBAL g, PCOL *colp, POPER pop, PPARM pfirst, bool neg); + +/***********************************************************************/ +/* Definition of class FILTER with all its method functions. */ +/* Note: Most virtual implementation functions are not in use yet */ +/* but could be in future system evolution. */ +/***********************************************************************/ +class DllExport FILTER : public XOBJECT { /* Filter description block */ +//friend PFIL PrepareFilter(PGLOBAL, PFIL, bool); + friend DllExport bool ApplyFilter(PGLOBAL, PFIL, PTDB = NULL); + public: + // Constructors + FILTER(PGLOBAL g, POPER pop, PPARM *tp = NULL); + FILTER(PGLOBAL g, OPVAL opc, PPARM *tp = NULL); + FILTER(PFIL fil1); + + // Implementation + virtual int GetType(void) {return TYPE_FILTER;} + virtual int GetResultType(void) {return TYPE_INT;} + virtual int GetLength(void) {return 1;} + virtual int GetLengthEx(void) {assert(FALSE); return 0;} + virtual int GetScale() {return 0;}; + PFIL GetNext(void) {return Next;} + OPVAL GetOpc(void) {return Opc;} + int GetOpm(void) {return Opm;} + int GetArgType(int i) {return Arg(i)->GetType();} + bool GetResult(void) {return Value->GetIntValue() != 0;} + PXOB &Arg(int i) {return Test[i].Arg;} + PVAL &Val(int i) {return Test[i].Value;} + bool &Conv(int i) {return Test[i].Conv;} + void SetNext(PFIL filp) {Next = filp;} + + // Methods + virtual void Reset(void); + virtual bool Compare(PXOB) {return FALSE;} // Not used yet + virtual bool Init(PGLOBAL); + virtual bool Eval(PGLOBAL); + virtual bool SetFormat(PGLOBAL, FORMAT&) {return TRUE;} // NUY + virtual int CheckColumn(PGLOBAL g, PSQL sqlp, PXOB &xp, int &ag); + virtual int RefNum(PSQL); + virtual PXOB SetSelect(PGLOBAL, PSQL, bool) {return NULL;} // NUY +//virtual PXOB CheckSubQuery(PGLOBAL, PSQL); + virtual bool CheckLocal(PTDB); + virtual int CheckSpcCol(PTDB tdbp, int n); + virtual void Print(PGLOBAL g, FILE *f, UINT n); + virtual void Print(PGLOBAL g, char *ps, UINT z); + PFIL Linearize(bool nosep); + PFIL Link(PGLOBAL g, PFIL fil2); + PFIL RemoveLastSep(void); +// PFIL SortJoin(PGLOBAL g); +// bool FindJoinFilter(POPJOIN opj, PFIL fprec, bool teq, +// bool tek, bool tk2, bool tc2, bool tix, bool thx); +// bool CheckHaving(PGLOBAL g, PSQL sqlp); + bool Convert(PGLOBAL g, bool having); + int SplitFilter(PFIL *fp); + int SplitFilter(PFIL *fp, PTDB tp, int n); + PFIL LinkFilter(PGLOBAL g, PFIL fp2); +// PFIL Copy(PTABS t); + + protected: + FILTER(void) {} // Standard constructor not to be used + void Constr(PGLOBAL g, OPVAL opc, int opm, PPARM *tp); + + // Members + PFIL Next; // Used for linearization + OPVAL Opc; // Comparison operator + int Opm; // Modificator + BYTE Bt; // Operator bitmap + struct { + int B_T; // Buffer type + PXOB Arg; // Points to argument + PVAL Value; // Points to argument value + bool Conv; // TRUE if argument must be converted + } Test[2]; + }; // end of class FILTER + +/***********************************************************************/ +/* Derived class FILTERX: used to replace a filter by a derived class */ +/* using an Eval method optimizing the filtering evaluation. */ +/* Note: this works only if the members of the derived class are the */ +/* same than the ones of the original class (NO added members). */ +/***********************************************************************/ +class FILTERX : public FILTER { + public: + // Methods + virtual bool Eval(PGLOBAL) = 0; // just to prevent direct FILTERX use + + // Fake operator new used to change a filter into a derived filter + void * operator new(size_t size, PFIL filp) {return filp;} +#if !defined(__BORLANDC__) + // Avoid warning C4291 by defining a matching dummy delete operator + void operator delete(void *, PFIL) {} +#endif + }; // end of class FILTERX + +/***********************************************************************/ +/* Derived class FILTEREQ: OP_EQ, no conversion and Xobject args. */ +/***********************************************************************/ +class FILTERCMP : public FILTERX { + public: + // Constructor + FILTERCMP(PGLOBAL g); + + // Methods + virtual bool Eval(PGLOBAL); + }; // end of class FILTEREQ + +/***********************************************************************/ +/* Derived class FILTERAND: OP_AND, no conversion and Xobject args. */ +/***********************************************************************/ +class FILTERAND : public FILTERX { + public: + // Methods + virtual bool Eval(PGLOBAL); + }; // end of class FILTERAND + +/***********************************************************************/ +/* Derived class FILTEROR: OP_OR, no conversion and Xobject args. */ +/***********************************************************************/ +class FILTEROR : public FILTERX { + public: + // Methods + virtual bool Eval(PGLOBAL); + }; // end of class FILTEROR + +/***********************************************************************/ +/* Derived class FILTERNOT: OP_NOT, no conversion and Xobject args. */ +/***********************************************************************/ +class FILTERNOT : public FILTERX { + public: + // Methods + virtual bool Eval(PGLOBAL); + }; // end of class FILTERNOT + +/***********************************************************************/ +/* Derived class FILTERIN: OP_IN, no conversion and Array 2nd arg. */ +/***********************************************************************/ +class FILTERIN : public FILTERX { + public: + // Methods + virtual bool Eval(PGLOBAL); + }; // end of class FILTERIN + +/***********************************************************************/ +/* Derived class FILTERTRUE: Always returns TRUE. */ +/***********************************************************************/ +class FILTERTRUE : public FILTERX { + public: + // Constructor + FILTERTRUE(PVAL valp) {Value = valp; Value->SetValue_bool(TRUE);} + + // Methods + virtual void Reset(void); + virtual bool Eval(PGLOBAL); + }; // end of class FILTERTRUE From f26be8cae28a6c947278080d45d546ac2b9fa69f Mon Sep 17 00:00:00 2001 From: Olivier Bertrand Date: Sun, 23 Mar 2014 18:49:19 +0100 Subject: [PATCH 003/279] - Work in progress modified: storage/connect/filter.h storage/connect/ha_connect.cc storage/connect/ha_connect.h storage/connect/mysql-test/connect/r/alter.result storage/connect/mysql-test/connect/r/xml.result --- storage/connect/filter.h | 4 ++++ storage/connect/ha_connect.cc | 14 +++++++++++++- storage/connect/ha_connect.h | 15 ++++++++++++++- .../connect/mysql-test/connect/r/alter.result | 16 ++++++++-------- storage/connect/mysql-test/connect/r/xml.result | 2 +- 5 files changed, 40 insertions(+), 11 deletions(-) diff --git a/storage/connect/filter.h b/storage/connect/filter.h index 364c0260ae1..301b3a128de 100644 --- a/storage/connect/filter.h +++ b/storage/connect/filter.h @@ -5,6 +5,8 @@ /* */ /* This file contains the FILTER and derived classes declares. */ /***********************************************************************/ +#ifndef __FILTER__ +#define __FILTER__ /***********************************************************************/ /* Include required application header files */ @@ -170,3 +172,5 @@ class FILTERTRUE : public FILTERX { virtual void Reset(void); virtual bool Eval(PGLOBAL); }; // end of class FILTERTRUE + +#endif // __FILTER__ diff --git a/storage/connect/ha_connect.cc b/storage/connect/ha_connect.cc index 086086553a3..d557f8e6bce 100644 --- a/storage/connect/ha_connect.cc +++ b/storage/connect/ha_connect.cc @@ -159,7 +159,7 @@ /***********************************************************************/ //efine CONNECT_INI "connect.ini" extern "C" { - char version[]= "Version 1.02.0001 February 03, 2014"; + char version[]= "Version 1.02.0002 March 16, 2014"; #if defined(XMSG) char msglang[]; // Default message language @@ -258,6 +258,18 @@ ha_create_table_option connect_field_option_list[]= HA_FOPTION_END }; +/* + CREATE TABLE option list (index options) + + These can be specified in the CREATE TABLE per index: + CREATE TABLE ( field ..., .., INDEX .... *here*, ... ) +*/ +ha_create_table_option connect_index_option_list[]= +{ + HA_IOPTION_BOOL("DYN", kindx, 0), + HA_IOPTION_BOOL("MAPPED", mapped, 0), +}; + /***********************************************************************/ /* Push G->Message as a MySQL warning. */ /***********************************************************************/ diff --git a/storage/connect/ha_connect.h b/storage/connect/ha_connect.h index 1924d8de185..d8395335edb 100644 --- a/storage/connect/ha_connect.h +++ b/storage/connect/ha_connect.h @@ -130,6 +130,19 @@ struct ha_field_option_struct char *special; }; +/* + index options can be declared similarly + using the ha_index_option_struct structure. + + Their values can be specified in the CREATE TABLE per index: + CREATE TABLE ( field ..., .., INDEX .... *here*, ... ) +*/ +struct ha_index_option_struct +{ + bool kindx; + bool mapped; +}; + /** @brief CONNECT_SHARE is a structure that will be shared among all open handlers. This example implements the minimum of what you will probably need. @@ -210,7 +223,7 @@ public: The name of the index type that will be used for display. Don't implement this method unless you really have indexes. */ - const char *index_type(uint inx) { return "XPLUG"; } + const char *index_type(uint inx) { return "XINDEX"; } /** @brief The file extensions. diff --git a/storage/connect/mysql-test/connect/r/alter.result b/storage/connect/mysql-test/connect/r/alter.result index beef3dcdb18..1994ec335f2 100644 --- a/storage/connect/mysql-test/connect/r/alter.result +++ b/storage/connect/mysql-test/connect/r/alter.result @@ -21,8 +21,8 @@ DROP INDEX xd ON t1; ALTER TABLE t1 ADD INDEX xc (c), ADD INDEX xd (d); SHOW INDEX FROM t1; Table Non_unique Key_name Seq_in_index Column_name Collation Cardinality Sub_part Packed Null Index_type Comment Index_comment -t1 1 xc 1 c A NULL NULL NULL XPLUG -t1 1 xd 1 d A NULL NULL NULL XPLUG +t1 1 xc 1 c A NULL NULL NULL XINDEX +t1 1 xd 1 d A NULL NULL NULL XINDEX ALTER TABLE t1 DROP INDEX xc, DROP INDEX xd; SHOW INDEX FROM t1; Table Non_unique Key_name Seq_in_index Column_name Collation Cardinality Sub_part Packed Null Index_type Comment Index_comment @@ -95,7 +95,7 @@ Warning 1105 This is an outward table, table data were not modified. SELECT * FROM t2; line - + c @@ -131,7 +131,7 @@ t1 CREATE TABLE `t1` ( SELECT * FROM t2; line - + 1 @@ -182,8 +182,8 @@ t1 CREATE TABLE `t1` ( ) ENGINE=CONNECT DEFAULT CHARSET=latin1 `TABLE_TYPE`=DBF SHOW INDEX FROM t1; Table Non_unique Key_name Seq_in_index Column_name Collation Cardinality Sub_part Packed Null Index_type Comment Index_comment -t1 1 xc 1 c A NULL NULL NULL XPLUG -t1 1 xd 1 d A NULL NULL NULL XPLUG +t1 1 xc 1 c A NULL NULL NULL XINDEX +t1 1 xd 1 d A NULL NULL NULL XINDEX SELECT * FROM t1; c d 1 One @@ -214,8 +214,8 @@ line ALTER TABLE t1 ADD INDEX xc (c), ADD INDEX xd (d); SHOW INDEX FROM t1; Table Non_unique Key_name Seq_in_index Column_name Collation Cardinality Sub_part Packed Null Index_type Comment Index_comment -t1 1 xc 1 c A NULL NULL NULL XPLUG -t1 1 xd 1 d A NULL NULL NULL XPLUG +t1 1 xc 1 c A NULL NULL NULL XINDEX +t1 1 xd 1 d A NULL NULL NULL XINDEX SELECT d FROM t1 WHERE c = 2; d Two diff --git a/storage/connect/mysql-test/connect/r/xml.result b/storage/connect/mysql-test/connect/r/xml.result index aeb1f5edbcd..4768573dc7c 100644 --- a/storage/connect/mysql-test/connect/r/xml.result +++ b/storage/connect/mysql-test/connect/r/xml.result @@ -413,7 +413,7 @@ DROP TABLE t1; SET @a=LOAD_FILE('MYSQLD_DATADIR/test/t1.xml'); SELECT CAST(@a AS CHAR CHARACTER SET latin1); CAST(@a AS CHAR CHARACTER SET latin1) - + ÀÁÂÃ From 0e20f02174099e0ee79fd5440c84fae9d2796b86 Mon Sep 17 00:00:00 2001 From: Olivier Bertrand Date: Sat, 26 Apr 2014 00:17:26 +0200 Subject: [PATCH 004/279] - Implement dynamic indexing modified: storage/connect/connect.cc storage/connect/filter.cpp storage/connect/filter.h storage/connect/ha_connect.cc storage/connect/ha_connect.h storage/connect/tabdos.cpp storage/connect/tabdos.h storage/connect/table.cpp storage/connect/xindex.cpp storage/connect/xindex.h storage/connect/xtable.h --- storage/connect/connect.cc | 41 +++++++++++++---- storage/connect/filter.cpp | 2 +- storage/connect/filter.h | 2 +- storage/connect/ha_connect.cc | 42 +++++++++++++++--- storage/connect/ha_connect.h | 3 ++ storage/connect/tabdos.cpp | 84 +++++++++++++++++++++++++++++++++-- storage/connect/tabdos.h | 1 + storage/connect/table.cpp | 6 +++ storage/connect/xindex.cpp | 25 +++++++---- storage/connect/xindex.h | 5 +++ storage/connect/xtable.h | 4 +- 11 files changed, 187 insertions(+), 28 deletions(-) diff --git a/storage/connect/connect.cc b/storage/connect/connect.cc index 1433a5c6ce4..f7bba541dfa 100644 --- a/storage/connect/connect.cc +++ b/storage/connect/connect.cc @@ -651,6 +651,14 @@ int CntIndexInit(PGLOBAL g, PTDB ptdb, int id) return 0; } // endif xdp + if (xdp->IsDynamic()) { + // This is a dynamically created index (KINDEX) + // It cannot be created now, before cond_push is executed + tdbp->SetXdp(xdp); + return (xdp->IsUnique()) ? 1 : 2; + } // endif dynamic + + // Static indexes must be initialized now for records_in_range // Allocate the key columns definition block tdbp->Knum= xdp->GetNparts(); tdbp->To_Key_Col= (PCOL*)PlugSubAlloc(g, NULL, tdbp->Knum * sizeof(PCOL)); @@ -738,10 +746,23 @@ RCODE CntIndexRead(PGLOBAL g, PTDB ptdb, OPVAL op, // Set reference values and index operator if (!tdbp->To_Link || !tdbp->To_Kindex) { - sprintf(g->Message, "Index not initialized for table %s", tdbp->Name); - return RC_FX; - } else - xbp= (XXBASE*)tdbp->To_Kindex; + if (!tdbp->To_Xdp) { + sprintf(g->Message, "Index not initialized for table %s", tdbp->Name); + return RC_FX; + } // endif !To_Xdp + + // Now it's time to make the dynamic index + tdbp->SetFilter(tdbp->To_Def->GetHandler()->CheckFilter(g)); + + if (tdbp->MakeDynamicIndex(g)) { + sprintf(g->Message, "Fail to make dynamic index %s", + tdbp->To_Xdp->GetName()); + return RC_FX; + } // endif MakeDynamicIndex + + } // endif !To_Kindex + + xbp= (XXBASE*)tdbp->To_Kindex; if (key) { for (n= 0; n < tdbp->Knum; n++) { @@ -829,10 +850,14 @@ int CntIndexRange(PGLOBAL g, PTDB ptdb, const uchar* *key, uint *len, } else tdbp= (PTDBDOX)ptdb; - if (!tdbp->To_Link || !tdbp->To_Kindex) { - sprintf(g->Message, "Index not initialized for table %s", tdbp->Name); - DBUG_PRINT("Range", ("%s", g->Message)); - return -1; + if (!tdbp->To_Kindex || !tdbp->To_Link) { + if (!tdbp->To_Xdp) { + sprintf(g->Message, "Index not initialized for table %s", tdbp->Name); + DBUG_PRINT("Range", ("%s", g->Message)); + return -1; + } else // Dynamic index + return tdbp->To_Xdp->GetMaxSame(); // TODO a better estimate + } else xbp= (XXBASE*)tdbp->To_Kindex; diff --git a/storage/connect/filter.cpp b/storage/connect/filter.cpp index 62453b7b17b..9b646ca58b9 100644 --- a/storage/connect/filter.cpp +++ b/storage/connect/filter.cpp @@ -1711,7 +1711,7 @@ PFIL PrepareFilter(PGLOBAL g, PFIL fp, bool having) /* New algorithm: evaluate from the root a de-linearized filter so */ /* AND/OR clauses can be optimized throughout the whole tree. */ /***********************************************************************/ -DllExport bool ApplyFilter(PGLOBAL g, PFIL filp, PTDB tdbp) +DllExport bool ApplyFilter(PGLOBAL g, PFIL filp) { if (!filp) return TRUE; diff --git a/storage/connect/filter.h b/storage/connect/filter.h index 301b3a128de..85dc6dd4795 100644 --- a/storage/connect/filter.h +++ b/storage/connect/filter.h @@ -26,7 +26,7 @@ PFIL MakeFilter(PGLOBAL g, PCOL *colp, POPER pop, PPARM pfirst, bool neg); /***********************************************************************/ class DllExport FILTER : public XOBJECT { /* Filter description block */ //friend PFIL PrepareFilter(PGLOBAL, PFIL, bool); - friend DllExport bool ApplyFilter(PGLOBAL, PFIL, PTDB = NULL); + friend DllExport bool ApplyFilter(PGLOBAL, PFIL); public: // Constructors FILTER(PGLOBAL g, POPER pop, PPARM *tp = NULL); diff --git a/storage/connect/ha_connect.cc b/storage/connect/ha_connect.cc index 32caa185140..e9e53144e76 100644 --- a/storage/connect/ha_connect.cc +++ b/storage/connect/ha_connect.cc @@ -170,7 +170,7 @@ #define SZWMIN 4194304 // Minimum work area size 4M extern "C" { - char version[]= "Version 1.02.0002 March 16, 2014"; + char version[]= "Version 1.03.0002 April 23, 2014"; #if defined(XMSG) char msglang[]; // Default message language @@ -324,7 +324,7 @@ ha_create_table_option connect_field_option_list[]= */ ha_create_table_option connect_index_option_list[]= { - HA_IOPTION_BOOL("DYN", kindx, 0), + HA_IOPTION_BOOL("DYNAMIC", kindx, 0), HA_IOPTION_BOOL("MAPPED", mapped, 0), }; @@ -658,7 +658,7 @@ TABTYPE ha_connect::GetRealType(PTOS pos) const char *ha_connect::index_type(uint inx) { switch (GetIndexType(GetRealType())) { - case 1: return "XPLUG"; + case 1: return "XINDEX"; case 2: return "REMOTE"; } // endswitch @@ -1142,6 +1142,14 @@ void *ha_connect::GetColumnOption(PGLOBAL g, void *field, PCOLINFO pcf) return fldp; } // end of GetColumnOption +/****************************************************************************/ +/* Return an index option structure. */ +/****************************************************************************/ +PXOS ha_connect::GetIndexOptionStruct(KEY *kp) +{ + return kp->option_struct; +} // end of GetIndexOptionStruct + /****************************************************************************/ /* Returns the index description structure used to make the index. */ /****************************************************************************/ @@ -1151,6 +1159,7 @@ PIXDEF ha_connect::GetIndexInfo(TABLE_SHARE *s) bool unique; PIXDEF xdp, pxd=NULL, toidx= NULL; PKPDEF kpp, pkp; + PXOS xosp; KEY kp; PGLOBAL& g= xp->g; @@ -1163,6 +1172,7 @@ PIXDEF ha_connect::GetIndexInfo(TABLE_SHARE *s) // Find the index to describe kp= s->key_info[n]; + xosp= kp.option_struct; // Now get index information pn= (char*)s->keynames.type_names[n]; @@ -1205,6 +1215,20 @@ PIXDEF ha_connect::GetIndexInfo(TABLE_SHARE *s) xdp->SetNParts(kp.user_defined_key_parts); + if (xosp) { + xdp->Dynamic= xosp->kindx; + xdp->Mapped= xosp->mapped; + } else if (kp.comment.str != NULL) { + char *pv, *oplist= kp.comment.str; + + if ((pv= GetListOption(g, "Dynamic", oplist))) + xdp->Dynamic= (!*pv || *pv == 'y' || *pv == 'Y' || atoi(pv) != 0); + + if ((pv= GetListOption(g, "Mapped", oplist))) + xdp->Mapped= (!*pv || *pv == 'y' || *pv == 'Y' || atoi(pv) != 0); + + } // endif comment + if (pxd) pxd->SetNext(xdp); else @@ -1844,6 +1868,15 @@ const char *ha_connect::GetValStr(OPVAL vop, bool neg) } // end of GetValStr +/***********************************************************************/ +/* Check the WHERE condition and return a CONNECT filter. */ +/***********************************************************************/ +PFIL ha_connect::CheckFilter(PGLOBAL g) +{ + return CondFilter(g, (Item *)pushed_cond); +} // end of CheckFilter + + /***********************************************************************/ /* Check the WHERE condition and return a CONNECT filter. */ /***********************************************************************/ @@ -2680,7 +2713,7 @@ int ha_connect::index_init(uint idx, bool sorted) htrc("index_init CONNECT: %s\n", g->Message); active_index= MAX_KEY; rc= HA_ERR_INTERNAL_ERROR; - } else { + } else if (((PTDBDOX)tdbp)->To_Kindex) { if (((PTDBDOX)tdbp)->To_Kindex->GetNum_K()) { if (((PTDBASE)tdbp)->GetFtype() != RECFM_NAF) ((PTDBDOX)tdbp)->GetTxfp()->ResetBuffer(g); @@ -2689,7 +2722,6 @@ int ha_connect::index_init(uint idx, bool sorted) } else // Void table indexing= 0; - rc= 0; } // endif indexing if (xtrace) diff --git a/storage/connect/ha_connect.h b/storage/connect/ha_connect.h index e7aac7a5915..40815a1a9ea 100644 --- a/storage/connect/ha_connect.h +++ b/storage/connect/ha_connect.h @@ -72,6 +72,7 @@ typedef class XCHK *PCHK; typedef class user_connect *PCONNECT; typedef struct ha_table_option_struct TOS, *PTOS; typedef struct ha_field_option_struct FOS, *PFOS; +typedef struct ha_index_option_struct XOS, *PXOS; extern handlerton *connect_hton; @@ -195,6 +196,7 @@ public: bool NoFieldOptionChange(TABLE *tab); PFOS GetFieldOptionStruct(Field *fp); void *GetColumnOption(PGLOBAL g, void *field, PCOLINFO pcf); + PXOS GetIndexOptionStruct(KEY *kp); PIXDEF GetIndexInfo(TABLE_SHARE *s= NULL); const char *GetDBName(const char *name); const char *GetTableName(void); @@ -345,6 +347,7 @@ virtual const COND *cond_push(const COND *cond); PCFIL CheckCond(PGLOBAL g, PCFIL filp, AMT tty, Item *cond); const char *GetValStr(OPVAL vop, bool neg); PFIL CondFilter(PGLOBAL g, Item *cond); +PFIL CheckFilter(PGLOBAL g); /** Number of rows in table. It will only be called if diff --git a/storage/connect/tabdos.cpp b/storage/connect/tabdos.cpp index 3427f7b76a7..a23ce388184 100644 --- a/storage/connect/tabdos.cpp +++ b/storage/connect/tabdos.cpp @@ -1477,12 +1477,17 @@ PBF TDBDOS::CheckBlockFilari(PGLOBAL g, PXOB *arg, int op, bool *cnv) } // end of CheckBlockFilari /***********************************************************************/ -/* ResetBlkFil: reset the block filter and restore filtering. */ +/* ResetBlkFil: reset the block filter and restore filtering, or make */ +/* the block filter if To_Filter was not set when opening the table. */ /***********************************************************************/ void TDBDOS::ResetBlockFilter(PGLOBAL g) { - if (!To_BlkFil) + if (!To_BlkFil) { + if (To_Filter) + To_BlkFil = InitBlockFilter(g, To_Filter); + return; + } // endif To_BlkFil To_BlkFil->Reset(g); @@ -1590,7 +1595,7 @@ int TDBDOS::MakeIndex(PGLOBAL g, PIXDEF pxdf, bool add) // Allocate all columns that will be used by indexes. // This must be done before opening the table so specific - // column initialization can be done ( in particular by TDBVCT) + // column initialization can be done (in particular by TDBVCT) for (n = 0, xdp = pxdf; xdp; xdp = xdp->GetNext()) for (kdp = xdp->GetToKeyParts(); kdp; kdp = kdp->GetNext()) { if (!(colp = ColDB(g, kdp->GetName(), 0))) { @@ -1687,6 +1692,79 @@ err: return RC_FX; } // end of MakeIndex +/***********************************************************************/ +/* Make a dynamic index. */ +/***********************************************************************/ +bool TDBDOS::MakeDynamicIndex(PGLOBAL g) + { + int k, rc; + bool brc; + PCOL colp; + PCOLDEF cdp; + PVAL valp; + PIXDEF xdp; + PKXBASE kxp; + PKPDEF kdp; + + if (!(xdp = To_Xdp)) { + strcpy(g->Message, "NULL dynamic index"); + return true; + } // endif To_Xdp + + // Allocate the key columns definition block + Knum = xdp->GetNparts(); + To_Key_Col = (PCOL*)PlugSubAlloc(g, NULL, Knum * sizeof(PCOL)); + + // Get the key column description list + for (k = 0, kdp = xdp->GetToKeyParts(); kdp; kdp = kdp->GetNext()) + if (!(colp = ColDB(g, kdp->GetName(), 0)) || colp->InitValue(g)) { + sprintf(g->Message, "Wrong column %s", kdp->GetName()); + return true; + } else + To_Key_Col[k++] = colp; + +#if defined(_DEBUG) + if (k != Knum) { + sprintf(g->Message, "Key part number mismatch for %s", + xdp->GetName()); + return 0; + } // endif k +#endif // _DEBUG + + // Allocate the pseudo constants that will contain the key values + To_Link = (PXOB*)PlugSubAlloc(g, NULL, Knum * sizeof(PXOB)); + + for (k = 0, kdp = xdp->GetToKeyParts(); kdp; k++, kdp = kdp->GetNext()) { + cdp = Key(k)->GetCdp(); + valp = AllocateValue(g, cdp->GetType(), cdp->GetLength()); + To_Link[k]= new(g) CONSTANT(valp); + } // endfor k + + // Make the index on xdp + if (!xdp->IsAuto()) { + if (Knum == 1) // Single index + kxp= new(g) XINDXS(this, xdp, NULL, To_Key_Col, To_Link); + else // Multi-Column index + kxp= new(g) XINDEX(this, xdp, NULL, To_Key_Col, To_Link); + + } else // Column contains same values as ROWID + kxp= new(g) XXROW(this); + + // Prepare error return + if (g->jump_level == MAX_JUMP) { + strcpy(g->Message, MSG(TOO_MANY_JUMPS)); + return true; + } // endif + + if ((rc = setjmp(g->jumper[++g->jump_level])) != 0) { + brc = true; + } else if (!(brc = kxp->Make(g, xdp))) + To_Kindex= kxp; + + g->jump_level--; + return brc; + } // end of MakeDynamicIndex + /***********************************************************************/ /* DOS GetProgMax: get the max value for progress information. */ /***********************************************************************/ diff --git a/storage/connect/tabdos.h b/storage/connect/tabdos.h index 17220a52962..4db28f78868 100644 --- a/storage/connect/tabdos.h +++ b/storage/connect/tabdos.h @@ -169,6 +169,7 @@ class DllExport TDBDOS : public TDBASE { // Optimization routines virtual int MakeIndex(PGLOBAL g, PIXDEF pxdf, bool add); + bool MakeDynamicIndex(PGLOBAL g); void ResetBlockFilter(PGLOBAL g); bool GetDistinctColumnValues(PGLOBAL g, int nrec); diff --git a/storage/connect/table.cpp b/storage/connect/table.cpp index cb7e732d2dd..f550420ef3a 100644 --- a/storage/connect/table.cpp +++ b/storage/connect/table.cpp @@ -139,6 +139,7 @@ TDBASE::TDBASE(PTABDEF tdp) : TDB(tdp) To_Link = NULL; To_Key_Col = NULL; To_Kindex = NULL; + To_Xdp = NULL; To_SetCols = NULL; MaxSize = -1; Knum = 0; @@ -149,8 +150,13 @@ TDBASE::TDBASE(PTABDEF tdp) : TDB(tdp) TDBASE::TDBASE(PTDBASE tdbp) : TDB(tdbp) { To_Def = tdbp->To_Def; + To_Link = tdbp->To_Link; + To_Key_Col = tdbp->To_Key_Col; + To_Kindex = tdbp->To_Kindex; + To_Xdp = tdbp->To_Xdp; To_SetCols = tdbp->To_SetCols; // ??? MaxSize = tdbp->MaxSize; + Knum = tdbp->Knum; Read_Only = tdbp->Read_Only; m_data_charset= tdbp->m_data_charset; } // end of TDBASE copy constructor diff --git a/storage/connect/xindex.cpp b/storage/connect/xindex.cpp index 66b8abe2e9b..5039f2d7aee 100755 --- a/storage/connect/xindex.cpp +++ b/storage/connect/xindex.cpp @@ -112,6 +112,8 @@ INDEXDEF::INDEXDEF(char *name, bool uniq, int n) Unique = uniq; Invalid = false; AutoInc = false; + Dynamic = false; + Mapped = false; Nparts = 0; ID = n; //Offset = 0; @@ -242,7 +244,8 @@ void XINDEX::Reset(void) void XINDEX::Close(void) { // Close file or view of file - X->Close(); + if (X) + X->Close(); // De-allocate data PlgDBfree(Record); @@ -286,7 +289,7 @@ bool XINDEX::Make(PGLOBAL g, PIXDEF sxp) int k, rc = RC_OK; int *bof, i, j, n, ndf, nkey; PKPDEF kdfp = Xdp->GetToKeyParts(); - bool brc = true; + bool brc = false; PCOL colp; PXCOL kp, prev = NULL, kcp = NULL; PDBUSER dup = (PDBUSER)g->Activityp->Aptr; @@ -378,11 +381,14 @@ bool XINDEX::Make(PGLOBAL g, PIXDEF sxp) // Check return code and do whatever must be done according to it switch (rc) { case RC_OK: - break; - case RC_EF: - goto end_of_file; + if (ApplyFilter(g, Tdbp->GetFilter())) + break; + + // passthru case RC_NF: continue; + case RC_EF: + goto end_of_file; default: sprintf(g->Message, MSG(RC_READING), rc, Tdbp->Name); goto err; @@ -579,14 +585,15 @@ bool XINDEX::Make(PGLOBAL g, PIXDEF sxp) Cur_K = Num_K; /*********************************************************************/ - /* Save the index so it has not to be recalculated. */ + /* Save the xindex so it has not to be recalculated. */ /*********************************************************************/ - if (!SaveIndex(g, sxp)) - brc = false; + if (X && SaveIndex(g, sxp)) + brc = true; err: // We don't need the index anymore - Close(); + if (X || brc) + Close(); if (brc) printf("%s\n", g->Message); diff --git a/storage/connect/xindex.h b/storage/connect/xindex.h index 6ed0416bad5..f447051a517 100644 --- a/storage/connect/xindex.h +++ b/storage/connect/xindex.h @@ -87,6 +87,7 @@ class DllExport INDEXDEF : public BLOCK { /* Index description block */ void SetNext(PIXDEF pxdf) {Next = pxdf;} PSZ GetName(void) {return (PSZ)Name;} bool IsUnique(void) {return Unique;} + bool IsDynamic(void) {return Dynamic;} bool IsAuto(void) {return AutoInc;} void SetAuto(bool b) {AutoInc = b;} void SetInvalid(bool b) {Invalid = b;} @@ -115,6 +116,8 @@ class DllExport INDEXDEF : public BLOCK { /* Index description block */ bool Unique; /* true if defined as unique */ bool Invalid; /* true if marked as Invalid */ bool AutoInc; /* true if unique key in auto increment */ + bool Dynamic; /* KINDEX style */ + bool Mapped; /* Use file mapping */ int Nparts; /* Number of key parts */ int ID; /* Index ID number */ int MaxSame; /* Max number of same values */ @@ -192,6 +195,7 @@ class DllExport XXBASE : public CSORT, public BLOCK { virtual void Print(PGLOBAL g, FILE *f, uint n); virtual void Print(PGLOBAL g, char *ps, uint z); virtual bool Init(PGLOBAL g) = 0; + virtual bool Make(PGLOBAL g, PIXDEF sxp) = 0; #if defined(XMAP) virtual bool MapInit(PGLOBAL g) = 0; #endif // XMAP @@ -420,6 +424,7 @@ class DllExport XXROW : public XXBASE { virtual int MaxRange(void) {return 1;} virtual int Range(PGLOBAL g, int limit = 0, bool incl = true); virtual int Qcompare(int *, int *) {assert(false); return 0;} + virtual bool Make(PGLOBAL g, PIXDEF sxp) {return false;} virtual void Close(void) {} protected: diff --git a/storage/connect/xtable.h b/storage/connect/xtable.h index daaa40f2224..f6c5a0fbfdd 100644 --- a/storage/connect/xtable.h +++ b/storage/connect/xtable.h @@ -145,6 +145,7 @@ class DllExport TDBASE : public TDB { inline PKXBASE GetKindex(void) {return To_Kindex;} inline PCOL GetSetCols(void) {return To_SetCols;} inline void SetSetCols(PCOL colp) {To_SetCols = colp;} + inline void SetXdp(PIXDEF xdp) {To_Xdp = xdp;} // Properties void SetKindex(PKXBASE kxp); @@ -191,8 +192,9 @@ class DllExport TDBASE : public TDB { PXOB *To_Link; // Points to column of previous relations PCOL *To_Key_Col; // Points to key columns in current file PKXBASE To_Kindex; // Points to table key index + PIXDEF To_Xdp; // To the index definition block PCOL To_SetCols; // Points to updated columns - int MaxSize; // Max size in number of lines + int MaxSize; // Max size in number of lines int Knum; // Size of key arrays bool Read_Only; // True for read only tables const CHARSET_INFO *m_data_charset; From cdc38712e4d359348346758e47e1ebc7d1743f82 Mon Sep 17 00:00:00 2001 From: James Le Cuirot Date: Sun, 27 Apr 2014 00:02:19 +0100 Subject: [PATCH 005/279] MySQL Bug #61340: Use CMake EXPORT feature to aid cross-compiling. This technique is documented at: http://www.cmake.org/Wiki/CMake_Cross_Compiling#Using_executables_in_the_build_created_during_the_build Basic steps are: # mkdir native cross # cd native # cmake /path/to/maria # make IMPORT_EXECUTABLES # cd ../cross # cmake -DCMAKE_TOOLCHAIN_FILE=foo -DIMPORT_EXECUTABLES=/path/to/native/import_executables.cmake /path/to/maria # make --- CMakeLists.txt | 10 ++++++++++ dbug/CMakeLists.txt | 16 +++++++++------- sql/CMakeLists.txt | 4 +++- 3 files changed, 22 insertions(+), 8 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index f8ebddc70fe..20067ad5363 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -380,6 +380,11 @@ ELSEIF(MYSQL_MAINTAINER_MODE MATCHES "AUTO") SET(CMAKE_CXX_FLAGS_DEBUG "${MY_MAINTAINER_CXX_WARNINGS} ${CMAKE_CXX_FLAGS_DEBUG}") ENDIF() +IF(CMAKE_CROSSCOMPILING) + SET(IMPORT_EXECUTABLES "IMPORT_EXECUTABLES_NOT_SET" CACHE FILEPATH "Path to import_executables.cmake from a native build") + INCLUDE(${IMPORT_EXECUTABLES}) +ENDIF() + IF(WITH_UNIT_TESTS) ENABLE_TESTING() ADD_SUBDIRECTORY(unittest/mytap) @@ -443,6 +448,11 @@ IF(WIN32) ENDIF() ADD_SUBDIRECTORY(packaging/solaris) +IF(NOT CMAKE_CROSSCOMPILING) + ADD_CUSTOM_TARGET(IMPORT_EXECUTABLES DEPENDS comp_err comp_sql factorial gen_lex_hash) + EXPORT(TARGETS comp_err comp_sql factorial gen_lex_hash FILE ${CMAKE_BINARY_DIR}/import_executables.cmake) +ENDIF() + CONFIGURE_FILE(config.h.cmake ${CMAKE_BINARY_DIR}/include/my_config.h) CONFIGURE_FILE(config.h.cmake ${CMAKE_BINARY_DIR}/include/config.h) CONFIGURE_FILE(${CMAKE_SOURCE_DIR}/include/mysql_version.h.in diff --git a/dbug/CMakeLists.txt b/dbug/CMakeLists.txt index fddf234a4f1..bc4288c157c 100644 --- a/dbug/CMakeLists.txt +++ b/dbug/CMakeLists.txt @@ -24,8 +24,10 @@ TARGET_LINK_LIBRARIES(dbug mysys) ADD_EXECUTABLE(tests tests.c) TARGET_LINK_LIBRARIES(tests dbug) -ADD_EXECUTABLE(factorial my_main.c factorial.c) -TARGET_LINK_LIBRARIES(factorial dbug) +IF(NOT CMAKE_CROSSCOMPILING) + ADD_EXECUTABLE(factorial my_main.c factorial.c) + TARGET_LINK_LIBRARIES(factorial dbug) +ENDIF() IF(NOT WIN32 AND NOT CMAKE_GENERATOR MATCHES Xcode) FIND_PROGRAM(GROFF groff) @@ -34,11 +36,11 @@ IF(NOT WIN32 AND NOT CMAKE_GENERATOR MATCHES Xcode) SET(SOURCE_INC factorial.r main.r example1.r example2.r example3.r) ADD_CUSTOM_COMMAND(OUTPUT ${OUTPUT_INC} DEPENDS factorial - COMMAND ./factorial 1 2 3 4 5 > output1.r - COMMAND ./factorial -\#t:o 2 3 > output2.r - COMMAND ./factorial -\#d:t:o 3 > output3.r - COMMAND ./factorial -\#d,result:o 4 > output4.r - COMMAND ./factorial -\#d:f,factorial:F:L:o 3 > output5.r) + COMMAND factorial 1 2 3 4 5 > output1.r + COMMAND factorial -\#t:o 2 3 > output2.r + COMMAND factorial -\#d:t:o 3 > output3.r + COMMAND factorial -\#d,result:o 4 > output4.r + COMMAND factorial -\#d:f,factorial:F:L:o 3 > output5.r) FOREACH(file ${SOURCE_INC}) STRING(REGEX REPLACE "\\.r" ".c" srcfile ${file}) ADD_CUSTOM_COMMAND(OUTPUT ${file} DEPENDS ${srcfile} diff --git a/sql/CMakeLists.txt b/sql/CMakeLists.txt index ad4b12813d3..7d2b329f300 100644 --- a/sql/CMakeLists.txt +++ b/sql/CMakeLists.txt @@ -218,7 +218,9 @@ RUN_BISON( ) # Gen_lex_hash -ADD_EXECUTABLE(gen_lex_hash gen_lex_hash.cc) +IF(NOT CMAKE_CROSSCOMPILING) + ADD_EXECUTABLE(gen_lex_hash gen_lex_hash.cc) +ENDIF() ADD_CUSTOM_COMMAND( OUTPUT ${CMAKE_CURRENT_BINARY_DIR}/lex_hash.h From e7c7256d1d67125921d9f688a382aef873f5b9ce Mon Sep 17 00:00:00 2001 From: Olivier Bertrand Date: Wed, 30 Apr 2014 10:48:29 +0200 Subject: [PATCH 006/279] - Implementation of adding selected columns to dynamic indexes. modified: storage/connect/connect.cc storage/connect/ha_connect.cc storage/connect/ha_connect.h storage/connect/tabdos.cpp storage/connect/tabdos.h storage/connect/tabvct.cpp storage/connect/tabvct.h storage/connect/xindex.cpp storage/connect/xindex.h --- storage/connect/connect.cc | 71 ++-------------- storage/connect/ha_connect.cc | 60 +++++++++----- storage/connect/ha_connect.h | 5 +- storage/connect/tabdos.cpp | 38 ++++++--- storage/connect/tabdos.h | 2 +- storage/connect/tabvct.cpp | 2 +- storage/connect/tabvct.h | 2 + storage/connect/xindex.cpp | 149 +++++++++++++++++++++++++++------- storage/connect/xindex.h | 4 + 9 files changed, 207 insertions(+), 126 deletions(-) diff --git a/storage/connect/connect.cc b/storage/connect/connect.cc index f7bba541dfa..daff6ffdc68 100644 --- a/storage/connect/connect.cc +++ b/storage/connect/connect.cc @@ -604,15 +604,8 @@ int CntCloseTable(PGLOBAL g, PTDB tdbp) /***********************************************************************/ int CntIndexInit(PGLOBAL g, PTDB ptdb, int id) { - int k; - PCOL colp; - PVAL valp; - PKXBASE xp; - PXLOAD pxp; PIXDEF xdp; - XKPDEF *kdp; PTDBDOX tdbp; - PCOLDEF cdp; DOXDEF *dfp; if (!ptdb) @@ -651,64 +644,20 @@ int CntIndexInit(PGLOBAL g, PTDB ptdb, int id) return 0; } // endif xdp +#if 0 if (xdp->IsDynamic()) { // This is a dynamically created index (KINDEX) - // It cannot be created now, before cond_push is executed + // It should not be created now, if called by index range tdbp->SetXdp(xdp); return (xdp->IsUnique()) ? 1 : 2; } // endif dynamic +#endif // 0 // Static indexes must be initialized now for records_in_range - // Allocate the key columns definition block - tdbp->Knum= xdp->GetNparts(); - tdbp->To_Key_Col= (PCOL*)PlugSubAlloc(g, NULL, tdbp->Knum * sizeof(PCOL)); - - // Get the key column description list - for (k= 0, kdp= (XKPDEF*)xdp->GetToKeyParts(); kdp; kdp= (XKPDEF*)kdp->Next) - if (!(colp= tdbp->ColDB(g, kdp->Name, 0)) || colp->InitValue(g)) { - sprintf(g->Message, "Wrong column %s", kdp->Name); - return 0; - } else - tdbp->To_Key_Col[k++]= colp; - -#if defined(_DEBUG) - if (k != tdbp->Knum) { - sprintf(g->Message, "Key part number mismatch for %s", - xdp->GetName()); - return 0; - } // endif k -#endif // _DEBUG - - // Allocate the pseudo constants that will contain the key values - tdbp->To_Link= (PXOB*)PlugSubAlloc(g, NULL, tdbp->Knum * sizeof(PXOB)); - - for (k= 0, kdp= (XKPDEF*)xdp->GetToKeyParts(); - kdp; k++, kdp= (XKPDEF*)kdp->Next) { - cdp= tdbp->Key(k)->GetCdp(); - valp= AllocateValue(g, cdp->GetType(), cdp->GetLength()); - tdbp->To_Link[k]= new(g) CONSTANT(valp); - } // endfor k - - // Make the index on xdp - if (!xdp->IsAuto()) { - if (dfp->Huge) - pxp= new(g) XHUGE; - else - pxp= new(g) XFILE; - - if (tdbp->Knum == 1) // Single index - xp= new(g) XINDXS(tdbp, xdp, pxp, tdbp->To_Key_Col, tdbp->To_Link); - else // Multi-Column index - xp= new(g) XINDEX(tdbp, xdp, pxp, tdbp->To_Key_Col, tdbp->To_Link); - - } else // Column contains same values as ROWID - xp= new(g) XXROW(tdbp); - - if (xp->Init(g)) + if (tdbp->InitialyzeIndex(g, xdp)) return 0; - tdbp->To_Kindex= xp; - return (xp->IsMul()) ? 2 : 1; + return (tdbp->To_Kindex->IsMul()) ? 2 : 1; } // end of CntIndexInit /***********************************************************************/ @@ -746,20 +695,18 @@ RCODE CntIndexRead(PGLOBAL g, PTDB ptdb, OPVAL op, // Set reference values and index operator if (!tdbp->To_Link || !tdbp->To_Kindex) { - if (!tdbp->To_Xdp) { +// if (!tdbp->To_Xdp) { sprintf(g->Message, "Index not initialized for table %s", tdbp->Name); return RC_FX; +#if 0 } // endif !To_Xdp - // Now it's time to make the dynamic index - tdbp->SetFilter(tdbp->To_Def->GetHandler()->CheckFilter(g)); - - if (tdbp->MakeDynamicIndex(g)) { + if (tdbp->InitialyzeIndex(g, NULL)) { sprintf(g->Message, "Fail to make dynamic index %s", tdbp->To_Xdp->GetName()); return RC_FX; } // endif MakeDynamicIndex - +#endif // 0 } // endif !To_Kindex xbp= (XXBASE*)tdbp->To_Kindex; diff --git a/storage/connect/ha_connect.cc b/storage/connect/ha_connect.cc index d56f3ca08e4..ae9de333054 100644 --- a/storage/connect/ha_connect.cc +++ b/storage/connect/ha_connect.cc @@ -324,8 +324,10 @@ ha_create_table_option connect_field_option_list[]= */ ha_create_table_option connect_index_option_list[]= { - HA_IOPTION_BOOL("DYNAMIC", kindx, 0), + HA_IOPTION_BOOL("DYNAMIC", dynamic, 0), + HA_IOPTION_BOOL("DYNAM", dynamic, 0), HA_IOPTION_BOOL("MAPPED", mapped, 0), + HA_IOPTION_END }; /***********************************************************************/ @@ -435,6 +437,7 @@ static int connect_init_func(void *p) connect_hton->flags= HTON_TEMPORARY_NOT_SUPPORTED | HTON_NO_PARTITION; connect_hton->table_options= connect_table_option_list; connect_hton->field_options= connect_field_option_list; + connect_hton->index_options= connect_index_option_list; connect_hton->tablefile_extensions= ha_connect_exts; connect_hton->discover_table_structure= connect_assisted_discovery; @@ -658,7 +661,13 @@ TABTYPE ha_connect::GetRealType(PTOS pos) const char *ha_connect::index_type(uint inx) { switch (GetIndexType(GetRealType())) { - case 1: return "XINDEX"; + case 1: + if (table_share) + return (GetIndexOption(&table_share->key_info[inx], "Dynamic")) + ? "KINDEX" : "XINDEX"; + else + return "XINDEX"; + case 2: return "REMOTE"; } // endswitch @@ -1150,6 +1159,31 @@ PXOS ha_connect::GetIndexOptionStruct(KEY *kp) return kp->option_struct; } // end of GetIndexOptionStruct +/****************************************************************************/ +/* Return a Boolean index option or false if not specified. */ +/****************************************************************************/ +bool ha_connect::GetIndexOption(KEY *kp, char *opname) +{ + bool opval= false; + PXOS options= GetIndexOptionStruct(kp); + + if (options) { + if (!stricmp(opname, "Dynamic")) + opval= options->dynamic; + else if (!stricmp(opname, "Mapped")) + opval= options->mapped; + + } else if (kp->comment.str != NULL) { + char *pv, *oplist= kp->comment.str; + + if ((pv= GetListOption(xp->g, opname, oplist))) + opval= (!*pv || *pv == 'y' || *pv == 'Y' || atoi(pv) != 0); + + } // endif comment + + return opval; +} // end of GetIndexOption + /****************************************************************************/ /* Returns the index description structure used to make the index. */ /****************************************************************************/ @@ -1159,7 +1193,6 @@ PIXDEF ha_connect::GetIndexInfo(TABLE_SHARE *s) bool unique; PIXDEF xdp, pxd=NULL, toidx= NULL; PKPDEF kpp, pkp; - PXOS xosp; KEY kp; PGLOBAL& g= xp->g; @@ -1172,7 +1205,6 @@ PIXDEF ha_connect::GetIndexInfo(TABLE_SHARE *s) // Find the index to describe kp= s->key_info[n]; - xosp= kp.option_struct; // Now get index information pn= (char*)s->keynames.type_names[n]; @@ -1214,20 +1246,8 @@ PIXDEF ha_connect::GetIndexInfo(TABLE_SHARE *s) } // endfor k xdp->SetNParts(kp.user_defined_key_parts); - - if (xosp) { - xdp->Dynamic= xosp->kindx; - xdp->Mapped= xosp->mapped; - } else if (kp.comment.str != NULL) { - char *pv, *oplist= kp.comment.str; - - if ((pv= GetListOption(g, "Dynamic", oplist))) - xdp->Dynamic= (!*pv || *pv == 'y' || *pv == 'Y' || atoi(pv) != 0); - - if ((pv= GetListOption(g, "Mapped", oplist))) - xdp->Mapped= (!*pv || *pv == 'y' || *pv == 'Y' || atoi(pv) != 0); - - } // endif comment + xdp->Dynamic= GetIndexOption(&kp, "Dynamic"); + xdp->Mapped= GetIndexOption(&kp, "Mapped"); if (pxd) pxd->SetNext(xdp); @@ -1867,7 +1887,7 @@ const char *ha_connect::GetValStr(OPVAL vop, bool neg) return val; } // end of GetValStr - +#if 0 /***********************************************************************/ /* Check the WHERE condition and return a CONNECT filter. */ /***********************************************************************/ @@ -1875,7 +1895,7 @@ PFIL ha_connect::CheckFilter(PGLOBAL g) { return CondFilter(g, (Item *)pushed_cond); } // end of CheckFilter - +#endif // 0 /***********************************************************************/ /* Check the WHERE condition and return a CONNECT filter. */ diff --git a/storage/connect/ha_connect.h b/storage/connect/ha_connect.h index 40815a1a9ea..ebdeeae8623 100644 --- a/storage/connect/ha_connect.h +++ b/storage/connect/ha_connect.h @@ -140,7 +140,7 @@ struct ha_field_option_struct */ struct ha_index_option_struct { - bool kindx; + bool dynamic; bool mapped; }; @@ -187,6 +187,7 @@ public: bool GetBooleanOption(char *opname, bool bdef); bool SetBooleanOption(char *opname, bool b); int GetIntegerOption(char *opname); + bool GetIndexOption(KEY *kp, char *opname); bool CheckString(const char *str1, const char *str2); bool SameString(TABLE *tab, char *opn); bool SetIntegerOption(char *opname, int n); @@ -347,7 +348,7 @@ virtual const COND *cond_push(const COND *cond); PCFIL CheckCond(PGLOBAL g, PCFIL filp, AMT tty, Item *cond); const char *GetValStr(OPVAL vop, bool neg); PFIL CondFilter(PGLOBAL g, Item *cond); -PFIL CheckFilter(PGLOBAL g); +//PFIL CheckFilter(PGLOBAL g); /** Number of rows in table. It will only be called if diff --git a/storage/connect/tabdos.cpp b/storage/connect/tabdos.cpp index a23ce388184..e596b438d5f 100644 --- a/storage/connect/tabdos.cpp +++ b/storage/connect/tabdos.cpp @@ -1695,21 +1695,23 @@ err: /***********************************************************************/ /* Make a dynamic index. */ /***********************************************************************/ -bool TDBDOS::MakeDynamicIndex(PGLOBAL g) +bool TDBDOS::InitialyzeIndex(PGLOBAL g, PIXDEF xdp) { int k, rc; - bool brc; + bool brc, dynamic; PCOL colp; PCOLDEF cdp; PVAL valp; - PIXDEF xdp; + PXLOAD pxp; PKXBASE kxp; PKPDEF kdp; - if (!(xdp = To_Xdp)) { + if (!xdp && !(xdp = To_Xdp)) { strcpy(g->Message, "NULL dynamic index"); return true; - } // endif To_Xdp + } else + dynamic = To_Filter && xdp->IsUnique() && xdp->IsDynamic(); +// dynamic = To_Filter && xdp->IsDynamic(); NIY // Allocate the key columns definition block Knum = xdp->GetNparts(); @@ -1742,13 +1744,22 @@ bool TDBDOS::MakeDynamicIndex(PGLOBAL g) // Make the index on xdp if (!xdp->IsAuto()) { + if (!dynamic) { + if (((PDOSDEF)To_Def)->Huge) + pxp = new(g) XHUGE; + else + pxp = new(g) XFILE; + + } else + pxp = NULL; + if (Knum == 1) // Single index - kxp= new(g) XINDXS(this, xdp, NULL, To_Key_Col, To_Link); - else // Multi-Column index - kxp= new(g) XINDEX(this, xdp, NULL, To_Key_Col, To_Link); + kxp = new(g) XINDXS(this, xdp, pxp, To_Key_Col, To_Link); + else // Multi-Column index + kxp = new(g) XINDEX(this, xdp, pxp, To_Key_Col, To_Link); } else // Column contains same values as ROWID - kxp= new(g) XXROW(this); + kxp = new(g) XXROW(this); // Prepare error return if (g->jump_level == MAX_JUMP) { @@ -1758,12 +1769,15 @@ bool TDBDOS::MakeDynamicIndex(PGLOBAL g) if ((rc = setjmp(g->jumper[++g->jump_level])) != 0) { brc = true; - } else if (!(brc = kxp->Make(g, xdp))) - To_Kindex= kxp; + } else + if (!(brc = (dynamic) ? kxp->Make(g, xdp) : kxp->Init(g))) { + kxp->SetDynamic(dynamic); + To_Kindex= kxp; + } // endif brc g->jump_level--; return brc; - } // end of MakeDynamicIndex + } // end of InitialyzeIndex /***********************************************************************/ /* DOS GetProgMax: get the max value for progress information. */ diff --git a/storage/connect/tabdos.h b/storage/connect/tabdos.h index d7a6af5a5ec..c6652c4d2d7 100644 --- a/storage/connect/tabdos.h +++ b/storage/connect/tabdos.h @@ -170,7 +170,7 @@ class DllExport TDBDOS : public TDBASE { // Optimization routines virtual int MakeIndex(PGLOBAL g, PIXDEF pxdf, bool add); - bool MakeDynamicIndex(PGLOBAL g); + bool InitialyzeIndex(PGLOBAL g, PIXDEF xdp); void ResetBlockFilter(PGLOBAL g); bool GetDistinctColumnValues(PGLOBAL g, int nrec); diff --git a/storage/connect/tabvct.cpp b/storage/connect/tabvct.cpp index ed258f3a80f..ba5cf36a640 100644 --- a/storage/connect/tabvct.cpp +++ b/storage/connect/tabvct.cpp @@ -32,7 +32,7 @@ /***********************************************************************/ /***********************************************************************/ -/* Include relevant MariaDB header file. */ +/* Include relevant MariaDB header file. */ /***********************************************************************/ #include "my_global.h" #if defined(WIN32) diff --git a/storage/connect/tabvct.h b/storage/connect/tabvct.h index e15150ab356..b744dbb2529 100644 --- a/storage/connect/tabvct.h +++ b/storage/connect/tabvct.h @@ -20,6 +20,7 @@ typedef class VCTCOL *PVCTCOL; /* VCT table. */ /***********************************************************************/ class DllExport VCTDEF : public DOSDEF { /* Logical table description */ + friend class TDBVCT; friend class VCTFAM; friend class VECFAM; friend class VMPFAM; @@ -64,6 +65,7 @@ class DllExport TDBVCT : public TDBFIX { virtual AMT GetAmType(void) {return TYPE_AM_VCT;} virtual PTDB Duplicate(PGLOBAL g) {return (PTDB)new(g) TDBVCT(g, this);} + bool IsSplit(void) {return ((VCTDEF*)To_Def)->Split;} // Methods virtual PTDB CopyOne(PTABS t); diff --git a/storage/connect/xindex.cpp b/storage/connect/xindex.cpp index 5039f2d7aee..2a7a10528fb 100755 --- a/storage/connect/xindex.cpp +++ b/storage/connect/xindex.cpp @@ -45,6 +45,7 @@ //nclude "array.h" #include "filamtxt.h" #include "tabdos.h" +#include "tabvct.h" /***********************************************************************/ /* Macro or external routine definition */ @@ -167,6 +168,8 @@ XXBASE::XXBASE(PTDBDOS tbxp, bool b) : CSORT(b), Op = OP_EQ; To_KeyCol = NULL; Mul = false; + Srtd = false; + Dynamic = false; Val_K = -1; Nblk = Sblk = 0; Thresh = 7; @@ -252,13 +255,14 @@ void XINDEX::Close(void) PlgDBfree(Index); PlgDBfree(Offset); - // De-allocate Key data - for (PXCOL kcp = To_KeyCol; kcp; kcp = kcp->Next) - kcp->FreeData(); + for (PXCOL kcp = To_KeyCol; kcp; kcp = kcp->Next) { + // Column values cannot be retrieved from key anymore + if (kcp->Colp) + kcp->Colp->SetKcol(NULL); - // Column values cannot be retrieved from key anymore - for (int k = 0; k < Nk; k++) - To_Cols[k]->SetKcol(NULL); + // De-allocate Key data + kcp->FreeData(); + } // endfor kcp } // end of Close @@ -278,6 +282,25 @@ int XINDEX::Qcompare(int *i1, int *i2) return k; } // end of Qcompare +/***********************************************************************/ +/* AddColumns: here we try to determine whether it is worthwhile to */ +/* add to the keys the values of the columns selected for this table. */ +/* Sure enough, it is done while records are read and permit to avoid */ +/* reading the table while doing the join (Dynamic index only) */ +/***********************************************************************/ +bool XINDEX::AddColumns(void) + { + if (!Dynamic) + return false; // Not applying to static index + else if (IsMul()) + return false; // Not done yet for multiple index + else if (Tbxp->GetAmType() == TYPE_AM_VCT && ((PTDBVCT)Tbxp)->IsSplit()) + return false; // This would require to read additional files + else + return true; + + } // end of AddColumns + /***********************************************************************/ /* Make: Make and index on key column(s). */ /***********************************************************************/ @@ -291,8 +314,13 @@ bool XINDEX::Make(PGLOBAL g, PIXDEF sxp) PKPDEF kdfp = Xdp->GetToKeyParts(); bool brc = false; PCOL colp; - PXCOL kp, prev = NULL, kcp = NULL; - PDBUSER dup = (PDBUSER)g->Activityp->Aptr; + PFIL filp = Tdbp->GetFilter(); + PXCOL kp, addcolp, prev = NULL, kcp = NULL; +//PDBUSER dup = (PDBUSER)g->Activityp->Aptr; + +#if defined(_DEBUG) + assert(X || Nk == 1); +#endif // _DEBUG /*********************************************************************/ /* Allocate the storage that will contain the keys and the file */ @@ -350,6 +378,50 @@ bool XINDEX::Make(PGLOBAL g, PIXDEF sxp) To_LastCol = prev; + if (AddColumns()) { + PCOL kolp = To_Cols[0]; // Temporary while imposing Nk = 1 + + i = 0; + + // Allocate the accompanying + for (colp = Tbxp->GetColumns(); colp; colp = colp->GetNext()) { + // Count how many columns to add +// for (k = 0; k < Nk; k++) +// if (colp == To_Cols[k]) +// break; + +// if (k == nk) + if (colp != kolp) + i++; + + } // endfor colp + + if (i && i < 10) // Should be a parameter + for (colp = Tbxp->GetColumns(); colp; colp = colp->GetNext()) { +// for (k = 0; k < Nk; k++) +// if (colp == To_Cols[k]) +// break; + +// if (k < nk) + if (colp == kolp) + continue; // This is a key column + + kcp = new(g) KXYCOL(this); + + if (kcp->Init(g, colp, n, true, NULL)) + return true; + + if (trace) + htrc("Adding colp=%p Buf_Type=%d size=%d\n", + colp, colp->GetResultType(), n); + + prev->Next = kcp; + prev = kcp; + } // endfor colp + + } // endif AddColumns + +#if 0 /*********************************************************************/ /* Get the starting information for progress. */ /*********************************************************************/ @@ -357,18 +429,19 @@ bool XINDEX::Make(PGLOBAL g, PIXDEF sxp) sprintf((char*)dup->Step, MSG(BUILD_INDEX), Xdp->GetName(), Tdbp->Name); dup->ProgMax = Tdbp->GetProgMax(g); dup->ProgCur = 0; +#endif // 0 /*********************************************************************/ /* Standard init: read the file and construct the index table. */ /* Note: reading will be sequential as To_Kindex is not set. */ /*********************************************************************/ for (i = nkey = 0; i < n && rc != RC_EF; i++) { -#if defined(THREAD) +#if 0 if (!dup->Step) { strcpy(g->Message, MSG(QUERY_CANCELLED)); longjmp(g->jumper[g->jump_level], 99); } // endif Step -#endif // THREAD +#endif // 0 /*******************************************************************/ /* Read a valid record from table file. */ @@ -376,12 +449,12 @@ bool XINDEX::Make(PGLOBAL g, PIXDEF sxp) rc = Tdbp->ReadDB(g); // Update progress information - dup->ProgCur = Tdbp->GetProgCur(); +// dup->ProgCur = Tdbp->GetProgCur(); // Check return code and do whatever must be done according to it switch (rc) { case RC_OK: - if (ApplyFilter(g, Tdbp->GetFilter())) + if (ApplyFilter(g, filp)) break; // passthru @@ -398,7 +471,11 @@ bool XINDEX::Make(PGLOBAL g, PIXDEF sxp) /* Get and Store the file position of the last read record for */ /* future direct access. */ /*******************************************************************/ - To_Rec[nkey] = Tdbp->GetRecpos(); + if (nkey == n) { + sprintf(g->Message, MSG(TOO_MANY_KEYS), nkey); + return true; + } else + To_Rec[nkey] = Tdbp->GetRecpos(); /*******************************************************************/ /* Get the keys and place them in the key blocks. */ @@ -407,11 +484,11 @@ bool XINDEX::Make(PGLOBAL g, PIXDEF sxp) k < Nk && kcp; k++, kcp = kcp->Next) { colp = To_Cols[k]; - colp->Reset(); - colp->ReadColumn(g); -// if (colp->ReadColumn(g)) -// goto err; + if (!colp->GetStatus(BUF_READ)) + colp->ReadColumn(g); + else + colp->Reset(); kcp->SetValue(colp, nkey); } // endfor k @@ -422,7 +499,7 @@ bool XINDEX::Make(PGLOBAL g, PIXDEF sxp) end_of_file: // Update progress information - dup->ProgCur = Tdbp->GetProgMax(g); +//dup->ProgCur = Tdbp->GetProgMax(g); /*********************************************************************/ /* Record the Index size and eventually resize memory allocation. */ @@ -457,6 +534,10 @@ bool XINDEX::Make(PGLOBAL g, PIXDEF sxp) goto err; // Error } // endif alloc + // We must separate keys and added columns before sorting + addcolp = To_LastCol->Next; + To_LastCol->Next = NULL; + // Call the sort program, it returns the number of distinct values if ((Ndif = Qsort(g, Num_K)) < 0) goto err; // Error during sort @@ -469,6 +550,9 @@ bool XINDEX::Make(PGLOBAL g, PIXDEF sxp) } else PlgDBfree(Offset); // Not used anymore + // Restore kcp list + To_LastCol->Next = addcolp; + // Use the index to physically reorder the xindex Srtd = Reorder(g); @@ -493,7 +577,7 @@ bool XINDEX::Make(PGLOBAL g, PIXDEF sxp) } else { Mul = false; // Current index is unique PlgDBfree(Offset); // Not used anymore - MaxSame = 1; // Reset it when remaking an index + MaxSame = 1; // Reset it when remaking an index } // endif Ndif /*********************************************************************/ @@ -508,7 +592,7 @@ bool XINDEX::Make(PGLOBAL g, PIXDEF sxp) /* except if the subset originally contains unique values. */ /*********************************************************************/ // Update progress information - dup->Step = STEP(REDUCE_INDEX); +//dup->Step = STEP(REDUCE_INDEX); ndf = Ndif; To_LastCol->Mxs = MaxSame; @@ -558,7 +642,7 @@ bool XINDEX::Make(PGLOBAL g, PIXDEF sxp) /* calculated, so the Record array can be discarted. */ /* Note: for Num_K = 1 any non null value is Ok. */ /*********************************************************************/ - if (Srtd && Tdbp->Ftype != RECFM_VAR) { + if (Srtd && !filp && Tdbp->Ftype != RECFM_VAR) { Incr = (Num_K > 1) ? To_Rec[1] : Num_K; PlgDBfree(Record); } // endif Srtd @@ -587,8 +671,14 @@ bool XINDEX::Make(PGLOBAL g, PIXDEF sxp) /*********************************************************************/ /* Save the xindex so it has not to be recalculated. */ /*********************************************************************/ - if (X && SaveIndex(g, sxp)) - brc = true; + if (X) { + if (SaveIndex(g, sxp)) + brc = true; + + } else // Dynamic index + // Indicate that key column values can be found from KEYCOL's + for (kcp = To_KeyCol; kcp; kcp = kcp->Next) + kcp->Colp->SetKcol(kcp); err: // We don't need the index anymore @@ -637,6 +727,7 @@ bool XINDEX::Reorder(PGLOBAL g) register int i, j, k, n; bool sorted = true; PXCOL kcp; +#if 0 PDBUSER dup = (PDBUSER)g->Activityp->Aptr; if (Num_K > 500000) { @@ -646,6 +737,7 @@ bool XINDEX::Reorder(PGLOBAL g) dup->ProgCur = 0; } else dup = NULL; +#endif // 0 if (!Pex) return Srtd; @@ -654,8 +746,8 @@ bool XINDEX::Reorder(PGLOBAL g) if (Pex[i] == Num_K) { // Already moved continue; } else if (Pex[i] == i) { // Already placed - if (dup) - dup->ProgCur++; +// if (dup) +// dup->ProgCur++; continue; } // endif's Pex @@ -684,8 +776,8 @@ bool XINDEX::Reorder(PGLOBAL g) To_Rec[j] = To_Rec[k]; } // endif k - if (dup) - dup->ProgCur++; +// if (dup) +// dup->ProgCur++; } // endfor j @@ -2834,7 +2926,7 @@ bool KXYCOL::Init(PGLOBAL g, PCOL colp, int n, bool sm, int kln) int len = colp->GetLength(), prec = colp->GetScale(); // Currently no indexing on NULL columns - if (colp->IsNullable()) { + if (colp->IsNullable() && kln) { sprintf(g->Message, "Cannot index nullable column %s", colp->GetName()); return true; } // endif nullable @@ -2877,6 +2969,7 @@ bool KXYCOL::Init(PGLOBAL g, PCOL colp, int n, bool sm, int kln) IsSorted = colp->GetOpt() == 2; //SetNulls(colp->IsNullable()); for when null columns will be indexable + Colp = colp; return false; } // end of Init diff --git a/storage/connect/xindex.h b/storage/connect/xindex.h index f447051a517..50f8282e5c3 100644 --- a/storage/connect/xindex.h +++ b/storage/connect/xindex.h @@ -177,6 +177,8 @@ class DllExport XXBASE : public CSORT, public BLOCK { virtual void Reset(void) = 0; virtual bool IsMul(void) {return false;} virtual bool IsRandom(void) {return true;} + virtual bool IsDynamic(void) {return Dynamic;} + virtual void SetDynamic(bool dyn) {Dynamic = dyn;} virtual bool HaveSame(void) {return false;} virtual int GetCurPos(void) {return Cur_K;} virtual void SetNval(int n) {assert(n == 1);} @@ -227,6 +229,7 @@ class DllExport XXBASE : public CSORT, public BLOCK { OPVAL Op; // Search operator bool Mul; // true if multiple bool Srtd; // true for sorted column + bool Dynamic; // true when dynamically made int Val_K; // Index of current value int Nblk; // Number of blocks int Sblk; // Block size @@ -275,6 +278,7 @@ class DllExport XINDEX : public XXBASE { bool GetAllSizes(PGLOBAL g, int &ndif, int &numk); protected: + bool AddColumns(void); bool NextValDif(void); // Members From cdbb79583778827e7ceb90f3e9230062327e0b09 Mon Sep 17 00:00:00 2001 From: Olivier Bertrand Date: Fri, 2 May 2014 15:55:45 +0200 Subject: [PATCH 007/279] - Adding fetched columns to Dynamic index key (unique only) Fix two bugs concerning added KXYCOL's: 1 - Not set during reading 2 - Val_K not set in FastFind modified: storage/connect/connect.cc storage/connect/filamtxt.h storage/connect/tabdos.cpp storage/connect/tabfix.cpp storage/connect/table.cpp storage/connect/valblk.h storage/connect/xindex.cpp storage/connect/xindex.h storage/connect/xtable.h --- storage/connect/connect.cc | 4 +-- storage/connect/filamtxt.h | 2 +- storage/connect/tabdos.cpp | 32 +++++++++++--------- storage/connect/tabfix.cpp | 9 ++++-- storage/connect/table.cpp | 8 +++-- storage/connect/valblk.h | 60 +++++++++++++++++++------------------- storage/connect/xindex.cpp | 26 ++++++++++++----- storage/connect/xindex.h | 2 +- storage/connect/xtable.h | 2 +- 9 files changed, 84 insertions(+), 61 deletions(-) diff --git a/storage/connect/connect.cc b/storage/connect/connect.cc index daff6ffdc68..9e551f2ccc7 100644 --- a/storage/connect/connect.cc +++ b/storage/connect/connect.cc @@ -442,7 +442,7 @@ RCODE CntReadNext(PGLOBAL g, PTDB tdbp) for (PCOL colp= tdbp->GetColumns(); colp; colp= colp->GetNext()) colp->SetKcol(NULL); - ((PTDBASE)tdbp)->SetKindex(NULL); + ((PTDBASE)tdbp)->SetKindex(g, NULL); } // endif index // Save stack and allocation environment and prepare error return @@ -585,7 +585,7 @@ int CntCloseTable(PGLOBAL g, PTDB tdbp) // Make all the eventual indexes tbxp= (TDBDOX*)tdbp; - tbxp->SetKindex(NULL); + tbxp->SetKindex(g, NULL); tbxp->To_Key_Col= NULL; rc= tbxp->ResetTableOpt(g, true, ((PTDBASE)tdbp)->GetDef()->Indexable() == 1); diff --git a/storage/connect/filamtxt.h b/storage/connect/filamtxt.h index 04375d9daa5..94d518044cc 100644 --- a/storage/connect/filamtxt.h +++ b/storage/connect/filamtxt.h @@ -65,7 +65,7 @@ class DllExport TXTFAM : public BLOCK { virtual bool DeferReading(void) {IsRead = false; return true;} virtual int ReadBuffer(PGLOBAL g) = 0; virtual int WriteBuffer(PGLOBAL g) = 0; - virtual int DeleteRecords(PGLOBAL g, int irc) = 0; + virtual int DeleteRecords(PGLOBAL g, int irc) = 0; virtual void CloseTableFile(PGLOBAL g) = 0; virtual void Rewind(void) = 0; diff --git a/storage/connect/tabdos.cpp b/storage/connect/tabdos.cpp index e596b438d5f..bb562defe17 100644 --- a/storage/connect/tabdos.cpp +++ b/storage/connect/tabdos.cpp @@ -1377,15 +1377,10 @@ PBF TDBDOS::CheckBlockFilari(PGLOBAL g, PXOB *arg, int op, bool *cnv) if (n == 3 || n == 6) { if (conv) { // The constant has not the good type and will not match - // the block min/max values. What we can do here is either - // abort with an error message or simply not do the block - // optimization (as column values can be converted when - // evaluating the filter.) Currently we prefer aborting - // because the user may count on the performance enhancing - // and silently not doing it is probably worse than just - // telling him to fix his query. + // the block min/max values. Warn and abort. sprintf(g->Message, "Block opt: %s", MSG(VALTYPE_NOMATCH)); - longjmp(g->jumper[g->jump_level], 99); + PushWarning(g, this); + return NULL; } // endif Conv if (type[0] == 1) { @@ -1484,7 +1479,10 @@ void TDBDOS::ResetBlockFilter(PGLOBAL g) { if (!To_BlkFil) { if (To_Filter) - To_BlkFil = InitBlockFilter(g, To_Filter); + if ((To_BlkFil = InitBlockFilter(g, To_Filter))) { + htrc("BlkFil=%p\n", To_BlkFil); + MaxSize = -1; // To be recalculated + } // endif To_BlkFil return; } // endif To_BlkFil @@ -1767,13 +1765,19 @@ bool TDBDOS::InitialyzeIndex(PGLOBAL g, PIXDEF xdp) return true; } // endif - if ((rc = setjmp(g->jumper[++g->jump_level])) != 0) { - brc = true; - } else - if (!(brc = (dynamic) ? kxp->Make(g, xdp) : kxp->Init(g))) { + if (!(rc = setjmp(g->jumper[++g->jump_level])) != 0) { + if (dynamic) { + ResetBlockFilter(g); kxp->SetDynamic(dynamic); + brc = kxp->Make(g, xdp); + } else + brc = kxp->Init(g); + + if (!brc) To_Kindex= kxp; - } // endif brc + + } else + brc = true; g->jump_level--; return brc; diff --git a/storage/connect/tabfix.cpp b/storage/connect/tabfix.cpp index aa6a44fc791..4b75a89bba4 100644 --- a/storage/connect/tabfix.cpp +++ b/storage/connect/tabfix.cpp @@ -133,8 +133,8 @@ int TDBFIX::ResetTableOpt(PGLOBAL g, bool dop, bool dox) //To_BlkIdx = NULL; // and block filtering To_BlkFil = NULL; // and index filtering RestoreNrec(); // May have been modified - MaxSize = -1; // Size must be recalculated - Cardinal = -1; // as well as Cardinality + MaxSize = -1; // Size must be recalculated + Cardinal = -1; // as well as Cardinality if (dop) { Columns = NULL; // Not used anymore @@ -177,6 +177,9 @@ void TDBFIX::RestoreNrec(void) Txfp->Nrec = (To_Def && To_Def->GetElemt()) ? To_Def->GetElemt() : DOS_BUFF_LEN; Txfp->Blksize = Txfp->Nrec * Txfp->Lrecl; + assert(Cardinal >= 0); + Txfp->Block = (Cardinal > 0) + ? (Cardinal + Txfp->Nrec - 1) / Txfp->Nrec : 0; } // endif Padded } // end of RestoreNrec @@ -332,7 +335,7 @@ bool TDBFIX::OpenDB(PGLOBAL g) To_BlkFil = InitBlockFilter(g, To_Filter); if (trace) - htrc("OpenDos: R%hd mode=%d\n", Tdb_No, Mode); + htrc("OpenFix: R%hd mode=%d BlkFil=%p\n", Tdb_No, Mode, To_BlkFil); /*********************************************************************/ /* Reset buffer access according to indexing and to mode. */ diff --git a/storage/connect/table.cpp b/storage/connect/table.cpp index f550420ef3a..6cd043ec9a9 100644 --- a/storage/connect/table.cpp +++ b/storage/connect/table.cpp @@ -314,10 +314,14 @@ int TDBASE::ResetTableOpt(PGLOBAL g, bool dop, bool dox) /***********************************************************************/ /* SetKindex: set or reset the index pointer. */ /***********************************************************************/ -void TDBASE::SetKindex(PKXBASE kxp) +void TDBASE::SetKindex(PGLOBAL g, PKXBASE kxp) { - if (To_Kindex) + if (To_Kindex) { + int pos = GetRecpos(); // To be reset in Txfp + To_Kindex->Close(); // Discard old index + SetRecpos(g, pos); // Ignore return value + } // endif To_Kindex To_Kindex = kxp; } // end of SetKindex diff --git a/storage/connect/valblk.h b/storage/connect/valblk.h index 3ff34c4bcdf..67d1c5a27ee 100644 --- a/storage/connect/valblk.h +++ b/storage/connect/valblk.h @@ -22,36 +22,36 @@ DllExport PVBLK AllocValBlock(PGLOBAL, void*, int, int, int, int, bool, bool, bool); const char *GetFmt(int type, bool un = false); -/***********************************************************************/ -/* DB static external variables. */ -/***********************************************************************/ -extern MBLOCK Nmblk; /* Used to initialize MBLOCK's */ - -/***********************************************************************/ -/* Class MBVALS is a utility class for (re)allocating VALBLK's. */ -/***********************************************************************/ -class MBVALS : public BLOCK { -//friend class LSTBLK; - friend class ARRAY; - public: - // Constructors - MBVALS(void) {Vblk = NULL; Mblk = Nmblk;} - - // Methods - void *GetMemp(void) {return Mblk.Memp;} - PVBLK Allocate(PGLOBAL g, int type, int len, int prec, - int n, bool sub = FALSE); - bool ReAllocate(PGLOBAL g, int n); - void Free(void); - - protected: - // Members - PVBLK Vblk; // Pointer to VALBLK - MBLOCK Mblk; // The memory block - }; // end of class MBVALS - -typedef class MBVALS *PMBV; - +/***********************************************************************/ +/* DB static external variables. */ +/***********************************************************************/ +extern MBLOCK Nmblk; /* Used to initialize MBLOCK's */ + +/***********************************************************************/ +/* Class MBVALS is a utility class for (re)allocating VALBLK's. */ +/***********************************************************************/ +class MBVALS : public BLOCK { +//friend class LSTBLK; + friend class ARRAY; + public: + // Constructors + MBVALS(void) {Vblk = NULL; Mblk = Nmblk;} + + // Methods + void *GetMemp(void) {return Mblk.Memp;} + PVBLK Allocate(PGLOBAL g, int type, int len, int prec, + int n, bool sub = FALSE); + bool ReAllocate(PGLOBAL g, int n); + void Free(void); + + protected: + // Members + PVBLK Vblk; // Pointer to VALBLK + MBLOCK Mblk; // The memory block + }; // end of class MBVALS + +typedef class MBVALS *PMBV; + /***********************************************************************/ /* Class VALBLK represent a base class for variable blocks. */ /***********************************************************************/ diff --git a/storage/connect/xindex.cpp b/storage/connect/xindex.cpp index 50d19ac67fc..52febd99687 100755 --- a/storage/connect/xindex.cpp +++ b/storage/connect/xindex.cpp @@ -242,7 +242,7 @@ void XINDEX::Reset(void) /***********************************************************************/ /* XINDEX Close: terminate index and free all allocated data. */ -/* Do not reset other values that are used at return to make. */ +/* Do not reset values that are used at return to make. */ /***********************************************************************/ void XINDEX::Close(void) { @@ -264,6 +264,9 @@ void XINDEX::Close(void) kcp->FreeData(); } // endfor kcp + if (Tdbp) + Tdbp->RestoreNrec(); + } // end of Close /***********************************************************************/ @@ -288,7 +291,7 @@ int XINDEX::Qcompare(int *i1, int *i2) /* Sure enough, it is done while records are read and permit to avoid */ /* reading the table while doing the join (Dynamic index only) */ /***********************************************************************/ -bool XINDEX::AddColumns(void) +bool XINDEX::AddColumns(PIXDEF xdp) { if (!Dynamic) return false; // Not applying to static index @@ -309,7 +312,7 @@ bool XINDEX::Make(PGLOBAL g, PIXDEF sxp) /*********************************************************************/ /* Table can be accessed through an index. */ /*********************************************************************/ - int k, rc = RC_OK; + int k, nk = Nk, rc = RC_OK; int *bof, i, j, n, ndf, nkey; PKPDEF kdfp = Xdp->GetToKeyParts(); bool brc = false; @@ -378,7 +381,7 @@ bool XINDEX::Make(PGLOBAL g, PIXDEF sxp) To_LastCol = prev; - if (AddColumns()) { + if (AddColumns(sxp)) { PCOL kolp = To_Cols[0]; // Temporary while imposing Nk = 1 i = 0; @@ -415,6 +418,7 @@ bool XINDEX::Make(PGLOBAL g, PIXDEF sxp) htrc("Adding colp=%p Buf_Type=%d size=%d\n", colp, colp->GetResultType(), n); + nk++; prev->Next = kcp; prev = kcp; } // endfor colp @@ -481,9 +485,10 @@ bool XINDEX::Make(PGLOBAL g, PIXDEF sxp) /* Get the keys and place them in the key blocks. */ /*******************************************************************/ for (k = 0, kcp = To_KeyCol; - k < Nk && kcp; + k < nk && kcp; k++, kcp = kcp->Next) { - colp = To_Cols[k]; +// colp = To_Cols[k]; + colp = kcp->Colp; if (!colp->GetStatus(BUF_READ)) colp->ReadColumn(g); @@ -542,6 +547,10 @@ bool XINDEX::Make(PGLOBAL g, PIXDEF sxp) if ((Ndif = Qsort(g, Num_K)) < 0) goto err; // Error during sort +// if (trace) + htrc("Make: Nk=%d n=%d Num_K=%d Ndif=%d addcolp=%p BlkFil=%p X=%p\n", + Nk, n, Num_K, Ndif, addcolp, Tdbp->To_BlkFil, X); + // Check whether the unique index is unique indeed if (!Mul) if (Ndif < Num_K) { @@ -2210,7 +2219,10 @@ int XINDXS::FastFind(int nk) n = 0; } // endif sup - kcp->Val_K = i; // Used by FillValue + // Loop on kcp because of dynamic indexing + for (; kcp; kcp = kcp->Next) + kcp->Val_K = i; // Used by FillValue + return ((n) ? Num_K : (Mul) ? Pof[i] : i); } // end of FastFind diff --git a/storage/connect/xindex.h b/storage/connect/xindex.h index 75ced0c85a8..c8b7f27de22 100644 --- a/storage/connect/xindex.h +++ b/storage/connect/xindex.h @@ -278,7 +278,7 @@ class DllExport XINDEX : public XXBASE { bool GetAllSizes(PGLOBAL g, int &ndif, int &numk); protected: - bool AddColumns(void); + bool AddColumns(PIXDEF xdp); bool NextValDif(void); // Members diff --git a/storage/connect/xtable.h b/storage/connect/xtable.h index f6c5a0fbfdd..10d89307be5 100644 --- a/storage/connect/xtable.h +++ b/storage/connect/xtable.h @@ -148,7 +148,7 @@ class DllExport TDBASE : public TDB { inline void SetXdp(PIXDEF xdp) {To_Xdp = xdp;} // Properties - void SetKindex(PKXBASE kxp); + void SetKindex(PGLOBAL g, PKXBASE kxp); PCOL Key(int i) {return (To_Key_Col) ? To_Key_Col[i] : NULL;} // Methods From 03819e5ec7205d819cab08c0e2a074ca6adbf3ea Mon Sep 17 00:00:00 2001 From: unknown Date: Mon, 5 May 2014 12:15:12 +0200 Subject: [PATCH 008/279] Raise version number after cloning 5.5.38 --- VERSION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/VERSION b/VERSION index a519214c0c5..67a35621511 100644 --- a/VERSION +++ b/VERSION @@ -1,4 +1,4 @@ MYSQL_VERSION_MAJOR=5 MYSQL_VERSION_MINOR=5 -MYSQL_VERSION_PATCH=38 +MYSQL_VERSION_PATCH=39 MYSQL_VERSION_EXTRA= From d32a4b93253be7e3764057dc28ad87c9c74c5454 Mon Sep 17 00:00:00 2001 From: Tor Didriksen Date: Mon, 5 May 2014 16:39:14 +0200 Subject: [PATCH 009/279] Backport from trunk: Bug #18593044 COMPILE FLAGS NOT PASSED TO DTRACE, BREAKS CROSS BUILD --- cmake/dtrace.cmake | 43 +++++++++++++++++++++++-------------------- 1 file changed, 23 insertions(+), 20 deletions(-) diff --git a/cmake/dtrace.cmake b/cmake/dtrace.cmake index 36d948a417a..66b07c03e9b 100644 --- a/cmake/dtrace.cmake +++ b/cmake/dtrace.cmake @@ -1,4 +1,4 @@ -# Copyright (c) 2009, 2013, Oracle and/or its affiliates. All rights reserved. +# Copyright (c) 2009, 2014, Oracle and/or its affiliates. All rights reserved. # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -74,13 +74,6 @@ IF(ENABLE_DTRACE) ${CMAKE_BINARY_DIR}/include/probes_mysql_dtrace.h ${CMAKE_BINARY_DIR}/include/probes_mysql_nodtrace.h ) - IF(CMAKE_SYSTEM_NAME MATCHES "Linux") - # Systemtap object - EXECUTE_PROCESS( - COMMAND ${DTRACE} -G -s ${CMAKE_SOURCE_DIR}/include/probes_mysql.d.base - -o ${CMAKE_BINARY_DIR}/probes_mysql.o - ) - ENDIF() ADD_CUSTOM_TARGET(gen_dtrace_header DEPENDS ${CMAKE_BINARY_DIR}/include/probes_mysql.d @@ -99,12 +92,7 @@ FUNCTION(DTRACE_INSTRUMENT target) IF(ENABLE_DTRACE) ADD_DEPENDENCIES(${target} gen_dtrace_header) - IF(CMAKE_SYSTEM_NAME MATCHES "Linux") - TARGET_LINK_LIBRARIES(${target} ${CMAKE_BINARY_DIR}/probes_mysql.o) - ENDIF() - - # On Solaris, invoke dtrace -G to generate object file and - # link it together with target. + # Invoke dtrace to generate object file and link it together with target. IF(CMAKE_SYSTEM_NAME MATCHES "SunOS") SET(objdir ${CMAKE_CURRENT_BINARY_DIR}/CMakeFiles/${target}.dir) SET(outfile ${objdir}/${target}_dtrace.o) @@ -121,6 +109,21 @@ FUNCTION(DTRACE_INSTRUMENT target) -P ${CMAKE_SOURCE_DIR}/cmake/dtrace_prelink.cmake WORKING_DIRECTORY ${objdir} ) + ELSEIF(CMAKE_SYSTEM_NAME MATCHES "Linux") + # dtrace on Linux runs gcc and uses flags from environment + SET(CFLAGS_SAVED $ENV{CFLAGS}) + SET(ENV{CFLAGS} ${CMAKE_C_FLAGS}) + SET(outfile "${CMAKE_BINARY_DIR}/probes_mysql.o") + # Systemtap object + EXECUTE_PROCESS( + COMMAND ${DTRACE} -G -s ${CMAKE_SOURCE_DIR}/include/probes_mysql.d.base + -o ${outfile} + ) + SET(ENV{CFLAGS} ${CFLAGS_SAVED}) + ENDIF() + + # Do not try to extend the library if we have not built the .o file + IF(outfile) # Add full object path to linker flags GET_TARGET_PROPERTY(target_type ${target} TYPE) IF(NOT target_type MATCHES "STATIC") @@ -132,12 +135,12 @@ FUNCTION(DTRACE_INSTRUMENT target) # but maybe one day this will be fixed. GET_TARGET_PROPERTY(target_location ${target} LOCATION) ADD_CUSTOM_COMMAND( - TARGET ${target} POST_BUILD - COMMAND ${CMAKE_AR} r ${target_location} ${outfile} - COMMAND ${CMAKE_RANLIB} ${target_location} - ) - # Used in DTRACE_INSTRUMENT_WITH_STATIC_LIBS - SET(TARGET_OBJECT_DIRECTORY_${target} ${objdir} CACHE INTERNAL "") + TARGET ${target} POST_BUILD + COMMAND ${CMAKE_AR} r ${target_location} ${outfile} + COMMAND ${CMAKE_RANLIB} ${target_location} + ) + # Used in DTRACE_INSTRUMENT_WITH_STATIC_LIBS + SET(TARGET_OBJECT_DIRECTORY_${target} ${objdir} CACHE INTERNAL "") ENDIF() ENDIF() ENDIF() From b6283d4f97628134e9010424cb4748e801a25cb8 Mon Sep 17 00:00:00 2001 From: Venkatesh Duggirala Date: Mon, 5 May 2014 22:22:15 +0530 Subject: [PATCH 010/279] Bug#17638477 UNINSTALL AND INSTALL SEMI-SYNC PLUGIN CAUSES SLAVES TO BREAK Problem: Uninstallation of semi sync plugin causes replication to break. Analysis: A semisync enabled replication is mutual agreement between Master and Slave when the connection (I/O thread) is established. Once I/O thread is started and if semisync is enabled on both master and slave, master appends special magic header to events using semisync plugin functions and sends it to slave. And slave expects that each event will have that special magic header format and reads those bytes using semisync plugin functions. When semi sync replication is in use if users execute uninstallation of the plugin on master, slave gets confused while interpreting that event's content because it expects special magic header at the beginning of the event. Slave SQL thread will be stopped with "Missing magic number in the header" error. Similar problem will happen if uninstallation of the plugin happens on slave when semi sync replication is in in use. Master sends the events with magic header and slave does not know about the added magic header and thinks that it received a corrupted event. Hence slave SQL thread stops with "Found corrupted event" error. Fix: Uninstallation of semisync plugin will be blocked when semisync replication is in use and will throw 'ER_UNKNOWN_ERROR' error. To detect that semisync replication is in use, this patch uses semisync status variable values. > On Master, it checks for 'Rpl_semi_sync_master_status' to be OFF before allowing the uninstallation of rpl_semi_sync_master plugin. >> Rpl_semi_sync_master_status is OFF when >>> there is no dump thread running >>> there are no semisync slaves > On Slave, it checks for 'Rpl_semi_sync_slave_status' to be OFF before allowing the uninstallation of rpl_semi_sync_slave plugin. >> Rpl_semi_sync_slave_status is OFF when >>> there is no I/O thread running >>> replication is asynchronous replication. --- mysql-test/include/stop_dump_threads.inc | 32 ++++ mysql-test/include/uninstall_semisync.inc | 5 + mysql-test/suite/rpl/r/rpl_semi_sync.result | 3 +- .../r/rpl_semi_sync_uninstall_plugin.result | 36 +++++ mysql-test/suite/rpl/t/rpl_semi_sync.test | 11 +- .../rpl_semi_sync_uninstall_plugin-master.opt | 1 + .../rpl_semi_sync_uninstall_plugin-slave.opt | 1 + .../rpl/t/rpl_semi_sync_uninstall_plugin.test | 143 ++++++++++++++++++ sql/sql_plugin.cc | 43 ++++++ sql/sql_show.cc | 53 +++++++ sql/sql_show.h | 1 + 11 files changed, 317 insertions(+), 12 deletions(-) create mode 100644 mysql-test/include/stop_dump_threads.inc create mode 100644 mysql-test/suite/rpl/r/rpl_semi_sync_uninstall_plugin.result create mode 100644 mysql-test/suite/rpl/t/rpl_semi_sync_uninstall_plugin-master.opt create mode 100644 mysql-test/suite/rpl/t/rpl_semi_sync_uninstall_plugin-slave.opt create mode 100644 mysql-test/suite/rpl/t/rpl_semi_sync_uninstall_plugin.test diff --git a/mysql-test/include/stop_dump_threads.inc b/mysql-test/include/stop_dump_threads.inc new file mode 100644 index 00000000000..ae33c963d9a --- /dev/null +++ b/mysql-test/include/stop_dump_threads.inc @@ -0,0 +1,32 @@ +# ==== Purpose ==== +# +# Stop all dump threads on the server of the current connection. +# +# ==== Usage ==== +# +# --source include/stop_dump_threads.inc + +--let $include_filename= stop_dump_threads.inc +--source include/begin_include_file.inc + + +--let $_sdt_show_rpl_debug_info_old= $show_rpl_debug_info +--let $show_rpl_debug_info= 1 +--disable_query_log +--disable_result_log + +--let $_sdt_dump_thread_id= `SELECT ID FROM INFORMATION_SCHEMA.PROCESSLIST WHERE COMMAND = 'Binlog dump'` + +while ($_sdt_dump_thread_id != '') +{ + eval KILL $_sdt_dump_thread_id; + --let $wait_condition= SELECT COUNT(*) = 0 FROM INFORMATION_SCHEMA.PROCESSLIST WHERE ID = $_sdt_dump_thread_id + --source include/wait_condition.inc + + --let $_sdt_dump_thread_id= `SELECT ID FROM INFORMATION_SCHEMA.PROCESSLIST WHERE COMMAND = 'Binlog dump'` +} + +--let $show_rpl_debug_info= $_sdt_show_rpl_debug_info_old + +--let $include_filename= stop_dump_threads.inc +--source include/end_include_file.inc diff --git a/mysql-test/include/uninstall_semisync.inc b/mysql-test/include/uninstall_semisync.inc index 11668d1db97..0a4c55fa4f2 100644 --- a/mysql-test/include/uninstall_semisync.inc +++ b/mysql-test/include/uninstall_semisync.inc @@ -13,6 +13,11 @@ UNINSTALL PLUGIN rpl_semi_sync_slave; --connection master +# After BUG#17638477 fix, uninstallation of rpl_semi_sync_master +# is not allowed when there are semi sync slaves. Hence kill +# all dump threads before uninstalling it. +SET GLOBAL rpl_semi_sync_master_enabled = OFF; +--source include/stop_dump_threads.inc UNINSTALL PLUGIN rpl_semi_sync_master; --enable_warnings diff --git a/mysql-test/suite/rpl/r/rpl_semi_sync.result b/mysql-test/suite/rpl/r/rpl_semi_sync.result index 0377716698c..8c6cb58324a 100644 --- a/mysql-test/suite/rpl/r/rpl_semi_sync.result +++ b/mysql-test/suite/rpl/r/rpl_semi_sync.result @@ -439,9 +439,8 @@ Rpl_semi_sync_slave_status OFF # # Clean up # +include/uninstall_semisync.inc include/stop_slave.inc -UNINSTALL PLUGIN rpl_semi_sync_slave; -UNINSTALL PLUGIN rpl_semi_sync_master; change master to master_user='root',master_password=''; include/start_slave.inc drop table t1; diff --git a/mysql-test/suite/rpl/r/rpl_semi_sync_uninstall_plugin.result b/mysql-test/suite/rpl/r/rpl_semi_sync_uninstall_plugin.result new file mode 100644 index 00000000000..d5cf690c978 --- /dev/null +++ b/mysql-test/suite/rpl/r/rpl_semi_sync_uninstall_plugin.result @@ -0,0 +1,36 @@ +include/master-slave.inc +[connection master] +INSTALL PLUGIN rpl_semi_sync_master SONAME 'semisync_master.so'; +INSTALL PLUGIN rpl_semi_sync_slave SONAME 'semisync_slave.so'; +UNINSTALL PLUGIN rpl_semi_sync_slave; +UNINSTALL PLUGIN rpl_semi_sync_master; +CREATE TABLE t1(i int); +INSERT INTO t1 values (1); +DROP TABLE t1; +include/install_semisync.inc +call mtr.add_suppression("Plugin 'rpl_semi_sync_slave' cannot be uninstalled now"); +UNINSTALL PLUGIN rpl_semi_sync_slave; +ERROR HY000: Unknown error +call mtr.add_suppression("Plugin 'rpl_semi_sync_master' cannot be uninstalled now"); +UNINSTALL PLUGIN rpl_semi_sync_master; +ERROR HY000: Unknown error +CREATE TABLE t1(i int); +INSERT INTO t1 values (2); +DROP TABLE t1; +include/assert.inc [semi sync slave status should be ON.] +include/assert.inc [semi sync master status should be ON.] +include/assert.inc [semi sync master clients should be 1.] +SET GLOBAL rpl_semi_sync_master_enabled = OFF; +include/assert.inc [semi sync master clients should be 1.] +UNINSTALL PLUGIN rpl_semi_sync_master; +ERROR HY000: Unknown error +include/stop_slave.inc +SET GLOBAL rpl_semi_sync_slave_enabled = OFF; +include/start_slave.inc +UNINSTALL PLUGIN rpl_semi_sync_slave; +include/assert.inc [semi sync master clients should be 0.] +UNINSTALL PLUGIN rpl_semi_sync_master; +CREATE TABLE t1(i int); +INSERT INTO t1 values (3); +DROP TABLE t1; +include/rpl_end.inc diff --git a/mysql-test/suite/rpl/t/rpl_semi_sync.test b/mysql-test/suite/rpl/t/rpl_semi_sync.test index 21967fb6f8c..b78bb772fc9 100644 --- a/mysql-test/suite/rpl/t/rpl_semi_sync.test +++ b/mysql-test/suite/rpl/t/rpl_semi_sync.test @@ -598,19 +598,10 @@ SHOW STATUS LIKE 'Rpl_semi_sync_slave_status'; --echo # --echo # Clean up --echo # +--source include/uninstall_semisync.inc connection slave; source include/stop_slave.inc; -UNINSTALL PLUGIN rpl_semi_sync_slave; - -connection master; -# The dump thread may still be running on the master, and so the following -# UNINSTALL could generate a warning about the plugin is busy. -disable_warnings; -UNINSTALL PLUGIN rpl_semi_sync_master; -enable_warnings; - -connection slave; change master to master_user='root',master_password=''; source include/start_slave.inc; diff --git a/mysql-test/suite/rpl/t/rpl_semi_sync_uninstall_plugin-master.opt b/mysql-test/suite/rpl/t/rpl_semi_sync_uninstall_plugin-master.opt new file mode 100644 index 00000000000..58029d28ace --- /dev/null +++ b/mysql-test/suite/rpl/t/rpl_semi_sync_uninstall_plugin-master.opt @@ -0,0 +1 @@ +$SEMISYNC_PLUGIN_OPT diff --git a/mysql-test/suite/rpl/t/rpl_semi_sync_uninstall_plugin-slave.opt b/mysql-test/suite/rpl/t/rpl_semi_sync_uninstall_plugin-slave.opt new file mode 100644 index 00000000000..58029d28ace --- /dev/null +++ b/mysql-test/suite/rpl/t/rpl_semi_sync_uninstall_plugin-slave.opt @@ -0,0 +1 @@ +$SEMISYNC_PLUGIN_OPT diff --git a/mysql-test/suite/rpl/t/rpl_semi_sync_uninstall_plugin.test b/mysql-test/suite/rpl/t/rpl_semi_sync_uninstall_plugin.test new file mode 100644 index 00000000000..094e044e0d2 --- /dev/null +++ b/mysql-test/suite/rpl/t/rpl_semi_sync_uninstall_plugin.test @@ -0,0 +1,143 @@ +############################################################################### +# Bug#17638477 UNINSTALL AND INSTALL SEMI-SYNC PLUGIN CAUSES SLAVES TO BREAK +# Problem: Uninstallation of Semi sync plugin should be blocked when it is +# in use. +# Test case: Uninstallation of semi sync should be allowed +# On Master: +# 1) When there is no dump thread +# 2) When there are no semi sync slaves (i.e., async replication). +# On Slave: +# 1) When there is no I/O thread +# 2) When there are no semi sync enabled I/O thread (i.e.,async replication). +############################################################################### + +--source include/have_semisync_plugin.inc +--source include/not_embedded.inc +--source include/have_binlog_format_statement.inc +--source include/master-slave.inc + +############################################################################### +# Case 1: Uninstallation of semi sync plugins should be allowed when it is +# not in use i.e., when asynchronous replication is active. +############################################################################### +# Step 1.1: Install semi sync master plugin on master +eval INSTALL PLUGIN rpl_semi_sync_master SONAME '$SEMISYNC_MASTER_PLUGIN'; + +# Step 1.2: Install semi sync slave plugin on slave +--connection slave +eval INSTALL PLUGIN rpl_semi_sync_slave SONAME '$SEMISYNC_SLAVE_PLUGIN'; + +# Step 1.3: Uninstallation of semisync plugin on master and slave should be +# allowed at this state as there is no semi sync replication enabled between +# master and slave. +UNINSTALL PLUGIN rpl_semi_sync_slave; +--connection master +UNINSTALL PLUGIN rpl_semi_sync_master; + +# Step 1.4: Check that replication is working fine at the end of the test case. +CREATE TABLE t1(i int); +INSERT INTO t1 values (1); +DROP TABLE t1; +--sync_slave_with_master + +############################################################################### +# Case 2: Uninstallation of semi sync plugins should be disallowed +# when it is in use i.e., when semi sync replication is active +############################################################################### +# Step 2.1: Install and enable semi sync replication between master and slave +--source include/install_semisync.inc + +# Step 2.2: Check that rpl_semi_sync_slave uninstallation on Slave is not +# possible at this state +--connection slave +call mtr.add_suppression("Plugin 'rpl_semi_sync_slave' cannot be uninstalled now"); +--error ER_UNKNOWN_ERROR +UNINSTALL PLUGIN rpl_semi_sync_slave; + +# Step 2.3: Check that rpl_semi_sync_master uninstallation on Master is not +# possible at this state +--connection master +call mtr.add_suppression("Plugin 'rpl_semi_sync_master' cannot be uninstalled now"); +--error ER_UNKNOWN_ERROR +UNINSTALL PLUGIN rpl_semi_sync_master; + +# Step 2.4: Check that replication is working fine at the end of the test case. +CREATE TABLE t1(i int); +INSERT INTO t1 values (2); +DROP TABLE t1; +--sync_slave_with_master + +# Step 2.5: Make sure rpl_semi_sync_master_status on Master and +# rpl_semi_sync_slave_staus on Slave are ON +--let $slave_status=[show status like "Rpl_semi_sync_slave_status", Value, 1] +--let assert_cond= "$slave_status" = "ON" +--let assert_text= semi sync slave status should be ON. +--source include/assert.inc + +--connection master +--let $master_status=[show status like "Rpl_semi_sync_master_status", Value, 1] +--let assert_cond= "$master_status" = "ON" +--let assert_text= semi sync master status should be ON. +--source include/assert.inc + +--let $master_clients=[show status like "Rpl_semi_sync_master_clients", Value, 1] +--let assert_cond= $master_clients = 1 +--let assert_text= semi sync master clients should be 1. +--source include/assert.inc + +############################################################################### +# Case 3: Uninstallation of semi sync plugin should be disallowed when there +# are semi sync slaves even though rpl_semi_sync_master_enabled= OFF;. +############################################################################### +# Step 3.1: Disable semi sync on master +--connection master +SET GLOBAL rpl_semi_sync_master_enabled = OFF; + +# Step 3.2: Check that still Rpl_semi_sync_master_clients is 1 +--let $master_clients=[show status like "Rpl_semi_sync_master_clients", Value, 1] +--let assert_cond= $master_clients = 1 +--let assert_text= semi sync master clients should be 1. +--source include/assert.inc + +# Step 3.3: Since Rpl_semi_sync_master_clients is 1, uninstallation of +# rpl_semi_sync_master should be disallowed. +--error ER_UNKNOWN_ERROR +UNINSTALL PLUGIN rpl_semi_sync_master; + +############################################################################### +# Case 4: Uninstallation of semi sync plugin should be allowed when it is not +# in use. Same as Case 1 but this case is to check the case after enabling and +# disabling semi sync replication. +############################################################################### + +# Step 4.1: Stop IO thread on slave. +--connection slave +--source include/stop_slave.inc + +# Step 4.2: Disable semi sync on slave. +SET GLOBAL rpl_semi_sync_slave_enabled = OFF; + +# Step 4.3: Start IO thread on slave. +--source include/start_slave.inc + +# Step 4.4: Uninstall semi sync plugin, it should be successful now. +UNINSTALL PLUGIN rpl_semi_sync_slave; + +# Step 4.5: On Master, check that semi sync slaves are now '0'. +--connection master +--let $master_clients=[show status like "Rpl_semi_sync_master_clients", Value, 1] +--let assert_cond= $master_clients = 0 +--let assert_text= semi sync master clients should be 0. +--source include/assert.inc + +# Step 4.6: So uninstalling semi sync plugin should be allowed +UNINSTALL PLUGIN rpl_semi_sync_master; + +# Step 4.7: Check that replication is working fine at the end of the test case +CREATE TABLE t1(i int); +INSERT INTO t1 values (3); +DROP TABLE t1; +--sync_slave_with_master + +# Cleanup +source include/rpl_end.inc; diff --git a/sql/sql_plugin.cc b/sql/sql_plugin.cc index d54326af7aa..9851cb2739a 100644 --- a/sql/sql_plugin.cc +++ b/sql/sql_plugin.cc @@ -1930,6 +1930,49 @@ bool mysql_uninstall_plugin(THD *thd, const LEX_STRING *name) goto err; } +#ifdef HAVE_REPLICATION + /* Block Uninstallation of semi_sync plugins (Master/Slave) + when they are busy + */ + char buff[20]; + /* + Master: If there are active semi sync slaves for this Master, + then that means it is busy and rpl_semi_sync_master plugin + cannot be uninstalled. To check whether the master + has any semi sync slaves or not, check Rpl_semi_sync_master_cliens + status variable value, if it is not 0, that means it is busy. + */ + if (!strcmp(name->str, "rpl_semi_sync_master") && + get_status_var(thd, + plugin->plugin->status_vars, + "Rpl_semi_sync_master_clients",buff) && + strcmp(buff,"0") ) + { + sql_print_error("Plugin 'rpl_semi_sync_master' cannot be uninstalled now. " + "Stop any active semisynchronous slaves of this master " + "first.\n"); + my_error(ER_UNKNOWN_ERROR, MYF(0), name->str); + goto err; + } + /* Slave: If there is semi sync enabled IO thread active on this Slave, + then that means plugin is busy and rpl_semi_sync_slave plugin + cannot be uninstalled. To check whether semi sync + IO thread is active or not, check Rpl_semi_sync_slave_status status + variable value, if it is ON, that means it is busy. + */ + if (!strcmp(name->str, "rpl_semi_sync_slave") && + get_status_var(thd, plugin->plugin->status_vars, + "Rpl_semi_sync_slave_status", buff) && + !strcmp(buff,"ON") ) + { + sql_print_error("Plugin 'rpl_semi_sync_slave' cannot be uninstalled now. " + "Stop any active semisynchronous I/O threads on this slave " + "first.\n"); + my_error(ER_UNKNOWN_ERROR, MYF(0), name->str); + goto err; + } +#endif + plugin->state= PLUGIN_IS_DELETED; if (plugin->ref_count) push_warning(thd, MYSQL_ERROR::WARN_LEVEL_WARN, diff --git a/sql/sql_show.cc b/sql/sql_show.cc index dcae4c63b02..ac63b2aaff9 100644 --- a/sql/sql_show.cc +++ b/sql/sql_show.cc @@ -2117,6 +2117,59 @@ void free_status_vars() delete_dynamic(&all_status_vars); } +/** + @brief Get the value of given status variable + + @param[in] thd thread handler + @param[in] list list of SHOW_VAR objects in which function should + search + @param[in] name name of the status variable + @param[in/out] value buffer in which value of the status variable + needs to be filled in + + @return status + @retval FALSE if variable is not found in the list + @retval TRUE if variable is found in the list + NOTE: Currently this function is implemented just to support 'bool' status + variables and 'long' status variables *only*. It can be extended very easily + for further show_types in future if required. + TODO: Currently show_status_arary switch case is tightly coupled with + pos, end, buff, value variables and also it stores the values in a 'table'. + Decouple the switch case to fill the buffer value so that it can be used + in show_status_array() and get_status_var() to avoid duplicate code. + */ + +bool get_status_var(THD* thd, SHOW_VAR *list, const char * name, char * const value) +{ + for (; list->name; list++) + { + int res= strcmp(list->name, name); + if (res == 0) + { + /* + if var->type is SHOW_FUNC, call the function. + Repeat as necessary, if new var is again SHOW_FUNC + */ + SHOW_VAR tmp; + for (; list->type == SHOW_FUNC; list= &tmp) + ((mysql_show_var_func)(list->value))(thd, &tmp, value); + switch (list->type) { + case SHOW_BOOL: + strmov(value, *(bool*) list->value ? "ON" : "OFF"); + break; + case SHOW_LONG: + int10_to_str(*(long*) list->value, value, 10); + break; + default: + /* not supported type */ + DBUG_ASSERT(0); + } + return TRUE; + } + } + return FALSE; +} + /* Removes an array of SHOW_VAR entries from the output of SHOW STATUS diff --git a/sql/sql_show.h b/sql/sql_show.h index 406c75d8cbe..b6d520441c7 100644 --- a/sql/sql_show.h +++ b/sql/sql_show.h @@ -113,6 +113,7 @@ int add_status_vars(SHOW_VAR *list); void remove_status_vars(SHOW_VAR *list); void init_status_vars(); void free_status_vars(); +bool get_status_var(THD* thd, SHOW_VAR *list, const char *name, char * const buff); void reset_status_vars(); bool show_create_trigger(THD *thd, const sp_name *trg_name); void view_store_options(THD *thd, TABLE_LIST *table, String *buff); From a2333a489a59ba702afbcf4e1f0c44706cf50f38 Mon Sep 17 00:00:00 2001 From: Venkatesh Duggirala Date: Tue, 6 May 2014 11:23:42 +0530 Subject: [PATCH 011/279] Bug#17638477 UNINSTALL AND INSTALL SEMI-SYNC PLUGIN CAUSES SLAVES TO BREAK Fixing post push failure --- mysql-test/suite/rpl/r/rpl_semi_sync.result | 1 + mysql-test/suite/rpl/t/rpl_semi_sync.test | 1 + 2 files changed, 2 insertions(+) diff --git a/mysql-test/suite/rpl/r/rpl_semi_sync.result b/mysql-test/suite/rpl/r/rpl_semi_sync.result index 8c6cb58324a..805af374df3 100644 --- a/mysql-test/suite/rpl/r/rpl_semi_sync.result +++ b/mysql-test/suite/rpl/r/rpl_semi_sync.result @@ -386,6 +386,7 @@ Rpl_semi_sync_slave_status ON include/stop_slave.inc [ on master ] set sql_log_bin=0; +include/stop_dump_threads.inc UNINSTALL PLUGIN rpl_semi_sync_master; set sql_log_bin=1; SHOW VARIABLES LIKE 'rpl_semi_sync_master_enabled'; diff --git a/mysql-test/suite/rpl/t/rpl_semi_sync.test b/mysql-test/suite/rpl/t/rpl_semi_sync.test index b78bb772fc9..ba87576717b 100644 --- a/mysql-test/suite/rpl/t/rpl_semi_sync.test +++ b/mysql-test/suite/rpl/t/rpl_semi_sync.test @@ -545,6 +545,7 @@ source include/stop_slave.inc; connection master; echo [ on master ]; set sql_log_bin=0; +--source include/stop_dump_threads.inc UNINSTALL PLUGIN rpl_semi_sync_master; set sql_log_bin=1; enable_query_log; From 548db492101346bf582295544c9a7a42d3308641 Mon Sep 17 00:00:00 2001 From: Mattias Jonsson Date: Tue, 6 May 2014 11:05:37 +0200 Subject: [PATCH 012/279] Bug#17909699: WRONG RESULTS WITH PARTITION BY LIST COLUMNS() Typo leading to not including the last list values (partition). Also improved pruning to skip last partition if not used. rb#4762 approved by Aditya and Marko. --- mysql-test/r/partition_pruning.result | 114 ++++++++++++++++++++++++++ mysql-test/t/partition_pruning.test | 48 +++++++++++ sql/sql_partition.cc | 39 +++++---- 3 files changed, 184 insertions(+), 17 deletions(-) diff --git a/mysql-test/r/partition_pruning.result b/mysql-test/r/partition_pruning.result index fde7a3b8104..3d572726507 100644 --- a/mysql-test/r/partition_pruning.result +++ b/mysql-test/r/partition_pruning.result @@ -3294,3 +3294,117 @@ explain partitions select * from t1 where a between 10 and 10+33; id select_type table partitions type possible_keys key key_len ref rows Extra 1 SIMPLE t1 p0,p1,p2 ALL NULL NULL NULL NULL 100 Using where drop table t0, t1; +# +# Bug#71095: Wrong results with PARTITION BY LIST COLUMNS() +# +CREATE TABLE t1 +(c1 int, +c2 int, +c3 int, +c4 int, +PRIMARY KEY (c1,c2)) +PARTITION BY LIST COLUMNS (c2) +(PARTITION p1 VALUES IN (1,2), +PARTITION p2 VALUES IN (3,4)); +INSERT INTO t1 VALUES (1, 1, 1, 1), (2, 3, 1, 1); +INSERT INTO t1 VALUES (1, 2, 1, 1), (2, 4, 1, 1); +SELECT * FROM t1 WHERE c1 = 1 AND c2 < 1; +c1 c2 c3 c4 +SELECT * FROM t1 WHERE c1 = 1 AND c2 <= 1; +c1 c2 c3 c4 +1 1 1 1 +SELECT * FROM t1 WHERE c1 = 1 AND c2 = 1; +c1 c2 c3 c4 +1 1 1 1 +SELECT * FROM t1 WHERE c1 = 1 AND c2 >= 1; +c1 c2 c3 c4 +1 1 1 1 +1 2 1 1 +SELECT * FROM t1 WHERE c1 = 1 AND c2 > 1; +c1 c2 c3 c4 +1 2 1 1 +SELECT * FROM t1 WHERE c1 = 1 AND c2 < 3; +c1 c2 c3 c4 +1 1 1 1 +1 2 1 1 +SELECT * FROM t1 WHERE c1 = 1 AND c2 <= 3; +c1 c2 c3 c4 +1 1 1 1 +1 2 1 1 +SELECT * FROM t1 WHERE c1 = 2 AND c2 <= 3; +c1 c2 c3 c4 +2 3 1 1 +SELECT * FROM t1 WHERE c1 = 2 AND c2 = 3; +c1 c2 c3 c4 +2 3 1 1 +SELECT * FROM t1 WHERE c1 = 2 AND c2 >= 3; +c1 c2 c3 c4 +2 3 1 1 +2 4 1 1 +SELECT * FROM t1 WHERE c1 = 2 AND c2 > 3; +c1 c2 c3 c4 +2 4 1 1 +SELECT * FROM t1 WHERE c1 = 2 AND c2 < 4; +c1 c2 c3 c4 +2 3 1 1 +SELECT * FROM t1 WHERE c1 = 2 AND c2 <= 4; +c1 c2 c3 c4 +2 3 1 1 +2 4 1 1 +SELECT * FROM t1 WHERE c1 = 2 AND c2 = 4; +c1 c2 c3 c4 +2 4 1 1 +SELECT * FROM t1 WHERE c1 = 2 AND c2 >= 4; +c1 c2 c3 c4 +2 4 1 1 +SELECT * FROM t1 WHERE c1 = 2 AND c2 > 4; +c1 c2 c3 c4 +EXPLAIN PARTITIONS SELECT * FROM t1 WHERE c1 = 1 AND c2 < 1; +id select_type table partitions type possible_keys key key_len ref rows Extra +1 SIMPLE NULL NULL NULL NULL NULL NULL NULL NULL Impossible WHERE noticed after reading const tables +EXPLAIN PARTITIONS SELECT * FROM t1 WHERE c1 = 1 AND c2 <= 1; +id select_type table partitions type possible_keys key key_len ref rows Extra +1 SIMPLE t1 p1 range PRIMARY PRIMARY 8 NULL 1 Using where +EXPLAIN PARTITIONS SELECT * FROM t1 WHERE c1 = 1 AND c2 = 1; +id select_type table partitions type possible_keys key key_len ref rows Extra +1 SIMPLE t1 p1 const PRIMARY PRIMARY 8 const,const 1 +EXPLAIN PARTITIONS SELECT * FROM t1 WHERE c1 = 1 AND c2 >= 1; +id select_type table partitions type possible_keys key key_len ref rows Extra +1 SIMPLE t1 p1,p2 range PRIMARY PRIMARY 8 NULL 2 Using where +EXPLAIN PARTITIONS SELECT * FROM t1 WHERE c1 = 1 AND c2 > 1; +id select_type table partitions type possible_keys key key_len ref rows Extra +1 SIMPLE t1 p1,p2 range PRIMARY PRIMARY 8 NULL 2 Using where +EXPLAIN PARTITIONS SELECT * FROM t1 WHERE c1 = 1 AND c2 < 3; +id select_type table partitions type possible_keys key key_len ref rows Extra +1 SIMPLE t1 p1 range PRIMARY PRIMARY 8 NULL 1 Using where +EXPLAIN PARTITIONS SELECT * FROM t1 WHERE c1 = 1 AND c2 <= 3; +id select_type table partitions type possible_keys key key_len ref rows Extra +1 SIMPLE t1 p1,p2 range PRIMARY PRIMARY 8 NULL 2 Using where +EXPLAIN PARTITIONS SELECT * FROM t1 WHERE c1 = 2 AND c2 <= 3; +id select_type table partitions type possible_keys key key_len ref rows Extra +1 SIMPLE t1 p1,p2 range PRIMARY PRIMARY 8 NULL 2 Using where +EXPLAIN PARTITIONS SELECT * FROM t1 WHERE c1 = 2 AND c2 = 3; +id select_type table partitions type possible_keys key key_len ref rows Extra +1 SIMPLE t1 p2 const PRIMARY PRIMARY 8 const,const 1 +EXPLAIN PARTITIONS SELECT * FROM t1 WHERE c1 = 2 AND c2 >= 3; +id select_type table partitions type possible_keys key key_len ref rows Extra +1 SIMPLE t1 p2 range PRIMARY PRIMARY 8 NULL 1 Using where +EXPLAIN PARTITIONS SELECT * FROM t1 WHERE c1 = 2 AND c2 > 3; +id select_type table partitions type possible_keys key key_len ref rows Extra +1 SIMPLE t1 p2 range PRIMARY PRIMARY 8 NULL 1 Using where +EXPLAIN PARTITIONS SELECT * FROM t1 WHERE c1 = 2 AND c2 < 4; +id select_type table partitions type possible_keys key key_len ref rows Extra +1 SIMPLE t1 p1,p2 range PRIMARY PRIMARY 8 NULL 2 Using where +EXPLAIN PARTITIONS SELECT * FROM t1 WHERE c1 = 2 AND c2 <= 4; +id select_type table partitions type possible_keys key key_len ref rows Extra +1 SIMPLE t1 p1,p2 range PRIMARY PRIMARY 8 NULL 2 Using where +EXPLAIN PARTITIONS SELECT * FROM t1 WHERE c1 = 2 AND c2 = 4; +id select_type table partitions type possible_keys key key_len ref rows Extra +1 SIMPLE t1 p2 const PRIMARY PRIMARY 8 const,const 1 +EXPLAIN PARTITIONS SELECT * FROM t1 WHERE c1 = 2 AND c2 >= 4; +id select_type table partitions type possible_keys key key_len ref rows Extra +1 SIMPLE t1 p2 range PRIMARY PRIMARY 8 NULL 1 Using where +EXPLAIN PARTITIONS SELECT * FROM t1 WHERE c1 = 2 AND c2 > 4; +id select_type table partitions type possible_keys key key_len ref rows Extra +1 SIMPLE NULL NULL NULL NULL NULL NULL NULL NULL Impossible WHERE noticed after reading const tables +DROP TABLE t1; diff --git a/mysql-test/t/partition_pruning.test b/mysql-test/t/partition_pruning.test index f2ca16b9259..fc09f40ed18 100644 --- a/mysql-test/t/partition_pruning.test +++ b/mysql-test/t/partition_pruning.test @@ -1410,3 +1410,51 @@ explain partitions select * from t1 where a between 10 and 13; explain partitions select * from t1 where a between 10 and 10+33; drop table t0, t1; + +--echo # +--echo # Bug#71095: Wrong results with PARTITION BY LIST COLUMNS() +--echo # +CREATE TABLE t1 +(c1 int, + c2 int, + c3 int, + c4 int, + PRIMARY KEY (c1,c2)) +PARTITION BY LIST COLUMNS (c2) +(PARTITION p1 VALUES IN (1,2), + PARTITION p2 VALUES IN (3,4)); +INSERT INTO t1 VALUES (1, 1, 1, 1), (2, 3, 1, 1); +INSERT INTO t1 VALUES (1, 2, 1, 1), (2, 4, 1, 1); +SELECT * FROM t1 WHERE c1 = 1 AND c2 < 1; +SELECT * FROM t1 WHERE c1 = 1 AND c2 <= 1; +SELECT * FROM t1 WHERE c1 = 1 AND c2 = 1; +SELECT * FROM t1 WHERE c1 = 1 AND c2 >= 1; +SELECT * FROM t1 WHERE c1 = 1 AND c2 > 1; +SELECT * FROM t1 WHERE c1 = 1 AND c2 < 3; +SELECT * FROM t1 WHERE c1 = 1 AND c2 <= 3; +SELECT * FROM t1 WHERE c1 = 2 AND c2 <= 3; +SELECT * FROM t1 WHERE c1 = 2 AND c2 = 3; +SELECT * FROM t1 WHERE c1 = 2 AND c2 >= 3; +SELECT * FROM t1 WHERE c1 = 2 AND c2 > 3; +SELECT * FROM t1 WHERE c1 = 2 AND c2 < 4; +SELECT * FROM t1 WHERE c1 = 2 AND c2 <= 4; +SELECT * FROM t1 WHERE c1 = 2 AND c2 = 4; +SELECT * FROM t1 WHERE c1 = 2 AND c2 >= 4; +SELECT * FROM t1 WHERE c1 = 2 AND c2 > 4; +EXPLAIN PARTITIONS SELECT * FROM t1 WHERE c1 = 1 AND c2 < 1; +EXPLAIN PARTITIONS SELECT * FROM t1 WHERE c1 = 1 AND c2 <= 1; +EXPLAIN PARTITIONS SELECT * FROM t1 WHERE c1 = 1 AND c2 = 1; +EXPLAIN PARTITIONS SELECT * FROM t1 WHERE c1 = 1 AND c2 >= 1; +EXPLAIN PARTITIONS SELECT * FROM t1 WHERE c1 = 1 AND c2 > 1; +EXPLAIN PARTITIONS SELECT * FROM t1 WHERE c1 = 1 AND c2 < 3; +EXPLAIN PARTITIONS SELECT * FROM t1 WHERE c1 = 1 AND c2 <= 3; +EXPLAIN PARTITIONS SELECT * FROM t1 WHERE c1 = 2 AND c2 <= 3; +EXPLAIN PARTITIONS SELECT * FROM t1 WHERE c1 = 2 AND c2 = 3; +EXPLAIN PARTITIONS SELECT * FROM t1 WHERE c1 = 2 AND c2 >= 3; +EXPLAIN PARTITIONS SELECT * FROM t1 WHERE c1 = 2 AND c2 > 3; +EXPLAIN PARTITIONS SELECT * FROM t1 WHERE c1 = 2 AND c2 < 4; +EXPLAIN PARTITIONS SELECT * FROM t1 WHERE c1 = 2 AND c2 <= 4; +EXPLAIN PARTITIONS SELECT * FROM t1 WHERE c1 = 2 AND c2 = 4; +EXPLAIN PARTITIONS SELECT * FROM t1 WHERE c1 = 2 AND c2 >= 4; +EXPLAIN PARTITIONS SELECT * FROM t1 WHERE c1 = 2 AND c2 > 4; +DROP TABLE t1; diff --git a/sql/sql_partition.cc b/sql/sql_partition.cc index e2ff4d07f1a..d7d362dbdfb 100644 --- a/sql/sql_partition.cc +++ b/sql/sql_partition.cc @@ -3304,19 +3304,28 @@ uint32 get_partition_id_cols_list_for_endpoint(partition_info *part_info, uint num_columns= part_info->part_field_list.elements; uint list_index; uint min_list_index= 0; + int cmp; + /* Notice that max_list_index = last_index + 1 here! */ uint max_list_index= part_info->num_list_values; DBUG_ENTER("get_partition_id_cols_list_for_endpoint"); /* Find the matching partition (including taking endpoint into account). */ do { - /* Midpoint, adjusted down, so it can never be > last index. */ + /* Midpoint, adjusted down, so it can never be >= max_list_index. */ list_index= (max_list_index + min_list_index) >> 1; - if (cmp_rec_and_tuple_prune(list_col_array + list_index*num_columns, - nparts, left_endpoint, include_endpoint) > 0) + cmp= cmp_rec_and_tuple_prune(list_col_array + list_index*num_columns, + nparts, left_endpoint, include_endpoint); + if (cmp > 0) + { min_list_index= list_index + 1; + } else + { max_list_index= list_index; + if (cmp == 0) + break; + } } while (max_list_index > min_list_index); list_index= max_list_index; @@ -3333,12 +3342,10 @@ uint32 get_partition_id_cols_list_for_endpoint(partition_info *part_info, nparts, left_endpoint, include_endpoint))); - if (!left_endpoint) - { - /* Set the end after this list tuple if not already after the last. */ - if (list_index < part_info->num_parts) - list_index++; - } + /* Include the right endpoint if not already passed end of array. */ + if (!left_endpoint && include_endpoint && cmp == 0 && + list_index < part_info->num_list_values) + list_index++; DBUG_RETURN(list_index); } @@ -7493,15 +7500,13 @@ static int cmp_rec_and_tuple_prune(part_column_list_val *val, field= val->part_info->part_field_array + n_vals_in_rec; if (!(*field)) { - /* - Full match, if right endpoint and not including the endpoint, - (rec < part) return lesser. - */ - if (!is_left_endpoint && !include_endpoint) - return -4; + /* Full match. Only equal if including endpoint. */ + if (include_endpoint) + return 0; - /* Otherwise they are equal! */ - return 0; + if (is_left_endpoint) + return +4; /* Start of range, part_tuple < rec, return higher. */ + return -4; /* End of range, rec < part_tupe, return lesser. */ } /* The prefix is equal and there are more partition columns to compare. From 5624a03c83adaa832cb25f2d3acbf52ce437b118 Mon Sep 17 00:00:00 2001 From: Rich Prohaska Date: Tue, 6 May 2014 13:17:49 -0400 Subject: [PATCH 013/279] #226 delete CMakeLists.in, no longer used --- storage/tokudb/CMakeLists.in | 29 ----------------------------- 1 file changed, 29 deletions(-) delete mode 100644 storage/tokudb/CMakeLists.in diff --git a/storage/tokudb/CMakeLists.in b/storage/tokudb/CMakeLists.in deleted file mode 100644 index 20c05126841..00000000000 --- a/storage/tokudb/CMakeLists.in +++ /dev/null @@ -1,29 +0,0 @@ -# Copyright (C) 2006 MySQL AB -# -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation; version 2 of the License. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program; if not, write to the Free Software -# Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA - -SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -DTOKUDB_VERSION=\\\"TOKUDB_VERSION_REPLACE_ME\\\"") -SET(CMAKE_CXX_FLAGS_DEBUG "${CMAKE_CXX_FLAGS_DEBUG} -DSAFEMALLOC -DSAFE_MUTEX") -SET(CMAKE_C_FLAGS_DEBUG "${CMAKE_C_FLAGS_DEBUG} -DSAFEMALLOC -DSAFE_MUTEX") - -INCLUDE_DIRECTORIES(TOKUDB_DIR_REPLACE_ME/windows - TOKUDB_DIR_REPLACE_ME/src - TOKUDB_DIR_REPLACE_ME/include - TOKUDB_DIR_REPLACE_ME/toku_include) - -INCLUDE("${PROJECT_SOURCE_DIR}/storage/mysql_storage_engine.cmake") -SET(TOKUDB_SOURCES hatoku_hton.cc ha_tokudb.cc hatoku_cmp.cc) -MYSQL_STORAGE_ENGINE(TOKUDB) - -TARGET_LINK_LIBRARIES(ha_tokudb PowrProf optimized TOKUDB_OBJ_DIR_REPLACE_ME/opt/ipo_libtokudb optimized TOKUDB_OBJ_DIR_REPLACE_ME/opt/libtokuportability debug TOKUDB_OBJ_DIR_REPLACE_ME/debug/static_libtokudb debug TOKUDB_OBJ_DIR_REPLACE_ME/debug/libtokuportability) From 3b43142623170e46bc128a58089cb26a17260383 Mon Sep 17 00:00:00 2001 From: Venkatesh Duggirala Date: Wed, 7 May 2014 14:33:58 +0530 Subject: [PATCH 014/279] Bug#17638477 UNINSTALL AND INSTALL SEMI-SYNC PLUGIN CAUSES SLAVES TO BREAK Fixing post push failure --- mysql-test/suite/rpl/r/rpl_semi_sync_uninstall_plugin.result | 4 ++-- mysql-test/suite/rpl/t/rpl_semi_sync_uninstall_plugin.test | 2 ++ 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/mysql-test/suite/rpl/r/rpl_semi_sync_uninstall_plugin.result b/mysql-test/suite/rpl/r/rpl_semi_sync_uninstall_plugin.result index d5cf690c978..bd659e71600 100644 --- a/mysql-test/suite/rpl/r/rpl_semi_sync_uninstall_plugin.result +++ b/mysql-test/suite/rpl/r/rpl_semi_sync_uninstall_plugin.result @@ -1,7 +1,7 @@ include/master-slave.inc [connection master] -INSTALL PLUGIN rpl_semi_sync_master SONAME 'semisync_master.so'; -INSTALL PLUGIN rpl_semi_sync_slave SONAME 'semisync_slave.so'; +INSTALL PLUGIN rpl_semi_sync_master SONAME 'SEMISYNC_MASTER_PLUGIN'; +INSTALL PLUGIN rpl_semi_sync_slave SONAME 'SEMISYNC_SLAVE_PLUGIN'; UNINSTALL PLUGIN rpl_semi_sync_slave; UNINSTALL PLUGIN rpl_semi_sync_master; CREATE TABLE t1(i int); diff --git a/mysql-test/suite/rpl/t/rpl_semi_sync_uninstall_plugin.test b/mysql-test/suite/rpl/t/rpl_semi_sync_uninstall_plugin.test index 094e044e0d2..2badd4203cf 100644 --- a/mysql-test/suite/rpl/t/rpl_semi_sync_uninstall_plugin.test +++ b/mysql-test/suite/rpl/t/rpl_semi_sync_uninstall_plugin.test @@ -21,10 +21,12 @@ # not in use i.e., when asynchronous replication is active. ############################################################################### # Step 1.1: Install semi sync master plugin on master +--replace_result $SEMISYNC_MASTER_PLUGIN SEMISYNC_MASTER_PLUGIN eval INSTALL PLUGIN rpl_semi_sync_master SONAME '$SEMISYNC_MASTER_PLUGIN'; # Step 1.2: Install semi sync slave plugin on slave --connection slave +--replace_result $SEMISYNC_SLAVE_PLUGIN SEMISYNC_SLAVE_PLUGIN eval INSTALL PLUGIN rpl_semi_sync_slave SONAME '$SEMISYNC_SLAVE_PLUGIN'; # Step 1.3: Uninstallation of semisync plugin on master and slave should be From 8ade414b28bd73dd5fc04d372b61d1c49aba0a8a Mon Sep 17 00:00:00 2001 From: Chaithra Gopalareddy Date: Wed, 7 May 2014 14:59:23 +0530 Subject: [PATCH 015/279] Bug#17909656 - WRONG RESULTS FOR A SIMPLE QUERY WITH GROUP BY Problem: If there is a predicate on a column referenced by MIN/MAX and that predicate is not present in all the disjunctions on keyparts earlier in the compound index, Loose Index Scan will not return correct result. Analysis: When loose index scan is chosen, range optimizer currently groups all the predicates that contain group parts separately and minmax parts separately. It therefore applies all the conditions on the group parts first to the fetched row. Then in the call to next_max, it processes the conditions which have min/max keypart. For ex in the following query: Select f1, max(f2) from t1 where (f1 = 10 and f2 = 13) or (f1 = 3) group by f1; Condition (f2 = 13) would be applied even for rows that satisfy (f1 = 3) thereby giving wrong results. Solution: Do not choose loose_index_scan for such cases. So a new rule WA2 is introduced to take care of the same. WA2: "If there are predicates on C, these predicates must be in conjuction to all predicates on all earlier keyparts in I." Todo the same, fix reuses the function get_constant_key_infix(). Since this funciton will fail for all multi-range conditions, it is re-written to recognize that if the sub-conditions are equivalent across the disjuncts: it will now succeed. And to achieve this a new helper function is introduced called all_same(). The fix also moves the test of NGA3 up to the former only caller, get_constant_key_infix(). mysql-test/r/group_min_max_innodb.result: Added test result change for Bug#17909656 mysql-test/t/group_min_max_innodb.test: Added test cases for Bug#17909656 sql/opt_range.cc: Introduced Rule WA2 because of Bug#17909656 --- mysql-test/r/group_min_max_innodb.result | 168 ++++++++++++++++++++ mysql-test/t/group_min_max_innodb.test | 93 +++++++++++ sql/opt_range.cc | 190 ++++++++++++++++------- 3 files changed, 398 insertions(+), 53 deletions(-) diff --git a/mysql-test/r/group_min_max_innodb.result b/mysql-test/r/group_min_max_innodb.result index 320c4b2b750..ad8dde69548 100644 --- a/mysql-test/r/group_min_max_innodb.result +++ b/mysql-test/r/group_min_max_innodb.result @@ -118,3 +118,171 @@ COUNT(DISTINCT a) 1 DROP TABLE t1; End of 5.5 tests +# +# Bug#17909656 - WRONG RESULTS FOR A SIMPLE QUERY WITH GROUP BY +# +CREATE TABLE t0 ( +i1 INTEGER NOT NULL +); +INSERT INTO t0 VALUES (1),(2),(3),(4),(5),(6),(7),(8),(9),(10), +(11),(12),(13),(14),(15),(16),(17),(18),(19),(20), +(21),(22),(23),(24),(25),(26),(27),(28),(29),(30); +CREATE TABLE t1 ( +c1 CHAR(1) NOT NULL, +i1 INTEGER NOT NULL, +i2 INTEGER NOT NULL, +UNIQUE KEY k1 (c1,i2) +) ENGINE=InnoDB; +INSERT INTO t1 SELECT 'A',i1,i1 FROM t0; +INSERT INTO t1 SELECT 'B',i1,i1 FROM t0; +INSERT INTO t1 SELECT 'C',i1,i1 FROM t0; +INSERT INTO t1 SELECT 'D',i1,i1 FROM t0; +INSERT INTO t1 SELECT 'E',i1,i1 FROM t0; +INSERT INTO t1 SELECT 'F',i1,i1 FROM t0; +CREATE TABLE t2 ( +c1 CHAR(1) NOT NULL, +i1 INTEGER NOT NULL, +i2 INTEGER NOT NULL, +UNIQUE KEY k2 (c1,i1,i2) +) ENGINE=InnoDB; +INSERT INTO t2 SELECT 'A',i1,i1 FROM t0; +INSERT INTO t2 SELECT 'B',i1,i1 FROM t0; +INSERT INTO t2 SELECT 'C',i1,i1 FROM t0; +INSERT INTO t2 SELECT 'D',i1,i1 FROM t0; +INSERT INTO t2 SELECT 'E',i1,i1 FROM t0; +INSERT INTO t2 SELECT 'F',i1,i1 FROM t0; +ANALYZE TABLE t1; +ANALYZE TABLE t2; +EXPLAIN SELECT c1, max(i2) FROM t1 WHERE (c1 = 'C' AND i2 = 17) OR ( c1 = 'F') +GROUP BY c1; +id select_type table type possible_keys key key_len ref rows Extra +1 SIMPLE t1 range k1 k1 5 NULL 31 Using where; Using index +SELECT c1, max(i2) FROM t1 WHERE (c1 = 'C' AND i2 = 17) OR ( c1 = 'F') +GROUP BY c1; +c1 max(i2) +C 17 +F 30 +EXPLAIN SELECT c1, max(i2) FROM t1 WHERE (c1 = 'C' OR ( c1 = 'F' AND i2 = 17)) +GROUP BY c1; +id select_type table type possible_keys key key_len ref rows Extra +1 SIMPLE t1 range k1 k1 5 NULL 31 Using where; Using index +SELECT c1, max(i2) FROM t1 WHERE (c1 = 'C' OR ( c1 = 'F' AND i2 = 17)) +GROUP BY c1; +c1 max(i2) +C 30 +F 17 +EXPLAIN SELECT c1, max(i2) FROM t1 WHERE (c1 = 'C' OR c1 = 'F' ) AND ( i2 = 17 ) +GROUP BY c1; +id select_type table type possible_keys key key_len ref rows Extra +1 SIMPLE t1 range k1 k1 5 NULL 1 Using where; Using index for group-by +SELECT c1, max(i2) FROM t1 WHERE (c1 = 'C' OR c1 = 'F' ) AND ( i2 = 17 ) +GROUP BY c1; +c1 max(i2) +C 17 +F 17 +EXPLAIN SELECT c1, max(i2) FROM t1 +WHERE ((c1 = 'C' AND (i2 = 40 OR i2 = 30)) OR ( c1 = 'F' AND (i2 = 40 ))) +GROUP BY c1; +id select_type table type possible_keys key key_len ref rows Extra +1 SIMPLE t1 range k1 k1 5 NULL 3 Using where; Using index +SELECT c1, max(i2) FROM t1 +WHERE ((c1 = 'C' AND (i2 = 40 OR i2 = 30)) OR ( c1 = 'F' AND (i2 = 40 ))) +GROUP BY c1; +c1 max(i2) +C 30 +EXPLAIN SELECT c1, i1, max(i2) FROM t2 +WHERE (c1 = 'C' OR ( c1 = 'F' AND i1 < 35)) AND ( i2 = 17 ) +GROUP BY c1,i1; +id select_type table type possible_keys key key_len ref rows Extra +1 SIMPLE t2 range k2 k2 9 NULL 59 Using where; Using index for group-by +SELECT c1, i1, max(i2) FROM t2 +WHERE (c1 = 'C' OR ( c1 = 'F' AND i1 < 35)) AND ( i2 = 17 ) +GROUP BY c1,i1; +c1 i1 max(i2) +C 17 17 +F 17 17 +EXPLAIN SELECT c1, i1, max(i2) FROM t2 +WHERE (((c1 = 'C' AND i1 < 40) OR ( c1 = 'F' AND i1 < 35)) AND ( i2 = 17 )) +GROUP BY c1,i1; +id select_type table type possible_keys key key_len ref rows Extra +1 SIMPLE t2 range k2 k2 9 NULL 58 Using where; Using index for group-by +SELECT c1, i1, max(i2) FROM t2 +WHERE (((c1 = 'C' AND i1 < 40) OR ( c1 = 'F' AND i1 < 35)) AND ( i2 = 17 )) +GROUP BY c1,i1; +c1 i1 max(i2) +C 17 17 +F 17 17 +EXPLAIN SELECT c1, i1, max(i2) FROM t2 +WHERE ((c1 = 'C' AND i1 < 40) OR ( c1 = 'F' AND i1 < 35) OR ( i2 = 17 )) +GROUP BY c1,i1; +id select_type table type possible_keys key key_len ref rows Extra +1 SIMPLE t2 range k2 k2 5 NULL 181 Using where; Using index for group-by +SELECT c1, i1, max(i2) FROM t2 +WHERE ((c1 = 'C' AND i1 < 40) OR ( c1 = 'F' AND i1 < 35) OR ( i2 = 17 )) +GROUP BY c1,i1; +c1 i1 max(i2) +A 17 17 +B 17 17 +C 1 1 +C 2 2 +C 3 3 +C 4 4 +C 5 5 +C 6 6 +C 7 7 +C 8 8 +C 9 9 +C 10 10 +C 11 11 +C 12 12 +C 13 13 +C 14 14 +C 15 15 +C 16 16 +C 17 17 +C 18 18 +C 19 19 +C 20 20 +C 21 21 +C 22 22 +C 23 23 +C 24 24 +C 25 25 +C 26 26 +C 27 27 +C 28 28 +C 29 29 +C 30 30 +D 17 17 +E 17 17 +F 1 1 +F 2 2 +F 3 3 +F 4 4 +F 5 5 +F 6 6 +F 7 7 +F 8 8 +F 9 9 +F 10 10 +F 11 11 +F 12 12 +F 13 13 +F 14 14 +F 15 15 +F 16 16 +F 17 17 +F 18 18 +F 19 19 +F 20 20 +F 21 21 +F 22 22 +F 23 23 +F 24 24 +F 25 25 +F 26 26 +F 27 27 +F 28 28 +F 29 29 +F 30 30 +DROP TABLE t0,t1,t2; diff --git a/mysql-test/t/group_min_max_innodb.test b/mysql-test/t/group_min_max_innodb.test index 7038eb2ff47..6967f847147 100644 --- a/mysql-test/t/group_min_max_innodb.test +++ b/mysql-test/t/group_min_max_innodb.test @@ -137,3 +137,96 @@ SELECT COUNT(DISTINCT a) FROM t1 WHERE b = 'b'; DROP TABLE t1; --echo End of 5.5 tests + +--echo # +--echo # Bug#17909656 - WRONG RESULTS FOR A SIMPLE QUERY WITH GROUP BY +--echo # + +CREATE TABLE t0 ( + i1 INTEGER NOT NULL +); + +INSERT INTO t0 VALUES (1),(2),(3),(4),(5),(6),(7),(8),(9),(10), + (11),(12),(13),(14),(15),(16),(17),(18),(19),(20), + (21),(22),(23),(24),(25),(26),(27),(28),(29),(30); + +CREATE TABLE t1 ( + c1 CHAR(1) NOT NULL, + i1 INTEGER NOT NULL, + i2 INTEGER NOT NULL, + UNIQUE KEY k1 (c1,i2) +) ENGINE=InnoDB; + +INSERT INTO t1 SELECT 'A',i1,i1 FROM t0; +INSERT INTO t1 SELECT 'B',i1,i1 FROM t0; +INSERT INTO t1 SELECT 'C',i1,i1 FROM t0; +INSERT INTO t1 SELECT 'D',i1,i1 FROM t0; +INSERT INTO t1 SELECT 'E',i1,i1 FROM t0; +INSERT INTO t1 SELECT 'F',i1,i1 FROM t0; + +CREATE TABLE t2 ( + c1 CHAR(1) NOT NULL, + i1 INTEGER NOT NULL, + i2 INTEGER NOT NULL, + UNIQUE KEY k2 (c1,i1,i2) +) ENGINE=InnoDB; + +INSERT INTO t2 SELECT 'A',i1,i1 FROM t0; +INSERT INTO t2 SELECT 'B',i1,i1 FROM t0; +INSERT INTO t2 SELECT 'C',i1,i1 FROM t0; +INSERT INTO t2 SELECT 'D',i1,i1 FROM t0; +INSERT INTO t2 SELECT 'E',i1,i1 FROM t0; +INSERT INTO t2 SELECT 'F',i1,i1 FROM t0; + +-- disable_result_log +ANALYZE TABLE t1; +ANALYZE TABLE t2; +-- enable_result_log + +let query= +SELECT c1, max(i2) FROM t1 WHERE (c1 = 'C' AND i2 = 17) OR ( c1 = 'F') +GROUP BY c1; +eval EXPLAIN $query; +eval $query; + +let query= +SELECT c1, max(i2) FROM t1 WHERE (c1 = 'C' OR ( c1 = 'F' AND i2 = 17)) +GROUP BY c1; +eval EXPLAIN $query; +eval $query; + +let query= +SELECT c1, max(i2) FROM t1 WHERE (c1 = 'C' OR c1 = 'F' ) AND ( i2 = 17 ) +GROUP BY c1; +eval EXPLAIN $query; +eval $query; + +let query= +SELECT c1, max(i2) FROM t1 +WHERE ((c1 = 'C' AND (i2 = 40 OR i2 = 30)) OR ( c1 = 'F' AND (i2 = 40 ))) +GROUP BY c1; +eval EXPLAIN $query; +eval $query; + +let query= +SELECT c1, i1, max(i2) FROM t2 +WHERE (c1 = 'C' OR ( c1 = 'F' AND i1 < 35)) AND ( i2 = 17 ) +GROUP BY c1,i1; +eval EXPLAIN $query; +eval $query; + +let query= +SELECT c1, i1, max(i2) FROM t2 +WHERE (((c1 = 'C' AND i1 < 40) OR ( c1 = 'F' AND i1 < 35)) AND ( i2 = 17 )) +GROUP BY c1,i1; +eval EXPLAIN $query; +eval $query; + +let query= +SELECT c1, i1, max(i2) FROM t2 +WHERE ((c1 = 'C' AND i1 < 40) OR ( c1 = 'F' AND i1 < 35) OR ( i2 = 17 )) +GROUP BY c1,i1; +eval EXPLAIN $query; +eval $query; + +DROP TABLE t0,t1,t2; diff --git a/sql/opt_range.cc b/sql/opt_range.cc index c7a7d2531af..05da107f0ea 100644 --- a/sql/opt_range.cc +++ b/sql/opt_range.cc @@ -1,4 +1,5 @@ -/* Copyright (c) 2000, 2013, Oracle and/or its affiliates. All rights reserved. +/* Copyright (c) 2000, 2014, Oracle and/or its affiliates. All rights + * reserved. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by @@ -304,31 +305,54 @@ public: :min_flag(0),elements(1),use_count(1),left(0),right(0),next_key_part(0), color(BLACK), type(type_arg) {} - inline bool is_same(SEL_ARG *arg) + /** + returns true if a range predicate is equal. Use all_same() + to check for equality of all the predicates on this keypart. + */ + inline bool is_same(const SEL_ARG *arg) const { if (type != arg->type || part != arg->part) - return 0; + return false; if (type != KEY_RANGE) - return 1; + return true; return cmp_min_to_min(arg) == 0 && cmp_max_to_max(arg) == 0; } + /** + returns true if all the predicates in the keypart tree are equal + */ + bool all_same(const SEL_ARG *arg) const + { + if (type != arg->type || part != arg->part) + return false; + if (type != KEY_RANGE) + return true; + if (arg == this) + return true; + const SEL_ARG *cmp_arg= arg->first(); + const SEL_ARG *cur_arg= first(); + for (; cur_arg && cmp_arg && cur_arg->is_same(cmp_arg); + cur_arg= cur_arg->next, cmp_arg= cmp_arg->next); + if (cur_arg || cmp_arg) + return false; + return true; + } inline void merge_flags(SEL_ARG *arg) { maybe_flag|=arg->maybe_flag; } inline void maybe_smaller() { maybe_flag=1; } /* Return true iff it's a single-point null interval */ inline bool is_null_interval() { return maybe_null && max_value[0] == 1; } - inline int cmp_min_to_min(SEL_ARG* arg) + inline int cmp_min_to_min(const SEL_ARG* arg) const { return sel_cmp(field,min_value, arg->min_value, min_flag, arg->min_flag); } - inline int cmp_min_to_max(SEL_ARG* arg) + inline int cmp_min_to_max(const SEL_ARG* arg) const { return sel_cmp(field,min_value, arg->max_value, min_flag, arg->max_flag); } - inline int cmp_max_to_max(SEL_ARG* arg) + inline int cmp_max_to_max(const SEL_ARG* arg) const { return sel_cmp(field,max_value, arg->max_value, max_flag, arg->max_flag); } - inline int cmp_max_to_min(SEL_ARG* arg) + inline int cmp_max_to_min(const SEL_ARG* arg) const { return sel_cmp(field,max_value, arg->min_value, max_flag, arg->min_flag); } @@ -507,6 +531,7 @@ public: void test_use_count(SEL_ARG *root); #endif SEL_ARG *first(); + const SEL_ARG *first() const; SEL_ARG *last(); void make_root(); inline bool simple_key() @@ -583,6 +608,18 @@ public: SEL_ARG *clone_tree(RANGE_OPT_PARAM *param); }; +/** + Helper function to compare two SEL_ARG's. +*/ +static bool all_same(const SEL_ARG *sa1, const SEL_ARG *sa2) +{ + if (sa1 == NULL && sa2 == NULL) + return true; + if ((sa1 != NULL && sa2 == NULL) || (sa1 == NULL && sa2 != NULL)) + return false; + return sa1->all_same(sa2); +} + class SEL_IMERGE; @@ -1770,6 +1807,13 @@ SEL_ARG *SEL_ARG::clone(RANGE_OPT_PARAM *param, SEL_ARG *new_parent, return tmp; } +/** + This gives the first SEL_ARG in the interval list, and the minimal element + in the red-black tree + + @return + SEL_ARG first SEL_ARG in the interval list +*/ SEL_ARG *SEL_ARG::first() { SEL_ARG *next_arg=this; @@ -1780,6 +1824,11 @@ SEL_ARG *SEL_ARG::first() return next_arg; } +const SEL_ARG *SEL_ARG::first() const +{ + return const_cast(this)->first(); +} + SEL_ARG *SEL_ARG::last() { SEL_ARG *next_arg=this; @@ -9356,6 +9405,8 @@ void QUICK_ROR_UNION_SELECT::add_keys_and_lengths(String *key_names, static inline uint get_field_keypart(KEY *index, Field *field); static inline SEL_ARG * get_index_range_tree(uint index, SEL_TREE* range_tree, PARAM *param, uint *param_idx); +static bool get_sel_arg_for_keypart(Field *field, SEL_ARG *index_range_tree, + SEL_ARG **cur_range); static bool get_constant_key_infix(KEY *index_info, SEL_ARG *index_range_tree, KEY_PART_INFO *first_non_group_part, KEY_PART_INFO *min_max_arg_part, @@ -9451,6 +9502,8 @@ cost_group_min_max(TABLE* table, KEY *index_info, uint used_key_parts, above tests. By transitivity then it also follows that each WA_i participates in the index I (if this was already tested for GA, NGA and C). + WA2. If there is a predicate on C, then it must be in conjunction + to all predicates on all earlier keyparts in I. C) Overall query form: SELECT EXPR([A_1,...,A_k], [B_1,...,B_m], [MIN(C)], [MAX(C)]) @@ -9856,6 +9909,25 @@ get_best_group_min_max(PARAM *param, SEL_TREE *tree, double read_time) } } + /** + Test WA2:If there are conditions on a column C participating in + MIN/MAX, those conditions must be conjunctions to all earlier + keyparts. Otherwise, Loose Index Scan cannot be used. + */ + if (tree && min_max_arg_item) + { + uint dummy; + SEL_ARG *index_range_tree= get_index_range_tree(cur_index, tree, param, + &dummy); + SEL_ARG *cur_range= NULL; + if (get_sel_arg_for_keypart(min_max_arg_part->field, + index_range_tree, &cur_range) || + (cur_range && cur_range->type != SEL_ARG::KEY_RANGE)) + { + goto next_index; + } + } + /* If we got to this point, cur_index_info passes the test. */ key_infix_parts= cur_key_infix_len ? (uint) (first_non_infix_part - first_non_group_part) : 0; @@ -10099,73 +10171,75 @@ check_group_min_max_predicates(COND *cond, Item_field *min_max_arg_item, /* - Get SEL_ARG tree, if any, for the keypart covering non grouping - attribute (NGA) field 'nga_field'. + Get the SEL_ARG tree 'tree' for the keypart covering 'field', if + any. 'tree' must be a unique conjunction to ALL predicates in earlier + keyparts of 'keypart_tree'. - This function enforces the NGA3 test: If 'keypart_tree' contains a - condition for 'nga_field', there can only be one range. In the - opposite case, this function returns with error and 'cur_range' - should not be used. + E.g., if 'keypart_tree' is for a composite index (kp1,kp2) and kp2 + covers 'field', all these conditions satisfies the requirement: - Note that the NGA1 and NGA2 requirements, like whether or not the - range predicate for 'nga_field' is equality, is not tested by this - function. + 1. "(kp1=2 OR kp1=3) AND kp2=10" => returns "kp2=10" + 2. "(kp1=2 AND kp2=10) OR (kp1=3 AND kp2=10)" => returns "kp2=10" + 3. "(kp1=2 AND (kp2=10 OR kp2=11)) OR (kp1=3 AND (kp2=10 OR kp2=11))" + => returns "kp2=10 OR kp2=11" - @param[in] nga_field The NGA field we want the SEL_ARG tree for + whereas these do not + 1. "(kp1=2 AND kp2=10) OR kp1=3" + 2. "(kp1=2 AND kp2=10) OR (kp1=3 AND kp2=11)" + 3. "(kp1=2 AND kp2=10) OR (kp1=3 AND (kp2=10 OR kp2=11))" + + This function effectively tests requirement WA2. In combination with + a test that the returned tree has no more than one range it is also + a test of NGA3. + + @param[in] field The field we want the SEL_ARG tree for @param[in] keypart_tree Root node of the SEL_ARG* tree for the index @param[out] cur_range The SEL_ARG tree, if any, for the keypart covering field 'keypart_field' - @retval true 'keypart_tree' contained a predicate for 'nga_field' but - multiple ranges exists. 'cur_range' should not be used. + @retval true 'keypart_tree' contained a predicate for 'field' that + is not conjunction to all predicates on earlier keyparts @retval false otherwise */ static bool -get_sel_arg_for_keypart(Field *nga_field, +get_sel_arg_for_keypart(Field *field, SEL_ARG *keypart_tree, SEL_ARG **cur_range) { - if(keypart_tree == NULL) + if (keypart_tree == NULL) return false; - if(keypart_tree->field->eq(nga_field)) + if (keypart_tree->field->eq(field)) { - /* - Enforce NGA3: If a condition for nga_field has been found, only - a single range is allowed. - */ - if (keypart_tree->prev || keypart_tree->next) - return true; // There are multiple ranges - *cur_range= keypart_tree; return false; } - SEL_ARG *found_tree= NULL; + SEL_ARG *tree_first_range= NULL; SEL_ARG *first_kp= keypart_tree->first(); - for (SEL_ARG *cur_kp= first_kp; cur_kp && !found_tree; - cur_kp= cur_kp->next) + for (SEL_ARG *cur_kp= first_kp; cur_kp; cur_kp= cur_kp->next) { + SEL_ARG *curr_tree= NULL; if (cur_kp->next_key_part) { - if (get_sel_arg_for_keypart(nga_field, + if (get_sel_arg_for_keypart(field, cur_kp->next_key_part, - &found_tree)) + &curr_tree)) return true; - } /* - Enforce NGA3: If a condition for nga_field has been found,only - a single range is allowed. - */ - if (found_tree && first_kp->next) - return true; // There are multiple ranges + Check if the SEL_ARG tree for 'field' is identical for all ranges in + 'keypart_tree + */ + if (cur_kp == first_kp) + tree_first_range= curr_tree; + else if (!all_same(tree_first_range, curr_tree)) + return true; } - *cur_range= found_tree; + *cur_range= tree_first_range; return false; } - /* Extract a sequence of constants from a conjunction of equality predicates. @@ -10188,7 +10262,8 @@ get_sel_arg_for_keypart(Field *nga_field, (const_ci = NG_i).. In addition, there can only be one range when there is such a gap. Thus all the NGF_i attributes must fill the 'gap' between the last group-by - attribute and the MIN/MAX attribute in the index (if present). If these + attribute and the MIN/MAX attribute in the index (if present). Also ensure + that there is only a single range on NGF_i (NGA3). If these conditions hold, copy each constant from its corresponding predicate into key_infix, in the order its NG_i attribute appears in the index, and update key_infix_len with the total length of the key parts in key_infix. @@ -10197,7 +10272,6 @@ get_sel_arg_for_keypart(Field *nga_field, TRUE if the index passes the test FALSE o/w */ - static bool get_constant_key_infix(KEY *index_info, SEL_ARG *index_range_tree, KEY_PART_INFO *first_non_group_part, @@ -10217,32 +10291,42 @@ get_constant_key_infix(KEY *index_info, SEL_ARG *index_range_tree, { cur_range= NULL; /* - Find the range tree for the current keypart. We assume that - index_range_tree points to the first keypart in the index. + Check NGA3: + 1. get_sel_arg_for_keypart gets the range tree for the 'field' and also + checks for a unique conjunction of this tree with all the predicates + on the earlier keyparts in the index. + 2. Check for multiple ranges on the found keypart tree. + + We assume that index_range_tree points to the leftmost keypart in + the index. */ - if(get_sel_arg_for_keypart(cur_part->field, index_range_tree, &cur_range)) + if (get_sel_arg_for_keypart(cur_part->field, index_range_tree, + &cur_range)) + return false; + + if (cur_range && cur_range->elements > 1) return false; if (!cur_range) { if (min_max_arg_part) - return FALSE; /* The current keypart has no range predicates at all. */ + return false; /* The current keypart has no range predicates at all. */ else { *first_non_infix_part= cur_part; - return TRUE; + return true; } } if ((cur_range->min_flag & NO_MIN_RANGE) || (cur_range->max_flag & NO_MAX_RANGE) || (cur_range->min_flag & NEAR_MIN) || (cur_range->max_flag & NEAR_MAX)) - return FALSE; + return false; uint field_length= cur_part->store_length; if (cur_range->maybe_null && cur_range->min_value[0] && cur_range->max_value[0]) - { + { /* cur_range specifies 'IS NULL'. In this case the argument points to a "null value" (is_null_string) that may not always be long @@ -10261,7 +10345,7 @@ get_constant_key_infix(KEY *index_info, SEL_ARG *index_range_tree, *key_infix_len+= field_length; } else - return FALSE; + return false; } if (!min_max_arg_part && (cur_part == last_part)) From e1da25f62117c8171ddb7e187ac3aedcacbe4088 Mon Sep 17 00:00:00 2001 From: Chaithra Gopalareddy Date: Wed, 7 May 2014 16:55:03 +0530 Subject: [PATCH 016/279] Fixing compilation error. Post push fix for Bug#17909656 --- sql/opt_range.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sql/opt_range.cc b/sql/opt_range.cc index 05da107f0ea..f7d8d0009d4 100644 --- a/sql/opt_range.cc +++ b/sql/opt_range.cc @@ -331,7 +331,7 @@ public: const SEL_ARG *cmp_arg= arg->first(); const SEL_ARG *cur_arg= first(); for (; cur_arg && cmp_arg && cur_arg->is_same(cmp_arg); - cur_arg= cur_arg->next, cmp_arg= cmp_arg->next); + cur_arg= cur_arg->next, cmp_arg= cmp_arg->next) ; if (cur_arg || cmp_arg) return false; return true; From b0493252e01a2afc64abe3238bcdb51cf7bdbd8f Mon Sep 17 00:00:00 2001 From: Rich Prohaska Date: Wed, 7 May 2014 08:20:41 -0400 Subject: [PATCH 017/279] #228 use thd_get/set_ha_data for tokudb_trx data --- storage/tokudb/ha_tokudb.cc | 49 +++++++++--------------- storage/tokudb/ha_tokudb_alter_56.cc | 2 +- storage/tokudb/ha_tokudb_alter_common.cc | 2 +- storage/tokudb/hatoku_hton.cc | 25 ++++++------ 4 files changed, 32 insertions(+), 46 deletions(-) diff --git a/storage/tokudb/ha_tokudb.cc b/storage/tokudb/ha_tokudb.cc index 296c472d36b..3f44c46afe5 100644 --- a/storage/tokudb/ha_tokudb.cc +++ b/storage/tokudb/ha_tokudb.cc @@ -120,14 +120,6 @@ extern "C" { #include "hatoku_defines.h" #include "hatoku_cmp.h" -static inline void *thd_data_get(THD *thd, int slot) { - return thd->ha_data[slot].ha_ptr; -} - -static inline void thd_data_set(THD *thd, int slot, void *data) { - thd->ha_data[slot].ha_ptr = data; -} - static inline uint get_key_parts(const KEY *key); #undef PACKAGE @@ -1016,8 +1008,7 @@ static uchar* pack_toku_field_blob( static int create_tokudb_trx_data_instance(tokudb_trx_data** out_trx) { int error; - tokudb_trx_data* trx = NULL; - trx = (tokudb_trx_data *) tokudb_my_malloc(sizeof(*trx), MYF(MY_ZEROFILL)); + tokudb_trx_data* trx = (tokudb_trx_data *) tokudb_my_malloc(sizeof(*trx), MYF(MY_ZEROFILL)); if (!trx) { error = ENOMEM; goto cleanup; @@ -1614,8 +1605,7 @@ int ha_tokudb::initialize_share( DB_TXN* txn = NULL; bool do_commit = false; THD* thd = ha_thd(); - tokudb_trx_data *trx = NULL; - trx = (tokudb_trx_data *) thd_data_get(ha_thd(), tokudb_hton->slot); + tokudb_trx_data *trx = (tokudb_trx_data *) thd_get_ha_data(ha_thd(), tokudb_hton); if (thd_sql_command(thd) == SQLCOM_CREATE_TABLE && trx && trx->sub_sp_level) { txn = trx->sub_sp_level; } @@ -3260,7 +3250,7 @@ void ha_tokudb::start_bulk_insert(ha_rows rows) { TOKUDB_HANDLER_DBUG_ENTER("%llu txn %p", (unsigned long long) rows, transaction); #endif THD* thd = ha_thd(); - tokudb_trx_data* trx = (tokudb_trx_data *) thd_data_get(thd, tokudb_hton->slot); + tokudb_trx_data* trx = (tokudb_trx_data *) thd_get_ha_data(thd, tokudb_hton); delay_updating_ai_metadata = true; ai_metadata_update_required = false; abort_loader = false; @@ -3328,7 +3318,7 @@ int ha_tokudb::end_bulk_insert(bool abort) { TOKUDB_HANDLER_DBUG_ENTER(""); int error = 0; THD* thd = ha_thd(); - tokudb_trx_data* trx = (tokudb_trx_data *) thd_data_get(thd, tokudb_hton->slot); + tokudb_trx_data* trx = (tokudb_trx_data *) thd_get_ha_data(thd, tokudb_hton); bool using_loader = (loader != NULL); if (ai_metadata_update_required) { tokudb_pthread_mutex_lock(&share->mutex); @@ -4060,7 +4050,7 @@ int ha_tokudb::write_row(uchar * record) { } } - trx = (tokudb_trx_data *) thd_data_get(thd, tokudb_hton->slot); + trx = (tokudb_trx_data *) thd_get_ha_data(thd, tokudb_hton); if (!error) { added_rows++; trx->stmt_progress.inserted++; @@ -4117,7 +4107,7 @@ int ha_tokudb::update_row(const uchar * old_row, uchar * new_row) { THD* thd = ha_thd(); DB_TXN* sub_trans = NULL; DB_TXN* txn = NULL; - tokudb_trx_data* trx = (tokudb_trx_data *) thd_data_get(thd, tokudb_hton->slot); + tokudb_trx_data* trx = (tokudb_trx_data *) thd_get_ha_data(thd, tokudb_hton); uint curr_num_DBs; LINT_INIT(error); @@ -4291,7 +4281,7 @@ int ha_tokudb::delete_row(const uchar * record) { bool has_null; THD* thd = ha_thd(); uint curr_num_DBs; - tokudb_trx_data* trx = (tokudb_trx_data *) thd_data_get(thd, tokudb_hton->slot);; + tokudb_trx_data* trx = (tokudb_trx_data *) thd_get_ha_data(thd, tokudb_hton);; ha_statistic_increment(&SSV::ha_delete_count); @@ -4855,7 +4845,7 @@ int ha_tokudb::index_read(uchar * buf, const uchar * key, uint key_len, enum ha_ int error = 0; uint32_t flags = 0; THD* thd = ha_thd(); - tokudb_trx_data* trx = (tokudb_trx_data *) thd_data_get(thd, tokudb_hton->slot);; + tokudb_trx_data* trx = (tokudb_trx_data *) thd_get_ha_data(thd, tokudb_hton);; struct smart_dbt_info info; struct index_read_info ir_info; @@ -5333,7 +5323,7 @@ int ha_tokudb::get_next(uchar* buf, int direction, DBT* key_to_compare, bool do_ int error = 0; uint32_t flags = SET_PRELOCK_FLAG(0); THD* thd = ha_thd(); - tokudb_trx_data* trx = (tokudb_trx_data *) thd_data_get(thd, tokudb_hton->slot);; + tokudb_trx_data* trx = (tokudb_trx_data *) thd_get_ha_data(thd, tokudb_hton);; bool need_val; HANDLE_INVALID_CURSOR(); @@ -5486,7 +5476,7 @@ int ha_tokudb::index_first(uchar * buf) { struct smart_dbt_info info; uint32_t flags = SET_PRELOCK_FLAG(0); THD* thd = ha_thd(); - tokudb_trx_data* trx = (tokudb_trx_data *) thd_data_get(thd, tokudb_hton->slot);; + tokudb_trx_data* trx = (tokudb_trx_data *) thd_get_ha_data(thd, tokudb_hton);; HANDLE_INVALID_CURSOR(); ha_statistic_increment(&SSV::ha_read_first_count); @@ -5529,7 +5519,7 @@ int ha_tokudb::index_last(uchar * buf) { struct smart_dbt_info info; uint32_t flags = SET_PRELOCK_FLAG(0); THD* thd = ha_thd(); - tokudb_trx_data* trx = (tokudb_trx_data *) thd_data_get(thd, tokudb_hton->slot);; + tokudb_trx_data* trx = (tokudb_trx_data *) thd_get_ha_data(thd, tokudb_hton);; HANDLE_INVALID_CURSOR(); ha_statistic_increment(&SSV::ha_read_last_count); @@ -5620,7 +5610,7 @@ int ha_tokudb::rnd_next(uchar * buf) { void ha_tokudb::track_progress(THD* thd) { - tokudb_trx_data* trx = (tokudb_trx_data *) thd_data_get(thd, tokudb_hton->slot); + tokudb_trx_data* trx = (tokudb_trx_data *) thd_get_ha_data(thd, tokudb_hton); if (trx) { ulonglong num_written = trx->stmt_progress.inserted + trx->stmt_progress.updated + trx->stmt_progress.deleted; bool update_status = @@ -6205,12 +6195,11 @@ int ha_tokudb::external_lock(THD * thd, int lock_type) { } int error = 0; - tokudb_trx_data *trx = NULL; - trx = (tokudb_trx_data *) thd_data_get(thd, tokudb_hton->slot); + tokudb_trx_data *trx = (tokudb_trx_data *) thd_get_ha_data(thd, tokudb_hton); if (!trx) { error = create_tokudb_trx_data_instance(&trx); if (error) { goto cleanup; } - thd_data_set(thd, tokudb_hton->slot, trx); + thd_set_ha_data(thd, tokudb_hton, trx); } if (trx->all == NULL) { trx->sp_level = NULL; @@ -6284,7 +6273,7 @@ int ha_tokudb::start_stmt(THD * thd, thr_lock_type lock_type) { TOKUDB_HANDLER_TRACE("q %s", thd->query()); int error = 0; - tokudb_trx_data *trx = (tokudb_trx_data *) thd_data_get(thd, tokudb_hton->slot); + tokudb_trx_data *trx = (tokudb_trx_data *) thd_get_ha_data(thd, tokudb_hton); DBUG_ASSERT(trx); /* @@ -6898,7 +6887,7 @@ int ha_tokudb::create(const char *name, TABLE * form, HA_CREATE_INFO * create_in newname = (char *)tokudb_my_malloc(get_max_dict_name_path_length(name),MYF(MY_WME)); if (newname == NULL){ error = ENOMEM; goto cleanup;} - trx = (tokudb_trx_data *) thd_data_get(ha_thd(), tokudb_hton->slot); + trx = (tokudb_trx_data *) thd_get_ha_data(ha_thd(), tokudb_hton); if (trx && trx->sub_sp_level && thd_sql_command(thd) == SQLCOM_CREATE_TABLE) { txn = trx->sub_sp_level; } @@ -7088,7 +7077,7 @@ int ha_tokudb::delete_or_rename_table (const char* from_name, const char* to_nam DB_TXN *parent_txn = NULL; tokudb_trx_data *trx = NULL; - trx = (tokudb_trx_data *) thd_data_get(thd, tokudb_hton->slot); + trx = (tokudb_trx_data *) thd_get_ha_data(thd, tokudb_hton); if (thd_sql_command(ha_thd()) == SQLCOM_CREATE_TABLE && trx && trx->sub_sp_level) { parent_txn = trx->sub_sp_level; } @@ -8234,12 +8223,12 @@ void ha_tokudb::cleanup_txn(DB_TXN *txn) { } void ha_tokudb::add_to_trx_handler_list() { - tokudb_trx_data *trx = (tokudb_trx_data *) thd_data_get(ha_thd(), tokudb_hton->slot); + tokudb_trx_data *trx = (tokudb_trx_data *) thd_get_ha_data(ha_thd(), tokudb_hton); trx->handlers = list_add(trx->handlers, &trx_handler_list); } void ha_tokudb::remove_from_trx_handler_list() { - tokudb_trx_data *trx = (tokudb_trx_data *) thd_data_get(ha_thd(), tokudb_hton->slot); + tokudb_trx_data *trx = (tokudb_trx_data *) thd_get_ha_data(ha_thd(), tokudb_hton); trx->handlers = list_delete(trx->handlers, &trx_handler_list); } diff --git a/storage/tokudb/ha_tokudb_alter_56.cc b/storage/tokudb/ha_tokudb_alter_56.cc index e0e1e7deee4..dbfce8764bc 100644 --- a/storage/tokudb/ha_tokudb_alter_56.cc +++ b/storage/tokudb/ha_tokudb_alter_56.cc @@ -752,7 +752,7 @@ bool ha_tokudb::commit_inplace_alter_table(TABLE *altered_table, Alter_inplace_i if (!commit) { // abort the alter transaction NOW so that any alters are rolled back. this allows the following restores to work. - tokudb_trx_data *trx = (tokudb_trx_data *) thd_data_get(thd, tokudb_hton->slot); + tokudb_trx_data *trx = (tokudb_trx_data *) thd_get_ha_data(thd, tokudb_hton); assert(ctx->alter_txn == trx->stmt); assert(trx->tokudb_lock_count > 0); // for partitioned tables, we use a single transaction to do all of the partition changes. the tokudb_lock_count diff --git a/storage/tokudb/ha_tokudb_alter_common.cc b/storage/tokudb/ha_tokudb_alter_common.cc index ecef0fb7415..414e8280daf 100644 --- a/storage/tokudb/ha_tokudb_alter_common.cc +++ b/storage/tokudb/ha_tokudb_alter_common.cc @@ -814,7 +814,7 @@ int ha_tokudb::write_frm_data(const uchar *frm_data, size_t frm_len) { if (TOKU_PARTITION_WRITE_FRM_DATA || table->part_info == NULL) { // write frmdata to status THD *thd = ha_thd(); - tokudb_trx_data *trx = (tokudb_trx_data *) thd_data_get(thd, tokudb_hton->slot); + tokudb_trx_data *trx = (tokudb_trx_data *) thd_get_ha_data(thd, tokudb_hton); assert(trx); DB_TXN *txn = trx->stmt; // use alter table transaction assert(txn); diff --git a/storage/tokudb/hatoku_hton.cc b/storage/tokudb/hatoku_hton.cc index 28394f2502c..bf7319b1203 100644 --- a/storage/tokudb/hatoku_hton.cc +++ b/storage/tokudb/hatoku_hton.cc @@ -624,8 +624,7 @@ int tokudb_end(handlerton * hton, ha_panic_function type) { static int tokudb_close_connection(handlerton * hton, THD * thd) { int error = 0; - tokudb_trx_data* trx = NULL; - trx = (tokudb_trx_data *) thd_data_get(thd, tokudb_hton->slot); + tokudb_trx_data* trx = (tokudb_trx_data *) thd_get_ha_data(thd, tokudb_hton); if (trx && trx->checkpoint_lock_taken) { error = db_env->checkpointing_resume(db_env); } @@ -723,7 +722,7 @@ static int tokudb_commit(handlerton * hton, THD * thd, bool all) { TOKUDB_DBUG_ENTER(""); DBUG_PRINT("trans", ("ending transaction %s", all ? "all" : "stmt")); uint32_t syncflag = THDVAR(thd, commit_sync) ? 0 : DB_TXN_NOSYNC; - tokudb_trx_data *trx = (tokudb_trx_data *) thd_data_get(thd, hton->slot); + tokudb_trx_data *trx = (tokudb_trx_data *) thd_get_ha_data(thd, hton); DB_TXN **txn = all ? &trx->all : &trx->stmt; DB_TXN *this_txn = *txn; if (this_txn) { @@ -752,7 +751,7 @@ static int tokudb_commit(handlerton * hton, THD * thd, bool all) { static int tokudb_rollback(handlerton * hton, THD * thd, bool all) { TOKUDB_DBUG_ENTER(""); DBUG_PRINT("trans", ("aborting transaction %s", all ? "all" : "stmt")); - tokudb_trx_data *trx = (tokudb_trx_data *) thd_data_get(thd, hton->slot); + tokudb_trx_data *trx = (tokudb_trx_data *) thd_get_ha_data(thd, hton); DB_TXN **txn = all ? &trx->all : &trx->stmt; DB_TXN *this_txn = *txn; if (this_txn) { @@ -782,7 +781,7 @@ static int tokudb_xa_prepare(handlerton* hton, THD* thd, bool all) { TOKUDB_DBUG_ENTER(""); int r = 0; DBUG_PRINT("trans", ("preparing transaction %s", all ? "all" : "stmt")); - tokudb_trx_data *trx = (tokudb_trx_data *) thd_data_get(thd, hton->slot); + tokudb_trx_data *trx = (tokudb_trx_data *) thd_get_ha_data(thd, hton); DB_TXN* txn = all ? trx->all : trx->stmt; if (txn) { if (tokudb_debug & TOKUDB_DEBUG_TXN) { @@ -861,7 +860,7 @@ static int tokudb_savepoint(handlerton * hton, THD * thd, void *savepoint) { TOKUDB_DBUG_ENTER(""); int error; SP_INFO save_info = (SP_INFO)savepoint; - tokudb_trx_data *trx = (tokudb_trx_data *) thd_data_get(thd, hton->slot); + tokudb_trx_data *trx = (tokudb_trx_data *) thd_get_ha_data(thd, hton); if (thd->in_sub_stmt) { assert(trx->stmt); error = txn_begin(db_env, trx->sub_sp_level, &(save_info->txn), DB_INHERIT_ISOLATION, thd); @@ -892,7 +891,7 @@ static int tokudb_rollback_to_savepoint(handlerton * hton, THD * thd, void *save DB_TXN* parent = NULL; DB_TXN* txn_to_rollback = save_info->txn; - tokudb_trx_data *trx = (tokudb_trx_data *) thd_data_get(thd, hton->slot); + tokudb_trx_data *trx = (tokudb_trx_data *) thd_get_ha_data(thd, hton); parent = txn_to_rollback->parent; if (!(error = txn_to_rollback->abort(txn_to_rollback))) { if (save_info->in_sub_stmt) { @@ -914,7 +913,7 @@ static int tokudb_release_savepoint(handlerton * hton, THD * thd, void *savepoin DB_TXN* parent = NULL; DB_TXN* txn_to_commit = save_info->txn; - tokudb_trx_data *trx = (tokudb_trx_data *) thd_data_get(thd, hton->slot); + tokudb_trx_data *trx = (tokudb_trx_data *) thd_get_ha_data(thd, hton); parent = txn_to_commit->parent; if (!(error = txn_to_commit->commit(txn_to_commit, 0))) { if (save_info->in_sub_stmt) { @@ -974,7 +973,7 @@ static int tokudb_discover3(handlerton *hton, THD* thd, const char *db, const ch bool do_commit; #if 100000 <= MYSQL_VERSION_ID && MYSQL_VERSION_ID <= 100099 - tokudb_trx_data *trx = (tokudb_trx_data *) thd_data_get(thd, tokudb_hton->slot); + tokudb_trx_data *trx = (tokudb_trx_data *) thd_get_ha_data(thd, tokudb_hton); if (thd_sql_command(thd) == SQLCOM_CREATE_TABLE && trx && trx->sub_sp_level) { do_commit = false; txn = trx->sub_sp_level; @@ -1129,15 +1128,14 @@ static bool tokudb_show_engine_status(THD * thd, stat_print_fn * stat_print) { static void tokudb_checkpoint_lock(THD * thd) { int error; const char *old_proc_info; - tokudb_trx_data* trx = NULL; - trx = (tokudb_trx_data *) thd_data_get(thd, tokudb_hton->slot); + tokudb_trx_data* trx = (tokudb_trx_data *) thd_get_ha_data(thd, tokudb_hton); if (!trx) { error = create_tokudb_trx_data_instance(&trx); // // can only fail due to memory allocation, so ok to assert // assert(!error); - thd_data_set(thd, tokudb_hton->slot, trx); + thd_set_ha_data(thd, tokudb_hton, trx); } if (trx->checkpoint_lock_taken) { @@ -1161,8 +1159,7 @@ cleanup: static void tokudb_checkpoint_unlock(THD * thd) { int error; const char *old_proc_info; - tokudb_trx_data* trx = NULL; - trx = (tokudb_trx_data *) thd_data_get(thd, tokudb_hton->slot); + tokudb_trx_data* trx = (tokudb_trx_data *) thd_get_ha_data(thd, tokudb_hton); if (!trx) { error = 0; goto cleanup; From 918837f7289a3cbd032a5cdb85eb55a466ad7c8e Mon Sep 17 00:00:00 2001 From: Tor Didriksen Date: Wed, 7 May 2014 17:09:14 +0200 Subject: [PATCH 018/279] Backport from trunk: Bug#18187290 ISSUE WITH BUILDING MYSQL USING CMAKE 2.8.12 We want to upgrade to VS2013 on Windows. In order to do this, we need to upgrade to cmake 2.8.12 This has introduced some incompatibilities for .pdb files, and "make install" no longer works. To reproduce: cmake --build . --target package --config debug The fix: Rather than installing .pdb files for static libraries, we use the /Z7 flag to store symbolic debugging information in the .obj files. --- cmake/install_macros.cmake | 17 ++++++++++--- cmake/os/Windows.cmake | 29 ++++++++++++++-------- extra/yassl/CMakeLists.txt | 3 +-- extra/yassl/taocrypt/CMakeLists.txt | 3 +-- libmysql/CMakeLists.txt | 1 - libmysql/authentication_win/CMakeLists.txt | 3 +-- mysys/CMakeLists.txt | 3 +-- strings/CMakeLists.txt | 3 +-- vio/CMakeLists.txt | 3 +-- zlib/CMakeLists.txt | 3 +-- 10 files changed, 38 insertions(+), 30 deletions(-) diff --git a/cmake/install_macros.cmake b/cmake/install_macros.cmake index 54f3225ed7e..be243a42924 100644 --- a/cmake/install_macros.cmake +++ b/cmake/install_macros.cmake @@ -21,26 +21,35 @@ MACRO (INSTALL_DEBUG_SYMBOLS targets) GET_TARGET_PROPERTY(location ${target} LOCATION) GET_TARGET_PROPERTY(type ${target} TYPE) IF(NOT INSTALL_LOCATION) - IF(type MATCHES "STATIC_LIBRARY" OR type MATCHES "MODULE_LIBRARY" OR type MATCHES "SHARED_LIBRARY") + IF(type MATCHES "STATIC_LIBRARY" + OR type MATCHES "MODULE_LIBRARY" + OR type MATCHES "SHARED_LIBRARY") SET(INSTALL_LOCATION "lib") ELSEIF(type MATCHES "EXECUTABLE") SET(INSTALL_LOCATION "bin") ELSE() - MESSAGE(FATAL_ERROR "cannot determine type of ${target}. Don't now where to install") + MESSAGE(FATAL_ERROR + "cannot determine type of ${target}. Don't now where to install") ENDIF() ENDIF() STRING(REPLACE ".exe" ".pdb" pdb_location ${location}) STRING(REPLACE ".dll" ".pdb" pdb_location ${pdb_location}) STRING(REPLACE ".lib" ".pdb" pdb_location ${pdb_location}) IF(CMAKE_GENERATOR MATCHES "Visual Studio") - STRING(REPLACE "${CMAKE_CFG_INTDIR}" "\${CMAKE_INSTALL_CONFIG_NAME}" pdb_location ${pdb_location}) + STRING(REPLACE + "${CMAKE_CFG_INTDIR}" "\${CMAKE_INSTALL_CONFIG_NAME}" + pdb_location ${pdb_location}) ENDIF() IF(target STREQUAL "mysqld") SET(comp Server) ELSE() SET(comp Debuginfo) ENDIF() - INSTALL(FILES ${pdb_location} DESTINATION ${INSTALL_LOCATION} COMPONENT ${comp}) + # No .pdb file for static libraries. + IF(NOT type MATCHES "STATIC_LIBRARY") + INSTALL(FILES ${pdb_location} + DESTINATION ${INSTALL_LOCATION} COMPONENT ${comp}) + ENDIF() ENDFOREACH() ENDIF() ENDMACRO() diff --git a/cmake/os/Windows.cmake b/cmake/os/Windows.cmake index c3809d2e6c2..54ea3cf95f9 100644 --- a/cmake/os/Windows.cmake +++ b/cmake/os/Windows.cmake @@ -1,4 +1,4 @@ -# Copyright (c) 2010, 2011, Oracle and/or its affiliates. All rights reserved. +# Copyright (c) 2010, 2014, Oracle and/or its affiliates. All rights reserved. # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -62,22 +62,30 @@ IF(MINGW AND CMAKE_SIZEOF_VOID_P EQUAL 4) ENDIF() IF(MSVC) - # Enable debug info also in Release build, and create PDB to be able to analyze - # crashes - FOREACH(lang C CXX) - SET(CMAKE_${lang}_FLAGS_RELEASE "${CMAKE_${lang}_FLAGS_RELEASE} /Zi") - ENDFOREACH() + # Enable debug info also in Release build, + # and create PDB to be able to analyze crashes. FOREACH(type EXE SHARED MODULE) - SET(CMAKE_{type}_LINKER_FLAGS_RELEASE "${CMAKE_${type}_LINKER_FLAGS_RELEASE} /debug") + SET(CMAKE_{type}_LINKER_FLAGS_RELEASE + "${CMAKE_${type}_LINKER_FLAGS_RELEASE} /debug") ENDFOREACH() # Force static runtime libraries + # - Choose debugging information: + # /Z7 + # Produces an .obj file containing full symbolic debugging + # information for use with the debugger. The symbolic debugging + # information includes the names and types of variables, as well as + # functions and line numbers. No .pdb file is produced by the compiler. + FOREACH(lang C CXX) + SET(CMAKE_${lang}_FLAGS_RELEASE "${CMAKE_${lang}_FLAGS_RELEASE} /Z7") + ENDFOREACH() FOREACH(flag - CMAKE_C_FLAGS_RELEASE CMAKE_C_FLAGS_RELWITHDEBINFO - CMAKE_C_FLAGS_DEBUG CMAKE_C_FLAGS_DEBUG_INIT + CMAKE_C_FLAGS_RELEASE CMAKE_C_FLAGS_RELWITHDEBINFO + CMAKE_C_FLAGS_DEBUG CMAKE_C_FLAGS_DEBUG_INIT CMAKE_CXX_FLAGS_RELEASE CMAKE_CXX_FLAGS_RELWITHDEBINFO - CMAKE_CXX_FLAGS_DEBUG CMAKE_CXX_FLAGS_DEBUG_INIT) + CMAKE_CXX_FLAGS_DEBUG CMAKE_CXX_FLAGS_DEBUG_INIT) STRING(REPLACE "/MD" "/MT" "${flag}" "${${flag}}") + STRING(REPLACE "/Zi" "/Z7" "${flag}" "${${flag}}") ENDFOREACH() # Remove support for exceptions @@ -107,7 +115,6 @@ IF(MSVC) SET(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} /wd4800 /wd4805 /wd4996") SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} /wd4800 /wd4805 /wd4996 /we4099") - IF(CMAKE_SIZEOF_VOID_P MATCHES 8) # _WIN64 is defined by the compiler itself. # Yet, we define it here again to work around a bug with Intellisense diff --git a/extra/yassl/CMakeLists.txt b/extra/yassl/CMakeLists.txt index a5d5898e192..5c5e4922b1d 100644 --- a/extra/yassl/CMakeLists.txt +++ b/extra/yassl/CMakeLists.txt @@ -1,4 +1,4 @@ -# Copyright (c) 2006, 2013, Oracle and/or its affiliates. All rights reserved. +# Copyright (c) 2006, 2014, Oracle and/or its affiliates. All rights reserved. # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -37,7 +37,6 @@ ENDIF() ADD_CONVENIENCE_LIBRARY(yassl ${YASSL_SOURCES}) RESTRICT_SYMBOL_EXPORTS(yassl) -INSTALL_DEBUG_SYMBOLS(yassl) IF(MSVC) INSTALL_DEBUG_TARGET(yassl DESTINATION ${INSTALL_LIBDIR}/debug) ENDIF() diff --git a/extra/yassl/taocrypt/CMakeLists.txt b/extra/yassl/taocrypt/CMakeLists.txt index 749193f1935..a08089870a3 100644 --- a/extra/yassl/taocrypt/CMakeLists.txt +++ b/extra/yassl/taocrypt/CMakeLists.txt @@ -1,4 +1,4 @@ -# Copyright (c) 2006, 2013, Oracle and/or its affiliates. All rights reserved. +# Copyright (c) 2006, 2014, Oracle and/or its affiliates. All rights reserved. # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -36,7 +36,6 @@ ENDIF() ADD_CONVENIENCE_LIBRARY(taocrypt ${TAOCRYPT_SOURCES}) RESTRICT_SYMBOL_EXPORTS(taocrypt) -INSTALL_DEBUG_SYMBOLS(taocrypt) IF(MSVC) INSTALL_DEBUG_TARGET(taocrypt DESTINATION ${INSTALL_LIBDIR}/debug) ENDIF() diff --git a/libmysql/CMakeLists.txt b/libmysql/CMakeLists.txt index 33de97d198b..374b38131d2 100644 --- a/libmysql/CMakeLists.txt +++ b/libmysql/CMakeLists.txt @@ -168,7 +168,6 @@ ENDIF() MERGE_LIBRARIES(mysqlclient STATIC ${LIBS} COMPONENT Development) # Visual Studio users need debug static library for debug projects -INSTALL_DEBUG_SYMBOLS(clientlib) IF(MSVC) INSTALL_DEBUG_TARGET(mysqlclient DESTINATION ${INSTALL_LIBDIR}/debug) INSTALL_DEBUG_TARGET(clientlib DESTINATION ${INSTALL_LIBDIR}/debug) diff --git a/libmysql/authentication_win/CMakeLists.txt b/libmysql/authentication_win/CMakeLists.txt index 8cf8862e46f..447a69bae24 100644 --- a/libmysql/authentication_win/CMakeLists.txt +++ b/libmysql/authentication_win/CMakeLists.txt @@ -1,4 +1,4 @@ -# Copyright (c) 2011, 2013, Oracle and/or its affiliates. All rights reserved. +# Copyright (c) 2011, 2014, Oracle and/or its affiliates. All rights reserved. # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -32,7 +32,6 @@ TARGET_LINK_LIBRARIES(auth_win_client Secur32) SOURCE_GROUP(Headers REGULAR_EXPRESSION ".*h$") -INSTALL_DEBUG_SYMBOLS(auth_win_client) IF(MSVC) INSTALL_DEBUG_TARGET(auth_win_client DESTINATION ${INSTALL_LIBDIR}/debug) ENDIF() diff --git a/mysys/CMakeLists.txt b/mysys/CMakeLists.txt index 648a87d1e95..6969acd8d04 100644 --- a/mysys/CMakeLists.txt +++ b/mysys/CMakeLists.txt @@ -1,4 +1,4 @@ -# Copyright (c) 2006, 2013, Oracle and/or its affiliates. All rights reserved. +# Copyright (c) 2006, 2014, Oracle and/or its affiliates. All rights reserved. # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -78,7 +78,6 @@ ADD_EXECUTABLE(thr_lock thr_lock.c) TARGET_LINK_LIBRARIES(thr_lock mysys) SET_TARGET_PROPERTIES(thr_lock PROPERTIES COMPILE_FLAGS "-DMAIN") -INSTALL_DEBUG_SYMBOLS(mysys) IF(MSVC) INSTALL_DEBUG_TARGET(mysys DESTINATION ${INSTALL_LIBDIR}/debug) ENDIF() diff --git a/strings/CMakeLists.txt b/strings/CMakeLists.txt index 35b4a472686..5cd42b2fc69 100644 --- a/strings/CMakeLists.txt +++ b/strings/CMakeLists.txt @@ -1,4 +1,4 @@ -# Copyright (c) 2006, 2013, Oracle and/or its affiliates. All rights reserved. +# Copyright (c) 2006, 2014, Oracle and/or its affiliates. All rights reserved. # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -32,7 +32,6 @@ ENDIF() ADD_DEFINITIONS(-DDISABLE_MYSQL_THREAD_H) ADD_CONVENIENCE_LIBRARY(strings ${STRINGS_SOURCES}) -INSTALL_DEBUG_SYMBOLS(strings) IF(MSVC) INSTALL_DEBUG_TARGET(strings DESTINATION ${INSTALL_LIBDIR}/debug) ENDIF() diff --git a/vio/CMakeLists.txt b/vio/CMakeLists.txt index ab763ccc28a..b83518cd749 100644 --- a/vio/CMakeLists.txt +++ b/vio/CMakeLists.txt @@ -1,4 +1,4 @@ -# Copyright (c) 2006, 2013, Oracle and/or its affiliates. All rights reserved. +# Copyright (c) 2006, 2014, Oracle and/or its affiliates. All rights reserved. # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -21,7 +21,6 @@ SET(VIO_SOURCES vio.c viosocket.c viossl.c viosslfactories.c) ADD_CONVENIENCE_LIBRARY(vio ${VIO_SOURCES}) TARGET_LINK_LIBRARIES(vio ${LIBSOCKET}) -INSTALL_DEBUG_SYMBOLS(vio) IF(MSVC) INSTALL_DEBUG_TARGET(vio DESTINATION ${INSTALL_LIBDIR}/debug) ENDIF() diff --git a/zlib/CMakeLists.txt b/zlib/CMakeLists.txt index 0be1f976b39..7668ce723b8 100644 --- a/zlib/CMakeLists.txt +++ b/zlib/CMakeLists.txt @@ -1,4 +1,4 @@ -# Copyright (c) 2006, 2013, Oracle and/or its affiliates. All rights reserved. +# Copyright (c) 2006, 2014, Oracle and/or its affiliates. All rights reserved. # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -23,7 +23,6 @@ SET(ZLIB_SOURCES adler32.c compress.c crc32.c crc32.h deflate.c deflate.h gzio. ADD_CONVENIENCE_LIBRARY(zlib ${ZLIB_SOURCES}) RESTRICT_SYMBOL_EXPORTS(zlib) -INSTALL_DEBUG_SYMBOLS(zlib) IF(MSVC) INSTALL_DEBUG_TARGET(zlib DESTINATION ${INSTALL_LIBDIR}/debug) ENDIF() From 858e5626c5cdee3b0359e906fb04bdbbb6d38190 Mon Sep 17 00:00:00 2001 From: Venkata Sidagam Date: Thu, 8 May 2014 14:41:01 +0530 Subject: [PATCH 019/279] Bug #18045646 LOCAL USER CAN RUN ARBITRARY CODE IN THE CONTEXT OF THE MYSQL SERVER Description: Using the temporary file vulnerability an attacker can create a file with arbitrary content at a location of his choice. This can be used to create the file /var/lib/mysql/my.cnf, which will be read as a configuration file by MySQL, because it is located in the home directory of the mysql user. With this configuration file, the attacker can specify his own plugin_dir variable, which then allows him to load arbitrary code via "INSTALL PLUGIN...". Analysis: While creating the ".TMD" file we are not checking if the file is already exits or not in mi_repair() function. And we are truncating if the ".TMD" file exits and going ahead This is creating the security breach. Fix: We need to use O_EXCL flag along with O_RDWR and O_TRUNC which will make sure if any user creates ".TMD" file, will fails the repair table with "cannot create ".TMD" file error". Actually we are initialing "param.tmpfile_createflag" member with O_RDWR | O_TRUNC | O_EXCL in myisamchk_init(). And we are modifying it in ha_myisam::repair() to O_RDWR | O_TRUNC. So, we need to remove the line which is modifying the "param.tmpfile_createflag". --- storage/myisam/ha_myisam.cc | 1 - 1 file changed, 1 deletion(-) diff --git a/storage/myisam/ha_myisam.cc b/storage/myisam/ha_myisam.cc index 72a29cd8130..602a0ae6cc1 100644 --- a/storage/myisam/ha_myisam.cc +++ b/storage/myisam/ha_myisam.cc @@ -1041,7 +1041,6 @@ int ha_myisam::repair(THD *thd, MI_CHECK ¶m, bool do_optimize) param.db_name= table->s->db.str; param.table_name= table->alias; - param.tmpfile_createflag = O_RDWR | O_TRUNC; param.using_global_keycache = 1; param.thd= thd; param.tmpdir= &mysql_tmpdir_list; From ee3c555ad9abd0f98434910ce7892819607c06a3 Mon Sep 17 00:00:00 2001 From: mithun Date: Thu, 8 May 2014 14:49:53 +0530 Subject: [PATCH 020/279] Bug #17059925: UNIONS COMPUTES ROWS_EXAMINED INCORRECTLY ISSUE: ------ For UNION of selects, rows examined by the query will be sum of rows examined by individual select operations and rows examined for union operation. The value of session level global counter that is used to count the rows examined by a select statement should be accumulated and reset before it is used for next select statement. But we have missed to reset the same. Because of this examined row count of a select query is accounted more than once. SOLUTION: --------- In union reset the session level global counter used to accumulate count of examined rows after its value is saved. mysql-test/r/union.result: Expected output of testcase added. mysql-test/t/union.test: Test to verify examined row count of Union operations. sql/sql_union.cc: Reset the value of thd->examined_row_count after accumulating the value. --- mysql-test/r/union.result | 34 ++++++++++++++++++++++++++++++++++ mysql-test/t/union.test | 31 +++++++++++++++++++++++++++++++ sql/sql_union.cc | 8 +++++++- 3 files changed, 72 insertions(+), 1 deletion(-) diff --git a/mysql-test/r/union.result b/mysql-test/r/union.result index 0fd1526684a..97c1a8b4d49 100644 --- a/mysql-test/r/union.result +++ b/mysql-test/r/union.result @@ -1826,3 +1826,37 @@ dev SELECT(SELECT 1 AS a FROM dual ORDER BY a DESC LIMIT 1) AS dev; dev 1 +# +# Bug #17059925 : UNIONS COMPUTES ROWS_EXAMINED INCORRECTLY +# +SET @old_slow_query_log= @@global.slow_query_log; +SET @old_log_output= @@global.log_output; +SET @old_long_query_time= @@long_query_time; +SET GLOBAL log_output= "TABLE"; +SET GLOBAL slow_query_log= ON; +SET SESSION long_query_time= 0; +CREATE TABLE t17059925 (a INT); +CREATE TABLE t2 (b INT); +CREATE TABLE t3 (c INT); +INSERT INTO t17059925 VALUES (1), (2), (3); +INSERT INTO t2 VALUES (4), (5), (6); +INSERT INTO t3 VALUES (7), (8), (9); +TRUNCATE table mysql.slow_log; +SELECT * FROM t17059925 UNION SELECT * FROM t2 UNION SELECT * FROM t3; +a +1 +2 +3 +4 +5 +6 +7 +8 +9 +SELECT sql_text, rows_examined FROM mysql.slow_log WHERE sql_text LIKE '%SELECT%t17059925%'; +sql_text rows_examined +SELECT * FROM t17059925 UNION SELECT * FROM t2 UNION SELECT * FROM t3 18 +DROP TABLE t17059925, t2, t3; +SET @@long_query_time= @old_long_query_time; +SET @@global.log_output= @old_log_output; +SET @@global.slow_query_log= @old_slow_query_log; diff --git a/mysql-test/t/union.test b/mysql-test/t/union.test index f5a5cad77e8..147c1d3834b 100644 --- a/mysql-test/t/union.test +++ b/mysql-test/t/union.test @@ -1225,3 +1225,34 @@ SELECT(SELECT 0 AS a FROM dual UNION SELECT 1 AS a FROM dual ORDER BY a DESC LIM SELECT(SELECT 1 AS a ORDER BY a) AS dev; SELECT(SELECT 1 AS a LIMIT 1) AS dev; SELECT(SELECT 1 AS a FROM dual ORDER BY a DESC LIMIT 1) AS dev; + + +--echo # +--echo # Bug #17059925 : UNIONS COMPUTES ROWS_EXAMINED INCORRECTLY +--echo # + +## Save current state of slow log variables +SET @old_slow_query_log= @@global.slow_query_log; +SET @old_log_output= @@global.log_output; +SET @old_long_query_time= @@long_query_time; +SET GLOBAL log_output= "TABLE"; +SET GLOBAL slow_query_log= ON; +SET SESSION long_query_time= 0; + +CREATE TABLE t17059925 (a INT); +CREATE TABLE t2 (b INT); +CREATE TABLE t3 (c INT); +INSERT INTO t17059925 VALUES (1), (2), (3); +INSERT INTO t2 VALUES (4), (5), (6); +INSERT INTO t3 VALUES (7), (8), (9); +TRUNCATE table mysql.slow_log; +--sorted_result +SELECT * FROM t17059925 UNION SELECT * FROM t2 UNION SELECT * FROM t3; +SELECT sql_text, rows_examined FROM mysql.slow_log WHERE sql_text LIKE '%SELECT%t17059925%'; +DROP TABLE t17059925, t2, t3; + +## Reset to initial values +SET @@long_query_time= @old_long_query_time; +SET @@global.log_output= @old_log_output; +SET @@global.slow_query_log= @old_slow_query_log; + diff --git a/sql/sql_union.cc b/sql/sql_union.cc index d0660e8f117..d230b903d2c 100644 --- a/sql/sql_union.cc +++ b/sql/sql_union.cc @@ -1,4 +1,4 @@ -/* Copyright (c) 2000, 2011, Oracle and/or its affiliates. All rights reserved. +/* Copyright (c) 2000, 2014, Oracle and/or its affiliates. All rights reserved. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by @@ -557,7 +557,13 @@ bool st_select_lex_unit::exec() 0); if (!saved_error) { + /* + Save the current examined row count locally and clear the global + counter, so that we can accumulate the number of evaluated rows for + the current query block. + */ examined_rows+= thd->examined_row_count; + thd->examined_row_count= 0; if (union_result->flush()) { thd->lex->current_select= lex_select_save; From b0b60f249807b6c2d423313350d9ad66693c2d1e Mon Sep 17 00:00:00 2001 From: unknown Date: Thu, 8 May 2014 14:20:18 +0200 Subject: [PATCH 021/279] MDEV-5262: Missing retry after temp error in parallel replication Start implementing that an event group can be re-tried in parallel replication if it fails with a temporary error (like deadlock). Patch is very incomplete, just some very basic retry works. Stuff still missing (not complete list): - Handle moving to the next relay log file, if event group to be retried spans multiple relay log files. - Handle refcounting of relay log files, to ensure that we do not purge a relay log file and then later attempt to re-execute events out of it. - Handle description_event_for_exec - we need to save this somehow for the possible retry - and use the correct one in case it differs between relay logs. - Do another retry attempt in case the first retry also fails. - Limit the max number of retries. - Lots of testing will be needed for the various edge cases. --- .../suite/rpl/r/rpl_parallel_retry.result | 62 +++++++ .../suite/rpl/t/rpl_parallel_retry.test | 91 ++++++++++ sql/rpl_parallel.cc | 163 ++++++++++++++++-- sql/rpl_parallel.h | 6 +- sql/rpl_rli.cc | 39 ++++- sql/rpl_rli.h | 36 ++++ sql/slave.cc | 14 +- sql/slave.h | 1 + 8 files changed, 394 insertions(+), 18 deletions(-) create mode 100644 mysql-test/suite/rpl/r/rpl_parallel_retry.result create mode 100644 mysql-test/suite/rpl/t/rpl_parallel_retry.test diff --git a/mysql-test/suite/rpl/r/rpl_parallel_retry.result b/mysql-test/suite/rpl/r/rpl_parallel_retry.result new file mode 100644 index 00000000000..352285a91bf --- /dev/null +++ b/mysql-test/suite/rpl/r/rpl_parallel_retry.result @@ -0,0 +1,62 @@ +include/rpl_init.inc [topology=1->2] +*** Test retry of transactions that fail to replicate due to deadlock or similar temporary error. *** +ALTER TABLE mysql.gtid_slave_pos ENGINE=InnoDB; +CREATE TABLE t1 (a int PRIMARY KEY, b INT) ENGINE=InnoDB; +INSERT INTO t1 VALUES (1,1); +SET sql_log_bin=0; +CREATE FUNCTION foo(x INT, d1 VARCHAR(500), d2 VARCHAR(500)) +RETURNS INT DETERMINISTIC +BEGIN +RETURN x; +END +|| +SET sql_log_bin=1; +SET @old_parallel_threads=@@GLOBAL.slave_parallel_threads; +include/stop_slave.inc +SET GLOBAL slave_parallel_threads=5; +include/start_slave.inc +SET sql_log_bin=0; +CREATE FUNCTION foo(x INT, d1 VARCHAR(500), d2 VARCHAR(500)) +RETURNS INT DETERMINISTIC +BEGIN +IF d1 != '' THEN +SET debug_sync = d1; +END IF; +IF d2 != '' THEN +SET debug_sync = d2; +END IF; +RETURN x; +END +|| +include/stop_slave.inc +SET @old_format= @@SESSION.binlog_format; +SET binlog_format='statement'; +SET gtid_seq_no = 100; +BEGIN; +INSERT INTO t1 VALUES (2,1); +UPDATE t1 SET b=b+1 WHERE a=1; +INSERT INTO t1 VALUES (3,1); +COMMIT; +SET binlog_format=@old_format; +SELECT * FROM t1 ORDER BY a; +a b +1 2 +2 1 +3 1 +SET @old_dbug= @@GLOBAL.debug_dbug; +SET GLOBAL debug_dbug="+d,rpl_parallel_simulate_temp_err_gtid_0_1_100"; +include/start_slave.inc +SET GLOBAL debug_dbug=@old_dbug; +retries +1 +SELECT * FROM t1 ORDER BY a; +a b +1 2 +2 1 +3 1 +include/stop_slave.inc +SET GLOBAL slave_parallel_threads=@old_parallel_threads; +include/start_slave.inc +DROP TABLE t1; +DROP function foo; +include/rpl_end.inc diff --git a/mysql-test/suite/rpl/t/rpl_parallel_retry.test b/mysql-test/suite/rpl/t/rpl_parallel_retry.test new file mode 100644 index 00000000000..edf00269737 --- /dev/null +++ b/mysql-test/suite/rpl/t/rpl_parallel_retry.test @@ -0,0 +1,91 @@ +--source include/have_innodb.inc +--source include/have_debug.inc +--source include/have_debug_sync.inc +--let $rpl_topology=1->2 +--source include/rpl_init.inc + +--echo *** Test retry of transactions that fail to replicate due to deadlock or similar temporary error. *** + +--connection server_1 +ALTER TABLE mysql.gtid_slave_pos ENGINE=InnoDB; +CREATE TABLE t1 (a int PRIMARY KEY, b INT) ENGINE=InnoDB; +INSERT INTO t1 VALUES (1,1); +--save_master_pos + +# Use a stored function to inject a debug_sync into the appropriate THD. +# The function does nothing on the master, and on the slave it injects the +# desired debug_sync action(s). +SET sql_log_bin=0; +--delimiter || +CREATE FUNCTION foo(x INT, d1 VARCHAR(500), d2 VARCHAR(500)) + RETURNS INT DETERMINISTIC + BEGIN + RETURN x; + END +|| +--delimiter ; +SET sql_log_bin=1; + +--connection server_2 +SET @old_parallel_threads=@@GLOBAL.slave_parallel_threads; +--source include/stop_slave.inc +SET GLOBAL slave_parallel_threads=5; +--source include/start_slave.inc +--sync_with_master +SET sql_log_bin=0; +--delimiter || +CREATE FUNCTION foo(x INT, d1 VARCHAR(500), d2 VARCHAR(500)) + RETURNS INT DETERMINISTIC + BEGIN + IF d1 != '' THEN + SET debug_sync = d1; + END IF; + IF d2 != '' THEN + SET debug_sync = d2; + END IF; + RETURN x; + END +|| +--delimiter ; +--source include/stop_slave.inc + +--connection server_1 +SET @old_format= @@SESSION.binlog_format; +SET binlog_format='statement'; +SET gtid_seq_no = 100; +BEGIN; +INSERT INTO t1 VALUES (2,1); +UPDATE t1 SET b=b+1 WHERE a=1; +#INSERT INTO t1 VALUES (3,foo(1, +# "ha_write_row_end SIGNAL q1_ready WAIT_FOR q1_cont", +# "")); +INSERT INTO t1 VALUES (3,1); +COMMIT; +SET binlog_format=@old_format; +SELECT * FROM t1 ORDER BY a; +--save_master_pos + +--connection server_2 +SET @old_dbug= @@GLOBAL.debug_dbug; +SET GLOBAL debug_dbug="+d,rpl_parallel_simulate_temp_err_gtid_0_1_100"; +let $old_retry= query_get_value(SHOW STATUS LIKE 'Slave_retried_transactions', Value, 1); +--source include/start_slave.inc +--sync_with_master +SET GLOBAL debug_dbug=@old_dbug; +let $new_retry= query_get_value(SHOW STATUS LIKE 'Slave_retried_transactions', Value, 1); +--disable_query_log +eval SELECT $new_retry - $old_retry AS retries; +--enable_query_log + +SELECT * FROM t1 ORDER BY a; + +--connection server_2 +--source include/stop_slave.inc +SET GLOBAL slave_parallel_threads=@old_parallel_threads; +--source include/start_slave.inc + +--connection server_1 +DROP TABLE t1; +DROP function foo; + +--source include/rpl_end.inc diff --git a/sql/rpl_parallel.cc b/sql/rpl_parallel.cc index 53769107661..f0147527957 100644 --- a/sql/rpl_parallel.cc +++ b/sql/rpl_parallel.cc @@ -7,15 +7,6 @@ /* Code for optional parallel execution of replicated events on the slave. - - ToDo list: - - - Retry of failed transactions is not yet implemented for the parallel case. - - - All the waits (eg. in struct wait_for_commit and in - rpl_parallel_thread_pool::get_thread()) need to be killable. And on kill, - everything needs to be correctly rolled back and stopped in all threads, - to ensure a consistent slave replication state. */ struct rpl_parallel_thread_pool global_rpl_thread_pool; @@ -197,6 +188,105 @@ unlock_or_exit_cond(THD *thd, mysql_mutex_t *lock, bool *did_enter_cond, } +static int +retry_handle_relay_log_rotate(Log_event *ev, IO_CACHE *rlog) +{ + /* ToDo */ + return 0; +} + + +static int +retry_event_group(rpl_group_info *rgi, rpl_parallel_thread *rpt, + rpl_parallel_thread::queued_event *orig_qev) +{ + IO_CACHE rlog; + File fd; + const char *errmsg= NULL; + inuse_relaylog *ir= rgi->relay_log; + uint64 event_count= 0; + uint64 events_to_execute= rgi->retry_event_count; + Relay_log_info *rli= rgi->rli; + int err= 0; + ulonglong cur_offset, old_offset; + char log_name[FN_REFLEN]; + THD *thd= rgi->thd; + +do_retry: + rgi->cleanup_context(thd, 1); + + mysql_mutex_lock(&rli->data_lock); + ++rli->retried_trans; + statistic_increment(slave_retried_transactions, LOCK_status); + mysql_mutex_unlock(&rli->data_lock); + + strcpy(log_name, ir->name); + if ((fd= open_binlog(&rlog, log_name, &errmsg)) <0) + return 1; + cur_offset= rgi->retry_start_offset; + my_b_seek(&rlog, cur_offset); + + do + { + Log_event_type event_type; + Log_event *ev; + + old_offset= cur_offset; + ev= Log_event::read_log_event(&rlog, 0, + rli->relay_log.description_event_for_exec /* ToDo: this needs fixing */, + opt_slave_sql_verify_checksum); + cur_offset= my_b_tell(&rlog); + + if (!ev) + { + err= 1; + goto err; + } + ev->thd= thd; + event_type= ev->get_type_code(); + if (Log_event::is_group_event(event_type)) + { + rpl_parallel_thread::queued_event *qev; + + mysql_mutex_lock(&rpt->LOCK_rpl_thread); + qev= rpt->retry_get_qev(ev, orig_qev, log_name, cur_offset, + cur_offset - old_offset); + mysql_mutex_unlock(&rpt->LOCK_rpl_thread); + if (!qev) + { + delete ev; + my_error(ER_OUT_OF_RESOURCES, MYF(0)); + err= 1; + goto err; + } + err= rpt_handle_event(qev, rpt); + ++event_count; + mysql_mutex_lock(&rpt->LOCK_rpl_thread); + rpt->free_qev(qev); + mysql_mutex_unlock(&rpt->LOCK_rpl_thread); + } + else + err= retry_handle_relay_log_rotate(ev, &rlog); + delete_or_keep_event_post_apply(rgi, event_type, ev); + + if (err) + { + /* ToDo: Need to here also handle second retry. */ + goto err; + } + + // ToDo: handle too many retries. + + } while (event_count < events_to_execute); + +err: + + end_io_cache(&rlog); + mysql_file_close(fd, MYF(MY_WME)); + return err; +} + + pthread_handler_t handle_rpl_parallel_thread(void *arg) { @@ -499,7 +589,23 @@ handle_rpl_parallel_thread(void *arg) everything is stopped and cleaned up correctly. */ if (likely(!rgi->worker_error) && !skip_event_group) + { + ++rgi->retry_event_count; err= rpt_handle_event(events, rpt); + DBUG_EXECUTE_IF("rpl_parallel_simulate_temp_err_gtid_0_1_100", + if (rgi->current_gtid.domain_id == 0 && + rgi->current_gtid.server_id == 1 && + rgi->current_gtid.seq_no == 100 && + rgi->retry_event_count == 4) + { + thd->clear_error(); + thd->get_stmt_da()->reset_diagnostics_area(); + my_error(ER_LOCK_DEADLOCK, MYF(0)); + err= 1; + };); + if (err && has_temporary_error(thd)) + err= retry_event_group(rgi, rpt, events); + } else err= thd->wait_for_prior_commit(); @@ -802,8 +908,7 @@ err: rpl_parallel_thread::queued_event * -rpl_parallel_thread::get_qev(Log_event *ev, ulonglong event_size, - Relay_log_info *rli) +rpl_parallel_thread::get_qev_common(Log_event *ev, ulonglong event_size) { queued_event *qev; mysql_mutex_assert_owner(&LOCK_rpl_thread); @@ -817,6 +922,17 @@ rpl_parallel_thread::get_qev(Log_event *ev, ulonglong event_size, qev->ev= ev; qev->event_size= event_size; qev->next= NULL; + return qev; +} + + +rpl_parallel_thread::queued_event * +rpl_parallel_thread::get_qev(Log_event *ev, ulonglong event_size, + Relay_log_info *rli) +{ + queued_event *qev= get_qev_common(ev, event_size); + if (!qev) + return NULL; strcpy(qev->event_relay_log_name, rli->event_relay_log_name); qev->event_relay_log_pos= rli->event_relay_log_pos; qev->future_event_relay_log_pos= rli->future_event_relay_log_pos; @@ -825,6 +941,24 @@ rpl_parallel_thread::get_qev(Log_event *ev, ulonglong event_size, } +rpl_parallel_thread::queued_event * +rpl_parallel_thread::retry_get_qev(Log_event *ev, queued_event *orig_qev, + const char *relay_log_name, + ulonglong event_pos, ulonglong event_size) +{ + queued_event *qev= get_qev_common(ev, event_size); + if (!qev) + return NULL; + qev->rgi= orig_qev->rgi; + strcpy(qev->event_relay_log_name, relay_log_name); + qev->event_relay_log_pos= event_pos; + qev->future_event_relay_log_pos= event_pos+event_size; + strcpy(qev->future_event_master_log_name, + orig_qev->future_event_master_log_name); + return qev; +} + + void rpl_parallel_thread::free_qev(rpl_parallel_thread::queued_event *qev) { @@ -836,7 +970,7 @@ rpl_parallel_thread::free_qev(rpl_parallel_thread::queued_event *qev) rpl_group_info* rpl_parallel_thread::get_rgi(Relay_log_info *rli, Gtid_log_event *gtid_ev, - rpl_parallel_entry *e) + rpl_parallel_entry *e, ulonglong event_size) { rpl_group_info *rgi; mysql_mutex_assert_owner(&LOCK_rpl_thread); @@ -864,6 +998,9 @@ rpl_parallel_thread::get_rgi(Relay_log_info *rli, Gtid_log_event *gtid_ev, return NULL; } rgi->parallel_entry= e; + rgi->relay_log= rli->last_inuse_relaylog; + rgi->retry_start_offset= rli->future_event_relay_log_pos-event_size; + rgi->retry_event_count= 0; return rgi; } @@ -1439,7 +1576,7 @@ rpl_parallel::do_event(rpl_group_info *serial_rgi, Log_event *ev, { Gtid_log_event *gtid_ev= static_cast(ev); - if (!(rgi= cur_thread->get_rgi(rli, gtid_ev, e))) + if (!(rgi= cur_thread->get_rgi(rli, gtid_ev, e, event_size))) { cur_thread->free_qev(qev); abandon_worker_thread(rli->sql_driver_thd, cur_thread, diff --git a/sql/rpl_parallel.h b/sql/rpl_parallel.h index 1808efd0926..3b6641523f6 100644 --- a/sql/rpl_parallel.h +++ b/sql/rpl_parallel.h @@ -106,11 +106,15 @@ struct rpl_parallel_thread { queued_size-= dequeue_size; } + queued_event *get_qev_common(Log_event *ev, ulonglong event_size); queued_event *get_qev(Log_event *ev, ulonglong event_size, Relay_log_info *rli); + queued_event *retry_get_qev(Log_event *ev, queued_event *orig_qev, + const char *relay_log_name, + ulonglong event_pos, ulonglong event_size); void free_qev(queued_event *qev); rpl_group_info *get_rgi(Relay_log_info *rli, Gtid_log_event *gtid_ev, - rpl_parallel_entry *e); + rpl_parallel_entry *e, ulonglong event_size); void free_rgi(rpl_group_info *rgi); group_commit_orderer *get_gco(uint64 wait_count, group_commit_orderer *prev); void free_gco(group_commit_orderer *gco); diff --git a/sql/rpl_rli.cc b/sql/rpl_rli.cc index cc543f7c377..3a3e22f970a 100644 --- a/sql/rpl_rli.cc +++ b/sql/rpl_rli.cc @@ -52,6 +52,7 @@ Relay_log_info::Relay_log_info(bool is_slave_recovery) info_fd(-1), cur_log_fd(-1), relay_log(&sync_relaylog_period), sync_counter(0), is_relay_log_recovery(is_slave_recovery), save_temporary_tables(0), mi(0), + inuse_relaylog_list(0), last_inuse_relaylog(0), cur_log_old_open_count(0), group_relay_log_pos(0), event_relay_log_pos(0), #if HAVE_valgrind @@ -98,8 +99,17 @@ Relay_log_info::Relay_log_info(bool is_slave_recovery) Relay_log_info::~Relay_log_info() { + inuse_relaylog *cur; DBUG_ENTER("Relay_log_info::~Relay_log_info"); + cur= inuse_relaylog_list; + while (cur) + { + DBUG_ASSERT(cur->queued_count == cur->dequeued_count); + inuse_relaylog *next= cur->next; + my_free(cur); + cur= next; + } mysql_mutex_destroy(&run_lock); mysql_mutex_destroy(&data_lock); mysql_mutex_destroy(&log_space_lock); @@ -1339,6 +1349,29 @@ void Relay_log_info::stmt_done(my_off_t event_master_log_pos, DBUG_VOID_RETURN; } + +int +Relay_log_info::alloc_inuse_relaylog(const char *name) +{ + inuse_relaylog *ir; + + if (!(ir= (inuse_relaylog *)my_malloc(sizeof(*ir), MYF(MY_WME|MY_ZEROFILL)))) + { + my_error(ER_OUTOFMEMORY, MYF(0), (int)sizeof(*ir)); + return 1; + } + strcpy(ir->name, name); + + if (!inuse_relaylog_list) + inuse_relaylog_list= ir; + else + last_inuse_relaylog->next= ir; + last_inuse_relaylog= ir; + + return 0; +} + + #if !defined(MYSQL_CLIENT) && defined(HAVE_REPLICATION) int rpl_load_gtid_slave_state(THD *thd) @@ -1623,7 +1656,7 @@ delete_or_keep_event_post_apply(rpl_group_info *rgi, void rpl_group_info::cleanup_context(THD *thd, bool error) { - DBUG_ENTER("Relay_log_info::cleanup_context"); + DBUG_ENTER("rpl_group_info::cleanup_context"); DBUG_PRINT("enter", ("error: %d", (int) error)); DBUG_ASSERT(this->thd == thd); @@ -1689,7 +1722,7 @@ void rpl_group_info::cleanup_context(THD *thd, bool error) void rpl_group_info::clear_tables_to_lock() { - DBUG_ENTER("Relay_log_info::clear_tables_to_lock()"); + DBUG_ENTER("rpl_group_info::clear_tables_to_lock()"); #ifndef DBUG_OFF /** When replicating in RBR and MyISAM Merge tables are involved @@ -1736,7 +1769,7 @@ void rpl_group_info::clear_tables_to_lock() void rpl_group_info::slave_close_thread_tables(THD *thd) { - DBUG_ENTER("Relay_log_info::slave_close_thread_tables(THD *thd)"); + DBUG_ENTER("rpl_group_info::slave_close_thread_tables(THD *thd)"); thd->get_stmt_da()->set_overwrite_status(true); thd->is_error() ? trans_rollback_stmt(thd) : trans_commit_stmt(thd); thd->get_stmt_da()->set_overwrite_status(false); diff --git a/sql/rpl_rli.h b/sql/rpl_rli.h index 00d16f52488..c2cdbcdc573 100644 --- a/sql/rpl_rli.h +++ b/sql/rpl_rli.h @@ -61,6 +61,7 @@ enum { *****************************************************************************/ struct rpl_group_info; +struct inuse_relaylog; class Relay_log_info : public Slave_reporting_capability { @@ -163,6 +164,13 @@ public: /* parent Master_info structure */ Master_info *mi; + /* + List of active relay log files. + (This can be more than one in case of parallel replication). + */ + inuse_relaylog *inuse_relaylog_list; + inuse_relaylog *last_inuse_relaylog; + /* Needed to deal properly with cur_log getting closed and re-opened with a different log under our feet @@ -398,6 +406,7 @@ public: void stmt_done(my_off_t event_log_pos, time_t event_creation_time, THD *thd, rpl_group_info *rgi); + int alloc_inuse_relaylog(const char *name); /** Is the replication inside a group? @@ -463,6 +472,25 @@ private: }; +/* + In parallel replication, if we need to re-try a transaction due to a + deadlock or other temporary error, we may need to go back and re-read events + out of an earlier relay log. + + This structure keeps track of the relaylogs that are potentially in use. + Each rpl_group_info has a pointer to one of those, corresponding to the + first GTID event. + + A reference count keeps track of how long a relay log is potentially in use. +*/ +struct inuse_relaylog { + inuse_relaylog *next; + uint64 queued_count; + uint64 dequeued_count; + char name[FN_REFLEN]; +}; + + /* This is data for various state needed to be kept for the processing of one event group (transaction) during replication. @@ -596,6 +624,14 @@ struct rpl_group_info /* Needs room for "Gtid D-S-N\x00". */ char gtid_info_buf[5+10+1+10+1+20+1]; + /* + Information to be able to re-try an event group in case of a deadlock or + other temporary error. + */ + inuse_relaylog *relay_log; + uint64 retry_start_offset; + uint64 retry_event_count; + rpl_group_info(Relay_log_info *rli_); ~rpl_group_info(); void reinit(Relay_log_info *rli); diff --git a/sql/slave.cc b/sql/slave.cc index f755cb63558..ab505a4011f 100644 --- a/sql/slave.cc +++ b/sql/slave.cc @@ -3094,7 +3094,8 @@ static ulong read_event(MYSQL* mysql, Master_info *mi, bool* suppress_warnings) that the error is temporary by pushing a warning with the error code ER_GET_TEMPORARY_ERRMSG, if the originating error is temporary. */ -static int has_temporary_error(THD *thd) +int +has_temporary_error(THD *thd) { DBUG_ENTER("has_temporary_error"); @@ -4478,6 +4479,9 @@ pthread_handler_t handle_slave_sql(void *arg) "Error initializing relay log position: %s", errmsg); goto err; } + if (rli->alloc_inuse_relaylog(rli->group_relay_log_name)) + goto err; + strcpy(rli->future_event_master_log_name, rli->group_master_log_name); THD_CHECK_SENTRY(thd); #ifndef DBUG_OFF @@ -6521,6 +6525,12 @@ static Log_event* next_event(rpl_group_info *rgi, ulonglong *event_size) mysql_mutex_unlock(log_lock); goto err; } + if (rli->alloc_inuse_relaylog(rli->linfo.log_file_name)) + { + if (!hot_log) + mysql_mutex_unlock(log_lock); + goto err; + } if (!hot_log) mysql_mutex_unlock(log_lock); continue; @@ -6536,6 +6546,8 @@ static Log_event* next_event(rpl_group_info *rgi, ulonglong *event_size) if ((rli->cur_log_fd=open_binlog(cur_log,rli->linfo.log_file_name, &errmsg)) <0) goto err; + if (rli->alloc_inuse_relaylog(rli->linfo.log_file_name)) + goto err; } else { diff --git a/sql/slave.h b/sql/slave.h index 7352ac0274b..4b5bc1686fb 100644 --- a/sql/slave.h +++ b/sql/slave.h @@ -229,6 +229,7 @@ int purge_relay_logs(Relay_log_info* rli, THD *thd, bool just_reset, void set_slave_thread_options(THD* thd); void set_slave_thread_default_charset(THD *thd, rpl_group_info *rgi); int rotate_relay_log(Master_info* mi); +int has_temporary_error(THD *thd); int apply_event_and_update_pos(Log_event* ev, THD* thd, struct rpl_group_info *rgi, rpl_parallel_thread *rpt); From 2870bd7423b5bb00360d5e24f44ded071c40be07 Mon Sep 17 00:00:00 2001 From: Venkatesh Duggirala Date: Thu, 8 May 2014 18:13:01 +0530 Subject: [PATCH 022/279] Bug#17283409 4-WAY DEADLOCK: ZOMBIES, PURGING BINLOGS, SHOW PROCESSLIST, SHOW BINLOGS Problem: A deadlock was occurring when 4 threads were involved in acquiring locks in the following way Thread 1: Dump thread ( Slave is reconnecting, so on Master, a new dump thread is trying kill zombie dump threads. It acquired thread's LOCK_thd_data and it is about to acquire mysys_var->current_mutex ( which LOCK_log) Thread 2: Application thread is executing show binlogs and acquired LOCK_log and it is about to acquire LOCK_index. Thread 3: Application thread is executing Purge binary logs and acquired LOCK_index and it is about to acquire LOCK_thread_count. Thread 4: Application thread is executing show processlist and acquired LOCK_thread_count and it is about to acquire zombie dump thread's LOCK_thd_data. Deadlock Cycle: Thread 1 -> Thread 2 -> Thread 3-> Thread 4 ->Thread 1 The same above deadlock was observed even when thread 4 is executing 'SELECT * FROM information_schema.processlist' command and acquired LOCK_thread_count and it is about to acquire zombie dump thread's LOCK_thd_data. Analysis: There are four locks involved in the deadlock. LOCK_log, LOCK_thread_count, LOCK_index and LOCK_thd_data. LOCK_log, LOCK_thread_count, LOCK_index are global mutexes where as LOCK_thd_data is local to a thread. We can divide these four locks in two groups. Group 1 consists of LOCK_log and LOCK_index and the order should be LOCK_log followed by LOCK_index. Group 2 consists of other two mutexes LOCK_thread_count, LOCK_thd_data and the order should be LOCK_thread_count followed by LOCK_thd_data. Unfortunately, there is no specific predefined lock order defined to follow in the MySQL system when it comes to locks across these two groups. In the above problematic example, there is no problem in the way we are acquiring the locks if you see each thread individually. But If you combine all 4 threads, they end up in a deadlock. Fix: Since everything seems to be fine in the way threads are taking locks, In this patch We are changing the duration of the locks in Thread 4 to break the deadlock. i.e., before the patch, Thread 4 ('show processlist' command) mysqld_list_processes() function acquires LOCK_thread_count for the complete duration of the function and it also acquires/releases each thread's LOCK_thd_data. LOCK_thread_count is used to protect addition and deletion of threads in global threads list. While show process list is looping through all the existing threads, it will be a problem if a thread is exited but there is no problem if a new thread is added to the system. Hence a new mutex is introduced "LOCK_thd_remove" which will protect deletion of a thread from global threads list. All threads which are getting exited should acquire LOCK_thd_remove followed by LOCK_thread_count. (It should take LOCK_thread_count also because other places of the code still thinks that exit thread is protected with LOCK_thread_count. In this fix, we are changing only 'show process list' query logic ) (Eg: unlink_thd logic will be protected with LOCK_thd_remove). Logic of mysqld_list_processes(or file_schema_processlist) will now be protected with 'LOCK_thd_remove' instead of 'LOCK_thread_count'. Now the new locking order after this patch is: LOCK_thd_remove -> LOCK_thd_data -> LOCK_log -> LOCK_index -> LOCK_thread_count --- include/mysql/thread_pool_priv.h | 4 +- mysql-test/r/show_processlist.result | 47 +++++++ .../suite/rpl/r/rpl_4threads_deadlock.result | 43 +++++++ .../suite/rpl/t/rpl_4threads_deadlock.test | 117 ++++++++++++++++++ mysql-test/t/show_processlist.test | 63 ++++++++++ sql/event_scheduler.cc | 8 +- sql/log.cc | 16 +-- sql/mysqld.cc | 17 ++- sql/mysqld.h | 6 +- sql/scheduler.cc | 3 +- sql/slave.cc | 6 +- sql/sql_class.cc | 29 ++++- sql/sql_insert.cc | 4 +- sql/sql_repl.cc | 5 +- sql/sql_show.cc | 42 +++++-- 15 files changed, 377 insertions(+), 33 deletions(-) create mode 100644 mysql-test/r/show_processlist.result create mode 100644 mysql-test/suite/rpl/r/rpl_4threads_deadlock.result create mode 100644 mysql-test/suite/rpl/t/rpl_4threads_deadlock.test create mode 100644 mysql-test/t/show_processlist.test diff --git a/include/mysql/thread_pool_priv.h b/include/mysql/thread_pool_priv.h index efc264b713d..1da174e5f8b 100644 --- a/include/mysql/thread_pool_priv.h +++ b/include/mysql/thread_pool_priv.h @@ -1,5 +1,5 @@ /* - Copyright (c) 2010, 2012, Oracle and/or its affiliates. All rights reserved. + Copyright (c) 2010, 2014, Oracle and/or its affiliates. All rights reserved. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by @@ -49,6 +49,8 @@ void thd_clear_errors(THD *thd); void thd_set_thread_stack(THD *thd, char *stack_start); void thd_lock_thread_count(THD *thd); void thd_unlock_thread_count(THD *thd); +void thd_lock_thread_remove(THD *thd); +void thd_unlock_thread_remove(THD *thd); void thd_close_connection(THD *thd); THD *thd_get_current_thd(); void thd_new_connection_setup(THD *thd, char *stack_start); diff --git a/mysql-test/r/show_processlist.result b/mysql-test/r/show_processlist.result new file mode 100644 index 00000000000..f133e480e1a --- /dev/null +++ b/mysql-test/r/show_processlist.result @@ -0,0 +1,47 @@ +Bug#17283409 4-WAY DEADLOCK: ZOMBIES, PURGING BINLOGS, SHOW PROCESSLIST, +SHOW BINLOGS +connect connection1,127.0.0.1,root,,test,$MASTER_MYPORT,; +connection default; +SET DEBUG_SYNC='before_one_element_read_from_threads_iterator SIGNAL parked1 WAIT_FOR go'; +SHOW PROCESSLIST; +connection connection1; +"Wait_for parked1" +SET DEBUG_SYNC='now WAIT_FOR parked1'; +connect connection2,127.0.0.1,root,,test,$MASTER_MYPORT,; +connect connection3,127.0.0.1,root,,test,$MASTER_MYPORT,; +SET DEBUG_SYNC='now SIGNAL go'; +connection default; +Id User Host db Command Time State Info + root test