Make sure the ORDER BY collating sequences are compatible with the

comparison collations before using the merge algorithm for compound
SELECT statements.  Candidate fix for ticket [6709574d2a8d8].

FossilOrigin-Name: fc3630cdef6e2cdbfb4e7b373d1a094753e55016

manifest

@@ -1,5 +1,5 @@
-C Modify\sthe\sfts3tokenize\stable\simplementation\sso\sthat\sit\sdoes\snot\suse\sthe\sSQL\sfunction\sfts3_tokenizer.\sThe\suser\smay\shave\sinstalled\san\sauthorizer\scallback\sthat\sprohibits\sthis.
+C Make\ssure\sthe\sORDER\sBY\scollating\ssequences\sare\scompatible\swith\sthe\ncomparison\scollations\sbefore\susing\sthe\smerge\salgorithm\sfor\scompound\nSELECT\sstatements.\s\sCandidate\sfix\sfor\sticket\s[6709574d2a8d8].
-D 2013-05-07T12:16:48.845
+D 2013-05-07T17:49:08.546
 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f
 F Makefile.in ce81671efd6223d19d4c8c6b88ac2c4134427111
 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23
@@ -190,7 +190,7 @@ F src/printf.c 4a9f882f1c1787a8b494a2987765acf9d97ac21f
 F src/random.c cd4a67b3953b88019f8cd4ccd81394a8ddfaba50
 F src/resolve.c 83cc2d942ee216bc56956c6e6fadb691c1727fa1
 F src/rowset.c 64655f1a627c9c212d9ab497899e7424a34222e0
-F src/select.c 6bfbe11e2fef81c5e18d30513ab6c69f171667eb
+F src/select.c a4641882279becc200f2680f55f3e89d4e7c7f78
 F src/shell.c 2109d54f67c815a100abd7dc6a6e25eddb3b97eb
 F src/sqlite.h.in 5a5a22a9b192d81a9e5dee00274e3a0484c4afb1
 F src/sqlite3.rc fea433eb0a59f4c9393c8e6d76a6e2596b1fe0c0
@@ -723,10 +723,11 @@ F test/select6.test e76bd10a56988f15726c097a5d5a7966fe82d3b2
 F test/select7.test dad6f00f0d49728a879d6eb6451d4752db0b0abe
 F test/select8.test 391de11bdd52339c30580dabbbbe97e3e9a3c79d
 F test/select9.test c0ca3cd87a8ebb04de2cb1402c77df55d911a0ea
-F test/selectA.test 06d1032fa9009314c95394f2ca2e60d9f7ae8532
+F test/selectA.test 99cf21df033b93033ea4f34aba14a500f48f04fe
 F test/selectB.test 954e4e49cf1f896d61794e440669e03a27ceea25
 F test/selectC.test 871fb55d884d3de5943c4057ebd22c2459e71977
 F test/selectD.test b0f02a04ef7737decb24e08be2c39b9664b43394
+F test/selectE.test fc02a1eb04c8eb537091482644b7d778ae8759b7
 F test/server1.test 46803bd3fe8b99b30dbc5ff38ffc756f5c13a118
 F test/shared.test 1da9dbad400cee0d93f252ccf76e1ae007a63746
 F test/shared2.test 03eb4a8d372e290107d34b6ce1809919a698e879
@@ -1061,7 +1062,10 @@ F tool/vdbe-compress.tcl f12c884766bd14277f4fcedcae07078011717381
 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4
 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381
 F tool/win/sqlite.vsix 97894c2790eda7b5bce3cc79cb2a8ec2fde9b3ac
-P e5b3cd747bb0b484e38b8611a81925e2cc144435
+P 0ba67b64de258883e4c43db09e131bb67083855e
-R 0fdf3312f3c095b75e4d2deab47476b0
+R 298d9b4b04cefe916bd24254e1442751
-U dan
+T *branch * tkt-6709574
-Z 58e39fd3c70070287886f650d61f780d
+T *sym-tkt-6709574 *
+T -sym-trunk *
+U drh
+Z 52d856a3284951cf8623e01987d271c7

manifest.uuid

@@ -1 +1 @@
-0ba67b64de258883e4c43db09e131bb67083855e
+fc3630cdef6e2cdbfb4e7b373d1a094753e55016

src/select.c

@@ -3260,6 +3260,69 @@ int sqlite3IndexedByLookup(Parse *pParse, struct SrcList_item *pFrom){
   }
   return SQLITE_OK;
 }
+/*
+** Detect compound SELECT statements that use an ORDER BY clause with 
+** an alternative collating sequence.
+**
+**    SELECT ... FROM t1 EXCEPT SELECT ... FROM t2 ORDER BY .. COLLATE ...
+**
+** These are rewritten as a subquery:
+**
+**    SELECT * FROM (SELECT ... FROM t1 EXCEPT SELECT ... FROM t2)
+**     ORDER BY ... COLLATE ...
+**
+** This transformation is necessary because the multiSelectOrderBy() routine
+** above that generates the code for a compound SELECT with an ORDER BY clause
+** uses a merge algorithm that requires the same collating sequence on the
+** result columns as on the ORDER BY clause.  See ticket
+** http://www.sqlite.org/src/info/6709574d2a
+**
+** This transformation is only needed for EXCEPT, INTERSECT, and UNION.
+** The UNION ALL operator works fine with multiSelectOrderBy() even when
+** there are COLLATE terms in the ORDER BY.
+*/
+static int convertCompoundSelectToSubquery(Walker *pWalker, Select *p){
+  int i;
+  Select *pNew;
+  Select *pX;
+  sqlite3 *db;
+  struct ExprList_item *a;
+  SrcList *pNewSrc;
+  Parse *pParse;
+  Token dummy;
+
+  if( p->pPrior==0 ) return WRC_Continue;
+  if( p->pOrderBy==0 ) return WRC_Continue;
+  for(pX=p; pX && (pX->op==TK_ALL || pX->op==TK_SELECT); pX=pX->pPrior){}
+  if( pX==0 ) return WRC_Continue;
+  a = p->pOrderBy->a;
+  for(i=p->pOrderBy->nExpr-1; i>=0; i--){
+    if( a[i].pExpr->flags & EP_Collate ) break;
+  }
+  if( i<0 ) return WRC_Continue;
+
+  /* If we reach this point, that means the transformation is required. */
+
+  pParse = pWalker->pParse;
+  db = pParse->db;
+  pNew = sqlite3DbMallocZero(db, sizeof(*pNew) );
+  if( pNew==0 ) return WRC_Abort;
+  memset(&dummy, 0, sizeof(dummy));
+  pNewSrc = sqlite3SrcListAppendFromTerm(pParse,0,0,0,&dummy,pNew,0,0);
+  if( pNewSrc==0 ) return WRC_Abort;
+  *pNew = *p;
+  p->pSrc = pNewSrc;
+  p->pEList = sqlite3ExprListAppend(pParse, 0, sqlite3Expr(db, TK_ALL, 0));
+  p->op = TK_SELECT;
+  p->pWhere = 0;
+  pNew->pGroupBy = 0;
+  pNew->pHaving = 0;
+  pNew->pOrderBy = 0;
+  p->pPrior = 0;
+  pNew->pLimit = 0;
+  pNew->pOffset = 0;
+  return WRC_Continue;
+}
 
 /*
 ** This routine is a Walker callback for "expanding" a SELECT statement.
@@ -3577,10 +3640,12 @@ static int exprWalkNoop(Walker *NotUsed, Expr *NotUsed2){
 static void sqlite3SelectExpand(Parse *pParse, Select *pSelect){
   Walker w;
   memset(&w, 0, sizeof(w));
-  w.xSelectCallback = selectExpander;
+  w.xSelectCallback = convertCompoundSelectToSubquery;
   w.xExprCallback = exprWalkNoop;
   w.pParse = pParse;
   sqlite3WalkSelect(&w, pSelect);
+  w.xSelectCallback = selectExpander;
+  sqlite3WalkSelect(&w, pSelect);
 }
 
 

test/selectA.test

@@ -281,13 +281,13 @@ do_test selectA-2.34 {
 do_test selectA-2.35 {
   execsql {
     SELECT x,y,z FROM t2 UNION SELECT a,b,c FROM t1
-    ORDER BY b COLLATE NOCASE,a,c
+    ORDER BY y COLLATE NOCASE,x,z
   }
 } {1 a a 9.9 b B {} C c hello d D abc e e hare m M {} U u 5200000.0 X x -23 Y y mad Z z}
 do_test selectA-2.36 {
   execsql {
     SELECT x,y,z FROM t2 UNION SELECT a,b,c FROM t1
-    ORDER BY b COLLATE NOCASE DESC,a,c
+    ORDER BY y COLLATE NOCASE DESC,x,z
   }
 } {mad Z z -23 Y y 5200000.0 X x {} U u hare m M abc e e hello d D {} C c 9.9 b B 1 a a}
 do_test selectA-2.37 {
@@ -311,7 +311,7 @@ do_test selectA-2.39 {
 do_test selectA-2.40 {
   execsql {
     SELECT x,y,z FROM t2 UNION SELECT a,b,c FROM t1
-    ORDER BY c COLLATE BINARY DESC,a,b
+    ORDER BY z COLLATE BINARY DESC,x,y
   }
 } {mad Z z -23 Y y 5200000.0 X x {} U u abc e e {} C c 1 a a hare m M hello d D 9.9 b B}
 do_test selectA-2.41 {
@@ -602,7 +602,7 @@ do_test selectA-2.85 {
 do_test selectA-2.86 {
   execsql {
     SELECT x,y,z FROM t2 UNION SELECT a,b,c FROM t3
-    ORDER BY b COLLATE NOCASE,a,c
+    ORDER BY y COLLATE NOCASE,x,z
   }
 } {1 a a 9.9 b B {} C c hello d D abc e e hare m M {} U u 5200000.0 X x -23 Y y mad Z z}
 do_test selectA-2.87 {
@@ -632,7 +632,7 @@ do_test selectA-2.90 {
 do_test selectA-2.91 {
   execsql {
     SELECT x,y,z FROM t2 UNION SELECT a,b,c FROM t3
-    ORDER BY c COLLATE BINARY DESC,a,b
+    ORDER BY z COLLATE BINARY DESC,x,y
   }
 } {mad Z z -23 Y y 5200000.0 X x {} U u abc e e {} C c 1 a a hare m M hello d D 9.9 b B}
 do_test selectA-2.92 {
@@ -893,13 +893,13 @@ do_test selectA-3.34 {
 do_test selectA-3.35 {
   execsql {
     SELECT x,y,z FROM t2 UNION SELECT a,b,c FROM t1
-    ORDER BY b COLLATE NOCASE,a,c
+    ORDER BY y COLLATE NOCASE,x,z
   }
 } {1 a a 9.9 b B {} C c hello d D abc e e hare m M {} U u 5200000.0 X x -23 Y y mad Z z}
 do_test selectA-3.36 {
   execsql {
     SELECT x,y,z FROM t2 UNION SELECT a,b,c FROM t1
-    ORDER BY b COLLATE NOCASE DESC,a,c
+    ORDER BY y COLLATE NOCASE DESC,x,z
   }
 } {mad Z z -23 Y y 5200000.0 X x {} U u hare m M abc e e hello d D {} C c 9.9 b B 1 a a}
 do_test selectA-3.37 {
@@ -923,7 +923,7 @@ do_test selectA-3.39 {
 do_test selectA-3.40 {
   execsql {
     SELECT x,y,z FROM t2 UNION SELECT a,b,c FROM t1
-    ORDER BY c COLLATE BINARY DESC,a,b
+    ORDER BY z COLLATE BINARY DESC,x,y
   }
 } {mad Z z -23 Y y 5200000.0 X x {} U u abc e e {} C c 1 a a hare m M hello d D 9.9 b B}
 do_test selectA-3.41 {
@@ -1214,7 +1214,7 @@ do_test selectA-3.85 {
 do_test selectA-3.86 {
   execsql {
     SELECT x,y,z FROM t2 UNION SELECT a,b,c FROM t3
-    ORDER BY b COLLATE NOCASE,a,c
+    ORDER BY y COLLATE NOCASE,x,z
   }
 } {1 a a 9.9 b B {} C c hello d D abc e e hare m M {} U u 5200000.0 X x -23 Y y mad Z z}
 do_test selectA-3.87 {
@@ -1244,7 +1244,7 @@ do_test selectA-3.90 {
 do_test selectA-3.91 {
   execsql {
     SELECT x,y,z FROM t2 UNION SELECT a,b,c FROM t3
-    ORDER BY c COLLATE BINARY DESC,a,b
+    ORDER BY z COLLATE BINARY DESC,x,y
   }
 } {mad Z z -23 Y y 5200000.0 X x {} U u abc e e {} C c 1 a a hare m M hello d D 9.9 b B}
 do_test selectA-3.92 {

test/selectE.test

@@ -0,0 +1,95 @@
+# 2013-05-07
+#
+# The author disclaims copyright to this source code.  In place of
+# a legal notice, here is a blessing:
+#
+#    May you do good and not evil.
+#    May you find forgiveness for yourself and forgive others.
+#    May you share freely, never taking more than you give.
+#
+#***********************************************************************
+# This file implements regression tests for compound SELECT statements
+# that have ORDER BY clauses with collating sequences that differ
+# from the collating sequence used for comparison in the compound.
+# 
+# Ticket 6709574d2a8d8b9be3a9cb1afbf4ff2de48ea4e7:
+# drh added on 2013-05-06 15:21:16:
+#
+# In the code shown below (which is intended to be run from the
+# sqlite3.exe command-line tool) the three SELECT statements should all
+# generate the same answer. But the third one does not. It is as if the
+# COLLATE clause on the ORDER BY somehow got pulled into the EXCEPT
+# operator. Note that the ".print" commands are instructions to the
+# sqlite3.exe shell program to output delimiter lines so that you can more
+# easily tell where the output of one query ends and the next query
+# begins. 
+# 
+#     CREATE TABLE t1(a);
+#     INSERT INTO t1 VALUES('abc'),('def');
+#     CREATE TABLE t2(a);
+#     INSERT INTO t2 VALUES('DEF');
+# 
+#     SELECT a FROM t1 EXCEPT SELECT a FROM t2 ORDER BY a;
+#     .print -----
+#     SELECT a FROM (SELECT a FROM t1 EXCEPT SELECT a FROM t2)
+#      ORDER BY a COLLATE nocase;
+#     .print -----
+#     SELECT a FROM t1 EXCEPT SELECT a FROM t2 ORDER BY a COLLATE nocase;
+# 
+# Bisecting shows that this problem was introduced in SQLite version 3.6.0
+# by check-in [8bbfa97837a74ef] on 2008-06-15. 
+#
+
+set testdir [file dirname $argv0]
+source $testdir/tester.tcl
+
+do_test selectE-1.0 {
+  db eval {
+    CREATE TABLE t1(a);
+    INSERT INTO t1 VALUES('abc'),('def'),('ghi');
+    CREATE TABLE t2(a);
+    INSERT INTO t2 VALUES('DEF'),('abc');
+    CREATE TABLE t3(a);
+    INSERT INTO t3 VALUES('def'),('jkl');
+
+    SELECT a FROM t1 EXCEPT SELECT a FROM t2
+     ORDER BY a COLLATE nocase;
+  }
+} {def ghi}
+do_test selectE-1.1 {
+  db eval {
+    SELECT a FROM t2 EXCEPT SELECT a FROM t3
+     ORDER BY a COLLATE nocase;
+  }
+} {abc DEF}
+do_test selectE-1.2 {
+  db eval {
+    SELECT a FROM t2 EXCEPT SELECT a FROM t3
+     ORDER BY a COLLATE binary;
+  }
+} {DEF abc}
+do_test selectE-1.3 {
+  db eval {
+    SELECT a FROM t2 EXCEPT SELECT a FROM t3
+     ORDER BY a;
+  }
+} {DEF abc}
+
+do_test selectE-2.1 {
+  db eval {
+    DELETE FROM t2;
+    DELETE FROM t3;
+    INSERT INTO t2 VALUES('ABC'),('def'),('GHI'),('jkl');
+    INSERT INTO t3 SELECT lower(a) FROM t2;
+    SELECT a COLLATE nocase FROM t2 EXCEPT SELECT a FROM t3
+     ORDER BY 1
+  }
+} {}
+do_test selectE-2.2 {
+  db eval {
+    SELECT a COLLATE nocase FROM t2 EXCEPT SELECT a FROM t3
+     ORDER BY 1 COLLATE binary
+  }
+} {}
+
+finish_test