Refactor collating-sequence handling as a fix for ticket [71e333e7d2e642].

The Expr.pColl field is removed from the Expr object.  The COLLATE operator
now becomes a separate instance of Expr in the expression tree.  The code
generator looks up the correct collating function as needed, rather than
referring to Expr.pColl.

FossilOrigin-Name: 8542e6180d4321d45b34f33e481658908ce1430d

manifest

@@ -1,5 +1,5 @@
-C Ensure\sthe\sVerifyCookie\ssub-routine\shas\sbeen\srun\sbefore\sthe\sdatabase\sis\saccessed\sin\san\sobscure\scase.\sFix\sfor\sticket\s[d6b36be38].
+C Refactor\scollating-sequence\shandling\sas\sa\sfix\sfor\sticket\s[71e333e7d2e642].\nThe\sExpr.pColl\sfield\sis\sremoved\sfrom\sthe\sExpr\sobject.\s\sThe\sCOLLATE\soperator\nnow\sbecomes\sa\sseparate\sinstance\sof\sExpr\sin\sthe\sexpression\stree.\s\sThe\scode\ngenerator\slooks\sup\sthe\scorrect\scollating\sfunction\sas\sneeded,\srather\sthan\nreferring\sto\sExpr.pColl.
-D 2012-12-07T19:28:26.162
+D 2012-12-08T21:51:24.433
 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f
 F Makefile.in 690d441a758cbffd13e814dc2724a721a6ebd400
 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23
@@ -124,21 +124,21 @@ F src/btmutex.c 976f45a12e37293e32cae0281b15a21d48a8aaa7
 F src/btree.c eccee944cb2221e919d7a855e5928d8643194b14
 F src/btree.h 3ad7964d6c5b1c7bff569aab6adfa075f8bf06cd
 F src/btreeInt.h 4e5c2bd0f9b36b2a815a6d84f771a61a65830621
-F src/build.c fd68fb1bee609b88f4f7c8ebe4dc9560c583c39d
+F src/build.c f4f86c07002c6f3ee96c1e34e0e993a962ef2c73
 F src/callback.c d7e46f40c3cf53c43550b7da7a1d0479910b62cc
 F src/complete.c dc1d136c0feee03c2f7550bafc0d29075e36deac
 F src/ctime.c 72a70dcfda75d3a1f81041ce4573e7afddcd8e4e
 F src/date.c 067a81c9942c497aafd2c260e13add8a7d0c7dd4
 F src/delete.c 9b8d308979114991e5dc7cee958316e07186941d
-F src/expr.c 3b25a95f3d309403940ba4a3212f197b8b6251d5
+F src/expr.c 0e41d66d868b37dbc0e041c342e0036fad27e705
 F src/fault.c 160a0c015b6c2629d3899ed2daf63d75754a32bb
-F src/fkey.c c82a04e7a92bb728f9ab972b76590403283be2af
+F src/fkey.c dcb7c37a4bf526ded7b24a01a60fe071bcd160a2
 F src/func.c 8147799b048065a1590805be464d05b4913e652c
 F src/global.c e59ecd2c553ad0d4bfbc84ca71231336f8993a7a
 F src/hash.c ac3470bbf1ca4ae4e306a8ecb0fdf1731810ffe4
 F src/hash.h 2894c932d84d9f892d4b4023a75e501f83050970
 F src/hwtime.h d32741c8f4df852c7d959236615444e2b1063b08
-F src/insert.c d7ed04b1fab887d6d26cf5e36753f5d6d1246290
+F src/insert.c dc197aa9293a26d300eb5378880e701f7b20fefa
 F src/journal.c eb7b9f5e783266521bcd9b2b93d419a219411f71
 F src/legacy.c a199d7683d60cef73089e892409113e69c23a99f
 F src/lempar.c cdf0a000315332fc9b50b62f3b5e22e080a0952b
@@ -164,7 +164,7 @@ F src/os_unix.c ad459bb62eb6f3f6aae26d97b1a28fbac7bf0260
 F src/os_win.c ce1f5db8a7bb4d6f2092b1b2cb9631bec54a6320
 F src/pager.c 4092c907222cfd451c74fe6bd2fd64b342f7190f
 F src/pager.h 1109a06578ec5574dc2c74cf8d9f69daf36fe3e0
-F src/parse.y f29df90bd3adc64b33114ab1de9fb7768fcf2099
+F src/parse.y c2b4a6454ad77299b1443e2c483a560a9f16e724
 F src/pcache.c f8043b433a57aba85384a531e3937a804432a346
 F src/pcache.h 1b5dcc3dc8103d03e625b177023ee67764fa6b7c
 F src/pcache1.c 9fd22671c270b35131ef480bbc00392b8b5f8ab9
@@ -172,14 +172,14 @@ F src/pragma.c 015723c48072781d2740e310ab04dc92956b76d1
 F src/prepare.c 931ad0d852a0df48f79adcba6ce79ca5f475625c
 F src/printf.c 4a9f882f1c1787a8b494a2987765acf9d97ac21f
 F src/random.c cd4a67b3953b88019f8cd4ccd81394a8ddfaba50
-F src/resolve.c 7b986a715ac281643309c29257bb58cfae7aa810
+F src/resolve.c cdd546d62da7763119ea1fa455a898959e03457f
 F src/rowset.c 64655f1a627c9c212d9ab497899e7424a34222e0
-F src/select.c 1462a391952c7659168460fe2d7eed7d41529be1
+F src/select.c e6daa524bbdfa98f4abdb8cb281498f0047d3161
 F src/shell.c e392dd1ccbb77cc1d75a8367a89b473c24bea019
 F src/sqlite.h.in e5552743b5b74dc675ebbdedd849ba8283424fa9
 F src/sqlite3.rc fea433eb0a59f4c9393c8e6d76a6e2596b1fe0c0
 F src/sqlite3ext.h 6904f4aadf976f95241311fbffb00823075d9477
-F src/sqliteInt.h 666935d603a7354d90e7ddfe040e3afdd1d8ee16
+F src/sqliteInt.h d1f0866c69d94fe018a32f78c31a043e3fc0d0de
 F src/sqliteLimit.h 164b0e6749d31e0daa1a4589a169d31c0dec7b3d
 F src/status.c bedc37ec1a6bb9399944024d63f4c769971955a9
 F src/table.c 2cd62736f845d82200acfa1287e33feb3c15d62e
@@ -237,11 +237,11 @@ F src/update.c 28d2d098b43a2c70dae399896ea8a02f622410ef
 F src/utf.c 8d819e2e5104a430fc2005f018db14347c95a38f
 F src/util.c 0af2e515dc0dabacec931bca39525f6c3f1c5455
 F src/vacuum.c 2727bdd08847fcb6b2d2da6d14f018910e8645d3
-F src/vdbe.c 5b49cb1f852e4604a4fb6c47e087ddab28af193f
+F src/vdbe.c 89138e8e8b5468f349ace581db9c49801f4d6543
 F src/vdbe.h b52887278cb173e66188da84dfab216bea61119d
 F src/vdbeInt.h 79abf9b31be406d35ca77d6999cb2d42aaf91e78
 F src/vdbeapi.c 4c2418161cf45392ba76a7ca92f9a5f06b96f89c
-F src/vdbeaux.c f68a124380954985d797fa7d91fe483e2e894aca
+F src/vdbeaux.c 7658c5d9db838db5780f6a1c4d1280d0646e3569
 F src/vdbeblob.c 32f2a4899d67f69634ea4dd93e3f651936d732cb
 F src/vdbemem.c cb55e84b8e2c15704968ee05f0fae25883299b74
 F src/vdbesort.c c61ca318681c0e7267da8be3abfca8469652a7e9
@@ -250,7 +250,7 @@ F src/vtab.c b05e5f1f4902461ba9f5fc49bb7eb7c3a0741a83
 F src/wal.c f5c7b5027d0ed0e9bc9afeb4a3a8dfea762ec7d2
 F src/wal.h 29c197540b19044e6cd73487017e5e47a1d3dac6
 F src/walker.c 3d75ba73de15e0f8cd0737643badbeb0e002f07b
-F src/where.c b5325975cbdd9b64f680fb88d7e727fc9368db68
+F src/where.c b613cf5be71a34c85e3a85782d9ea44bba8ea373
 F test/8_3_names.test 631ea964a3edb091cf73c3b540f6bcfdb36ce823
 F test/aggerror.test a867e273ef9e3d7919f03ef4f0e8c0d2767944f2
 F test/aggnested.test 45c0201e28045ad38a530b5a144b73cd4aa2cfd6
@@ -323,7 +323,7 @@ F test/capi3e.test f7408dda65c92b9056199fdc180f893015f83dde
 F test/cast.test 4c275cbdc8202d6f9c54a3596701719868ac7dc3
 F test/check.test 193f47ed43a8d29aca12b30cd30ceb105fbe710d
 F test/coalesce.test cee0dccb9fbd2d494b77234bccf9dc6c6786eb91
-F test/collate1.test e3eaa48c21e150814be1a7b852d2a8af24458d04
+F test/collate1.test fd02c4d8afc71879c4bb952586389961a21fb0ce
 F test/collate2.test 04cebe4a033be319d6ddbb3bbc69464e01700b49
 F test/collate3.test d28d2cfab2c3a3d4628ae4b2b7afc9965daa3b4c
 F test/collate4.test d37682293d3c32223dec2e6afdeaf9de18415248
@@ -376,7 +376,7 @@ F test/descidx1.test 533dcbda614b0463b0ea029527fd27e5a9ab2d66
 F test/descidx2.test 9f1a0c83fd57f8667c82310ca21b30a350888b5d
 F test/descidx3.test fe720e8b37d59f4cef808b0bf4e1b391c2e56b6f
 F test/diskfull.test 106391384780753ea6896b7b4f005d10e9866b6e
-F test/distinct.test c239558222e5ae357aade535bfe61aaabcb00bbf
+F test/distinct.test 84da1414b2e6887fffd5ed571311b344c5b082ce
 F test/distinctagg.test 1a6ef9c87a58669438fc771450d7a72577417376
 F test/e_createtable.test 0a2465736199cb5e084645a8714ee04299b81721
 F test/e_delete.test 89aa84d3d1bd284a0689ede04bce10226a5aeaa5
@@ -547,7 +547,7 @@ F test/insert4.test 87f6798f31d60c4e177622fcc3663367e6ecbd90
 F test/insert5.test 394f96728d1258f406fe5f5aeb0aaf29487c39a6
 F test/instr.test a34e1d46a9eefb098a7167ef0e730a4a3d82fba0
 F test/intarray.test 066b7d7ac38d25bf96f87f1b017bfc687551cdd4
-F test/interrupt.test 42e7cf98646fd9cb4a3b131a93ed3c50b9e149f1
+F test/interrupt.test dfe9a67a94b0b2d8f70545ba1a6cca10780d71cc
 F test/intpkey.test 7af30f6ae852d8d1c2b70e4bf1551946742e92d8
 F test/io.test 36d251507d72e92b965fb2f0801c2f0b56335bcf
 F test/ioerr.test 40bb2cfcab63fb6aa7424cd97812a84bc16b5fb8
@@ -705,7 +705,7 @@ F test/shared4.test 72d90821e8d2fc918a08f16d32880868d8ee8e9d
 F test/shared6.test 866bb4982c45ce216c61ded5e8fde4e7e2f3ffa9
 F test/shared7.test 960760bc8d03e1419e70dea69cf41db62853616e
 F test/shared8.test b27befbefbe7f4517f1d6b7ff8f64a41ec74165d
-F test/shared9.test 614a3ca431adc73c857632deb4eff75bcaee40ec
+F test/shared9.test 5f2a8f79b4d6c7d107a01ffa1ed05ae7e6333e21
 F test/shared_err.test 0079c05c97d88cfa03989b7c20a8b266983087aa
 F test/sharedlock.test ffa0a3c4ac192145b310f1254f8afca4d553eabf
 F test/shell1.test b7896eb84028f3bc8300caf1fc796a73728aad0b
@@ -830,7 +830,7 @@ F test/tkt2686.test 6ee01c9b9e9c48f6d3a1fdd553b1cc4258f903d6
 F test/tkt2767.test 569000d842678f9cf2db7e0d1b27cbc9011381b0
 F test/tkt2817.test f31839e01f4243cff7399ef654d3af3558cb8d8d
 F test/tkt2820.test 39940276b3436d125deb7d8ebeee053e4cf13213
-F test/tkt2822.test a2b27a58df62d1b2e712f91dbe42ad3b7e0e77cc
+F test/tkt2822.test f391776423a7c0d0949edfce375708bfb0f3141e
 F test/tkt2832.test a9b0b74a02dca166a04d9e37739c414b10929caa
 F test/tkt2854.test e432965db29e27e16f539b2ba7f502eb2ccc49af
 F test/tkt2920.test a8737380e4ae6424e00c0273dc12775704efbebf
@@ -1025,7 +1025,7 @@ F tool/vdbe-compress.tcl f12c884766bd14277f4fcedcae07078011717381
 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4
 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381
 F tool/win/sqlite.vsix 97894c2790eda7b5bce3cc79cb2a8ec2fde9b3ac
-P df1827b19e47724b62fdf6fbedfd4e61add343d3
+P 2d5f37c99a9e5377409697f5392a1ca55970964e cdbfa664839a409589ec7cebfc9111235d2f3b38
-R d9a49317c83cd75d961bd50ea4d2b4bf
+R fd5c21e14054b78f0a72082ceca653a9
-U dan
+U drh
-Z 4659749d4c8a1f0da88866b82eed162a
+Z 9b6ace21d2c8985753646add41f3f044

manifest.uuid

@@ -1 +1 @@
-2d5f37c99a9e5377409697f5392a1ca55970964e
+8542e6180d4321d45b34f33e481658908ce1430d

src/build.c

@@ -2691,10 +2691,8 @@ Index *sqlite3CreateIndex(
   for(i=0; i<pList->nExpr; i++){
     Expr *pExpr = pList->a[i].pExpr;
     if( pExpr ){
-      CollSeq *pColl = pExpr->pColl;
+      CollSeq *pColl = sqlite3ExprCollSeq(pParse, pExpr);
-      /* Either pColl!=0 or there was an OOM failure.  But if an OOM
+      if( pColl ){
-      ** failure we have quit before reaching this point. */
-      if( ALWAYS(pColl) ){
         nExtra += (1 + sqlite3Strlen30(pColl->zName));
       }
     }
@@ -2757,6 +2755,7 @@ Index *sqlite3CreateIndex(
     const char *zColName = pListItem->zName;
     Column *pTabCol;
     int requestedSortOrder;
+    CollSeq *pColl;                /* Collating sequence */
     char *zColl;                   /* Collation sequence name */
 
     for(j=0, pTabCol=pTab->aCol; j<pTab->nCol; j++, pTabCol++){
@@ -2769,14 +2768,11 @@ Index *sqlite3CreateIndex(
       goto exit_create_index;
     }
     pIndex->aiColumn[i] = j;
-    /* Justification of the ALWAYS(pListItem->pExpr->pColl):  Because of
+    if( pListItem->pExpr
-    ** the way the "idxlist" non-terminal is constructed by the parser,
+     && (pColl = sqlite3ExprCollSeq(pParse, pListItem->pExpr))!=0
-    ** if pListItem->pExpr is not null then either pListItem->pExpr->pColl
+    ){
-    ** must exist or else there must have been an OOM error.  But if there
-    ** was an OOM error, we would never reach this point. */
-    if( pListItem->pExpr && ALWAYS(pListItem->pExpr->pColl) ){
       int nColl;
-      zColl = pListItem->pExpr->pColl->zName;
+      zColl = pColl->zName;
       nColl = sqlite3Strlen30(zColl) + 1;
       assert( nExtra>=nColl );
       memcpy(zExtra, zColl, nColl);

src/expr.c

@@ -31,7 +31,9 @@
 ** SELECT * FROM t1 WHERE (select a from t1);
 */
 char sqlite3ExprAffinity(Expr *pExpr){
-  int op = pExpr->op;
+  int op;
+  pExpr = sqlite3ExprSkipCollate(pExpr);
+  op = pExpr->op;
   if( op==TK_SELECT ){
     assert( pExpr->flags&EP_xIsSelect );
     return sqlite3ExprAffinity(pExpr->x.pSelect->pEList->a[0].pExpr);
@@ -56,66 +58,94 @@ char sqlite3ExprAffinity(Expr *pExpr){
 }
 
 /*
-** Set the explicit collating sequence for an expression to the
+** Set the collating sequence for expression pExpr to be the collating
-** collating sequence supplied in the second argument.
+** sequence named by pToken.   Return a pointer to a new Expr node that
+** implements the COLLATE operator.
+**
+** If a memory allocation error occurs, that fact is recorded in pParse->db
+** and the pExpr parameter is returned unchanged.
 */
-Expr *sqlite3ExprSetColl(Expr *pExpr, CollSeq *pColl){
+Expr *sqlite3ExprAddCollateToken(Parse *pParse, Expr *pExpr, Token *pCollName){
-  if( pExpr && pColl ){
+  if( pCollName->n>0 ){
-    pExpr->pColl = pColl;
+    Expr *pNew = sqlite3ExprAlloc(pParse->db, TK_COLLATE, pCollName, 1);
-    pExpr->flags |= EP_ExpCollate;
+    if( pNew ){
+      pNew->pLeft = pExpr;
+      pNew->flags |= EP_Collate;
+      pExpr = pNew;
+    }
+  }
+  return pExpr;
+}
+Expr *sqlite3ExprAddCollateString(Parse *pParse, Expr *pExpr, const char *zC){
+  Token s;
+  assert( zC!=0 );
+  s.z = zC;
+  s.n = sqlite3Strlen30(s.z);
+  return sqlite3ExprAddCollateToken(pParse, pExpr, &s);
+}
+
+/*
+** Skip over any TK_COLLATE and/or TK_AS operators at the root of
+** an expression.
+*/
+Expr *sqlite3ExprSkipCollate(Expr *pExpr){
+  while( pExpr && (pExpr->op==TK_COLLATE || pExpr->op==TK_AS) ){
+    pExpr = pExpr->pLeft;
   }
   return pExpr;
 }
 
 /*
-** Set the collating sequence for expression pExpr to be the collating
+** Return the collation sequence for the expression pExpr. If
-** sequence named by pToken.   Return a pointer to the revised expression.
+** there is no defined collating sequence, return NULL.
-** The collating sequence is marked as "explicit" using the EP_ExpCollate
+**
-** flag.  An explicit collating sequence will override implicit
+** The collating sequence might be determined by a COLLATE operator
-** collating sequences.
+** or by the presence of a column with a defined collating sequence.
-*/
+** COLLATE operators take first precedence.  Left operands take
-Expr *sqlite3ExprSetCollByToken(Parse *pParse, Expr *pExpr, Token *pCollName){
+** precedence over right operands.
-  char *zColl = 0;            /* Dequoted name of collation sequence */
-  CollSeq *pColl;
-  sqlite3 *db = pParse->db;
-  zColl = sqlite3NameFromToken(db, pCollName);
-  pColl = sqlite3LocateCollSeq(pParse, zColl);
-  sqlite3ExprSetColl(pExpr, pColl);
-  sqlite3DbFree(db, zColl);
-  return pExpr;
-}
-
-/*
-** Return the default collation sequence for the expression pExpr. If
-** there is no default collation type, return 0.
 */
 CollSeq *sqlite3ExprCollSeq(Parse *pParse, Expr *pExpr){
+  sqlite3 *db = pParse->db;
   CollSeq *pColl = 0;
   Expr *p = pExpr;
   while( p ){
-    int op;
+    int op = p->op;
-    pColl = p->pColl;
+    if( op==TK_CAST || op==TK_UPLUS ){
-    if( pColl ) break;
+      p = p->pLeft;
-    op = p->op;
+      continue;
-    if( p->pTab!=0 && (
+    }
-        op==TK_AGG_COLUMN || op==TK_COLUMN || op==TK_REGISTER || op==TK_TRIGGER
+    assert( op!=TK_REGISTER || p->op2!=TK_COLLATE );
-    )){
+    if( op==TK_COLLATE ){
-      /* op==TK_REGISTER && p->pTab!=0 happens when pExpr was originally
+      if( db->init.busy ){
-      ** a TK_COLUMN but was previously evaluated and cached in a register */
+        /* Do not report errors when parsing while the schema */
-      const char *zColl;
+        pColl = sqlite3FindCollSeq(db, ENC(db), p->u.zToken, 0);
-      int j = p->iColumn;
+      }else{
-      if( j>=0 ){
+        pColl = sqlite3GetCollSeq(pParse, ENC(db), 0, p->u.zToken);
-        sqlite3 *db = pParse->db;
-        zColl = p->pTab->aCol[j].zColl;
-        pColl = sqlite3FindCollSeq(db, ENC(db), zColl, 0);
-        pExpr->pColl = pColl;
       }
       break;
     }
-    if( op!=TK_CAST && op!=TK_UPLUS ){
+    if( p->pTab!=0
+     && (op==TK_AGG_COLUMN || op==TK_COLUMN
+          || op==TK_REGISTER || op==TK_TRIGGER)
+    ){
+      /* op==TK_REGISTER && p->pTab!=0 happens when pExpr was originally
+      ** a TK_COLUMN but was previously evaluated and cached in a register */
+      int j = p->iColumn;
+      if( j>=0 ){
+        const char *zColl = p->pTab->aCol[j].zColl;
+        pColl = sqlite3FindCollSeq(db, ENC(db), zColl, 0);
+      }
+      break;
+    }
+    if( p->flags & EP_Collate ){
+      if( ALWAYS(p->pLeft) && (p->pLeft->flags & EP_Collate)!=0 ){
+        p = p->pLeft;
+      }else{
+        p = p->pRight;
+      }
+    }else{
       break;
     }
-    p = p->pLeft;
   }
   if( sqlite3CheckCollSeq(pParse, pColl) ){ 
     pColl = 0;
@@ -219,12 +249,10 @@ CollSeq *sqlite3BinaryCompareCollSeq(
 ){
   CollSeq *pColl;
   assert( pLeft );
-  if( pLeft->flags & EP_ExpCollate ){
+  if( pLeft->flags & EP_Collate ){
-    assert( pLeft->pColl );
+    pColl = sqlite3ExprCollSeq(pParse, pLeft);
-    pColl = pLeft->pColl;
+  }else if( pRight && (pRight->flags & EP_Collate)!=0 ){
-  }else if( pRight && pRight->flags & EP_ExpCollate ){
+    pColl = sqlite3ExprCollSeq(pParse, pRight);
-    assert( pRight->pColl );
-    pColl = pRight->pColl;
   }else{
     pColl = sqlite3ExprCollSeq(pParse, pLeft);
     if( !pColl ){
@@ -454,17 +482,11 @@ void sqlite3ExprAttachSubtrees(
   }else{
     if( pRight ){
       pRoot->pRight = pRight;
-      if( pRight->flags & EP_ExpCollate ){
+      pRoot->flags |= EP_Collate & pRight->flags;
-        pRoot->flags |= EP_ExpCollate;
-        pRoot->pColl = pRight->pColl;
-      }
     }
     if( pLeft ){
       pRoot->pLeft = pLeft;
-      if( pLeft->flags & EP_ExpCollate ){
+      pRoot->flags |= EP_Collate & pLeft->flags;
-        pRoot->flags |= EP_ExpCollate;
-        pRoot->pColl = pLeft->pColl;
-      }
     }
     exprSetHeight(pRoot);
   }
@@ -722,7 +744,7 @@ static int dupedExprStructSize(Expr *p, int flags){
     assert( !ExprHasProperty(p, EP_FromJoin) ); 
     assert( (p->flags2 & EP2_MallocedToken)==0 );
     assert( (p->flags2 & EP2_Irreducible)==0 );
-    if( p->pLeft || p->pRight || p->pColl || p->x.pList ){
+    if( p->pLeft || p->pRight || p->x.pList ){
       nSize = EXPR_REDUCEDSIZE | EP_Reduced;
     }else{
       nSize = EXPR_TOKENONLYSIZE | EP_TokenOnly;
@@ -2746,6 +2768,7 @@ int sqlite3ExprCodeTarget(Parse *pParse, Expr *pExpr, int target){
       sqlite3ReleaseTempReg(pParse, r4);
       break;
     }
+    case TK_COLLATE: 
     case TK_UPLUS: {
       inReg = sqlite3ExprCodeTarget(pParse, pExpr->pLeft, target);
       break;
@@ -3115,6 +3138,12 @@ void sqlite3ExplainExpr(Vdbe *pOut, Expr *pExpr){
     case TK_ISNULL:  zUniOp = "ISNULL"; break;
     case TK_NOTNULL: zUniOp = "NOTNULL"; break;
 
+    case TK_COLLATE: {
+      sqlite3ExplainExpr(pOut, pExpr->pLeft);
+      sqlite3ExplainPrintf(pOut,".COLLATE(%s)",pExpr->u.zToken);
+      break;
+    }
+
     case TK_AGG_FUNCTION:
     case TK_CONST_FUNC:
     case TK_FUNCTION: {
@@ -3333,6 +3362,9 @@ static int evalConstExpr(Walker *pWalker, Expr *pExpr){
     case TK_REGISTER: {
       return WRC_Prune;
     }
+    case TK_COLLATE: {
+      return WRC_Continue;
+    }
     case TK_FUNCTION:
     case TK_AGG_FUNCTION:
     case TK_CONST_FUNC: {
@@ -3354,9 +3386,11 @@ static int evalConstExpr(Walker *pWalker, Expr *pExpr){
   }
   if( isAppropriateForFactoring(pExpr) ){
     int r1 = ++pParse->nMem;
-    int r2;
+    int r2 = sqlite3ExprCodeTarget(pParse, pExpr, r1);
-    r2 = sqlite3ExprCodeTarget(pParse, pExpr, r1);
+    /* If r2!=r1, it means that register r1 is never used.  That is harmless
-    if( NEVER(r1!=r2) ) sqlite3ReleaseTempReg(pParse, r1);
+    ** but suboptimal, so we want to know about the situation to fix it.
+    ** Hence the following assert: */
+    assert( r2==r1 );
     pExpr->op2 = pExpr->op;
     pExpr->op = TK_REGISTER;
     pExpr->iTable = r2;
@@ -3773,7 +3807,15 @@ int sqlite3ExprCompare(Expr *pA, Expr *pB){
     return 2;
   }
   if( (pA->flags & EP_Distinct)!=(pB->flags & EP_Distinct) ) return 2;
-  if( pA->op!=pB->op ) return 2;
+  if( pA->op!=pB->op ){
+    if( pA->op==TK_COLLATE && sqlite3ExprCompare(pA->pLeft, pB)<2 ){
+      return 1;
+    }
+    if( pB->op==TK_COLLATE && sqlite3ExprCompare(pA, pB->pLeft)<2 ){
+      return 1;
+    }
+    return 2;
+  }
   if( sqlite3ExprCompare(pA->pLeft, pB->pLeft) ) return 2;
   if( sqlite3ExprCompare(pA->pRight, pB->pRight) ) return 2;
   if( sqlite3ExprListCompare(pA->x.pList, pB->x.pList) ) return 2;
@@ -3785,11 +3827,9 @@ int sqlite3ExprCompare(Expr *pA, Expr *pB){
   }else if( pA->op!=TK_COLUMN && ALWAYS(pA->op!=TK_AGG_COLUMN) && pA->u.zToken){
     if( ExprHasProperty(pB, EP_IntValue) || NEVER(pB->u.zToken==0) ) return 2;
     if( strcmp(pA->u.zToken,pB->u.zToken)!=0 ){
-      return 2;
+      return pA->op==TK_COLLATE ? 1 : 2;
     }
   }
-  if( (pA->flags & EP_ExpCollate)!=(pB->flags & EP_ExpCollate) ) return 1;
-  if( (pA->flags & EP_ExpCollate)!=0 && pA->pColl!=pB->pColl ) return 2;
   return 0;
 }
 

src/fkey.c

@@ -511,12 +511,15 @@ static void fkScanChildren(
       ** expression to the parent key column defaults.  */
       if( pIdx ){
         Column *pCol;
+        const char *zColl;
         iCol = pIdx->aiColumn[i];
         pCol = &pTab->aCol[iCol];
         if( pTab->iPKey==iCol ) iCol = -1;
         pLeft->iTable = regData+iCol+1;
         pLeft->affinity = pCol->affinity;
-        pLeft->pColl = sqlite3LocateCollSeq(pParse, pCol->zColl);
+        zColl = pCol->zColl;
+        if( zColl==0 ) zColl = db->pDfltColl->zName;
+        pLeft = sqlite3ExprAddCollateString(pParse, pLeft, zColl);
       }else{
         pLeft->iTable = regData;
         pLeft->affinity = SQLITE_AFF_INTEGER;

src/insert.c

@@ -1274,25 +1274,20 @@ void sqlite3GenerateConstraintChecks(
     onError = overrideError!=OE_Default ? overrideError : OE_Abort;
     for(i=0; i<pCheck->nExpr; i++){
       int allOk = sqlite3VdbeMakeLabel(v);
-      Expr *pDup = sqlite3ExprDup(db, pCheck->a[i].pExpr, 0);
+      sqlite3ExprIfTrue(pParse, pCheck->a[i].pExpr, allOk, SQLITE_JUMPIFNULL);
-      if( !db->mallocFailed ){
+      if( onError==OE_Ignore ){
-        assert( pDup!=0 );
+        sqlite3VdbeAddOp2(v, OP_Goto, 0, ignoreDest);
-        sqlite3ExprIfTrue(pParse, pDup, allOk, SQLITE_JUMPIFNULL);
+      }else{
-        if( onError==OE_Ignore ){
+        char *zConsName = pCheck->a[i].zName;
-          sqlite3VdbeAddOp2(v, OP_Goto, 0, ignoreDest);
+        if( onError==OE_Replace ) onError = OE_Abort; /* IMP: R-15569-63625 */
+        if( zConsName ){
+          zConsName = sqlite3MPrintf(db, "constraint %s failed", zConsName);
         }else{
-          char *zConsName = pCheck->a[i].zName;
+          zConsName = 0;
-          if( onError==OE_Replace ) onError = OE_Abort; /* IMP: R-15569-63625 */
-          if( zConsName ){
-            zConsName = sqlite3MPrintf(db, "constraint %s failed", zConsName);
-          }else{
-            zConsName = 0;
-          }
-          sqlite3HaltConstraint(pParse, onError, zConsName, P4_DYNAMIC);
         }
-        sqlite3VdbeResolveLabel(v, allOk);
+        sqlite3HaltConstraint(pParse, onError, zConsName, P4_DYNAMIC);
       }
-      sqlite3ExprDelete(db, pDup);
+      sqlite3VdbeResolveLabel(v, allOk);
     }
   }
 #endif /* !defined(SQLITE_OMIT_CHECK) */

src/parse.y

@@ -815,7 +815,7 @@ expr(A) ::= VARIABLE(X).     {
   spanSet(&A, &X, &X);
 }
 expr(A) ::= expr(E) COLLATE ids(C). {
-  A.pExpr = sqlite3ExprSetCollByToken(pParse, E.pExpr, &C);
+  A.pExpr = sqlite3ExprAddCollateToken(pParse, E.pExpr, &C);
   A.zStart = E.zStart;
   A.zEnd = &C.z[C.n];
 }
@@ -1140,22 +1140,14 @@ uniqueflag(A) ::= .        {A = OE_None;}
 idxlist_opt(A) ::= .                         {A = 0;}
 idxlist_opt(A) ::= LP idxlist(X) RP.         {A = X;}
 idxlist(A) ::= idxlist(X) COMMA nm(Y) collate(C) sortorder(Z).  {
-  Expr *p = 0;
+  Expr *p = sqlite3ExprAddCollateToken(pParse, 0, &C);
-  if( C.n>0 ){
-    p = sqlite3Expr(pParse->db, TK_COLUMN, 0);
-    sqlite3ExprSetCollByToken(pParse, p, &C);
-  }
   A = sqlite3ExprListAppend(pParse,X, p);
   sqlite3ExprListSetName(pParse,A,&Y,1);
   sqlite3ExprListCheckLength(pParse, A, "index");
   if( A ) A->a[A->nExpr-1].sortOrder = (u8)Z;
 }
 idxlist(A) ::= nm(Y) collate(C) sortorder(Z). {
-  Expr *p = 0;
+  Expr *p = sqlite3ExprAddCollateToken(pParse, 0, &C);
-  if( C.n>0 ){
-    p = sqlite3PExpr(pParse, TK_COLUMN, 0, 0, 0);
-    sqlite3ExprSetCollByToken(pParse, p, &C);
-  }
   A = sqlite3ExprListAppend(pParse,0, p);
   sqlite3ExprListSetName(pParse, A, &Y, 1);
   sqlite3ExprListCheckLength(pParse, A, "index");

src/resolve.c

@@ -68,6 +68,15 @@ static void incrAggFunctionDepth(Expr *pExpr, int N){
 ** from the result in the result-set.  We might fix this someday.  Or
 ** then again, we might not...
 **
+** If the reference is followed by a COLLATE operator, then make sure
+** the COLLATE operator is preserved.  For example:
+**
+**     SELECT a+b, c+d FROM t1 ORDER BY 1 COLLATE nocase;
+**
+** Should be transformed into:
+**
+**     SELECT a+b, c+d FROM t1 ORDER BY (a+b) COLLATE nocase;
+**
 ** The nSubquery parameter specifies how many levels of subquery the
 ** alias is removed from the original expression.  The usually value is
 ** zero but it might be more if the alias is contained within a subquery
@@ -91,8 +100,9 @@ static void resolveAlias(
   assert( pOrig!=0 );
   assert( pOrig->flags & EP_Resolved );
   db = pParse->db;
+  pDup = sqlite3ExprDup(db, pOrig, 0);
+  if( pDup==0 ) return;
   if( pOrig->op!=TK_COLUMN && zType[0]!='G' ){
-    pDup = sqlite3ExprDup(db, pOrig, 0);
     incrAggFunctionDepth(pDup, nSubquery);
     pDup = sqlite3PExpr(pParse, TK_AS, pDup, 0, 0);
     if( pDup==0 ) return;
@@ -100,32 +110,26 @@ static void resolveAlias(
       pEList->a[iCol].iAlias = (u16)(++pParse->nAlias);
     }
     pDup->iTable = pEList->a[iCol].iAlias;
-  }else if( ExprHasProperty(pOrig, EP_IntValue) || pOrig->u.zToken==0 ){
-    pDup = sqlite3ExprDup(db, pOrig, 0);
-    if( pDup==0 ) return;
-  }else{
-    char *zToken = pOrig->u.zToken;
-    assert( zToken!=0 );
-    pOrig->u.zToken = 0;
-    pDup = sqlite3ExprDup(db, pOrig, 0);
-    pOrig->u.zToken = zToken;
-    if( pDup==0 ) return;
-    assert( (pDup->flags & (EP_Reduced|EP_TokenOnly))==0 );
-    pDup->flags2 |= EP2_MallocedToken;
-    pDup->u.zToken = sqlite3DbStrDup(db, zToken);
   }
-  if( pExpr->flags & EP_ExpCollate ){
+  if( pExpr->op==TK_COLLATE ){
-    pDup->pColl = pExpr->pColl;
+    pDup = sqlite3ExprAddCollateString(pParse, pDup, pExpr->u.zToken);
-    pDup->flags |= EP_ExpCollate;
   }
 
   /* Before calling sqlite3ExprDelete(), set the EP_Static flag. This 
   ** prevents ExprDelete() from deleting the Expr structure itself,
   ** allowing it to be repopulated by the memcpy() on the following line.
+  ** The pExpr->u.zToken might point into memory that will be freed by the
+  ** sqlite3DbFree(db, pDup) on the last line of this block, so be sure to
+  ** make a copy of the token before doing the sqlite3DbFree().
   */
   ExprSetProperty(pExpr, EP_Static);
   sqlite3ExprDelete(db, pExpr);
   memcpy(pExpr, pDup, sizeof(*pExpr));
+  if( !ExprHasProperty(pExpr, EP_IntValue) && pExpr->u.zToken!=0 ){
+    assert( (pExpr->flags & (EP_Reduced|EP_TokenOnly))==0 );
+    pExpr->u.zToken = sqlite3DbStrDup(db, pExpr->u.zToken);
+    pExpr->flags2 |= EP2_MallocedToken;
+  }
   sqlite3DbFree(db, pDup);
 }
 
@@ -812,7 +816,7 @@ static int resolveCompoundOrderBy(
       int iCol = -1;
       Expr *pE, *pDup;
       if( pItem->done ) continue;
-      pE = pItem->pExpr;
+      pE = sqlite3ExprSkipCollate(pItem->pExpr);
       if( sqlite3ExprIsInteger(pE, &iCol) ){
         if( iCol<=0 || iCol>pEList->nExpr ){
           resolveOutOfRangeError(pParse, "ORDER", i+1, pEList->nExpr);
@@ -830,14 +834,20 @@ static int resolveCompoundOrderBy(
         }
       }
       if( iCol>0 ){
-        CollSeq *pColl = pE->pColl;
+        /* Convert the ORDER BY term into an integer column number iCol,
-        int flags = pE->flags & EP_ExpCollate;
+        ** taking care to preserve the COLLATE clause if it exists */
+        Expr *pNew = sqlite3Expr(db, TK_INTEGER, 0);
+        if( pNew==0 ) return 1;
+        pNew->flags |= EP_IntValue;
+        pNew->u.iValue = iCol;
+        if( pItem->pExpr==pE ){
+          pItem->pExpr = pNew;
+        }else{
+          assert( pItem->pExpr->op==TK_COLLATE );
+          assert( pItem->pExpr->pLeft==pE );
+          pItem->pExpr->pLeft = pNew;
+        }
         sqlite3ExprDelete(db, pE);
-        pItem->pExpr = pE = sqlite3Expr(db, TK_INTEGER, 0);
-        if( pE==0 ) return 1;
-        pE->pColl = pColl;
-        pE->flags |= EP_IntValue | flags;
-        pE->u.iValue = iCol;
         pItem->iOrderByCol = (u16)iCol;
         pItem->done = 1;
       }else{
@@ -942,11 +952,11 @@ static int resolveOrderGroupBy(
       pItem->iOrderByCol = (u16)iCol;
       continue;
     }
-    if( sqlite3ExprIsInteger(pE, &iCol) ){
+    if( sqlite3ExprIsInteger(sqlite3ExprSkipCollate(pE), &iCol) ){
       /* The ORDER BY term is an integer constant.  Again, set the column
       ** number so that sqlite3ResolveOrderGroupBy() will convert the
       ** order-by term to a copy of the result-set expression */
-      if( iCol<1 ){
+      if( iCol<1 || iCol>0xffff ){
         resolveOutOfRangeError(pParse, zType, i+1, nResult);
         return 1;
       }

src/select.c

@@ -1335,7 +1335,7 @@ static int selectColumnsFromExprList(
   for(i=0, pCol=aCol; i<nCol; i++, pCol++){
     /* Get an appropriate name for the column
     */
-    p = pEList->a[i].pExpr;
+    p = sqlite3ExprSkipCollate(pEList->a[i].pExpr);
     assert( p->pRight==0 || ExprHasProperty(p->pRight, EP_IntValue)
                || p->pRight->u.zToken==0 || p->pRight->u.zToken[0]!=0 );
     if( (zName = pEList->a[i].zName)!=0 ){
@@ -2333,12 +2333,13 @@ static int multiSelectOrderBy(
       for(i=0; i<nOrderBy; i++){
         CollSeq *pColl;
         Expr *pTerm = pOrderBy->a[i].pExpr;
-        if( pTerm->flags & EP_ExpCollate ){
+        if( pTerm->flags & EP_Collate ){
-          pColl = pTerm->pColl;
+          pColl = sqlite3ExprCollSeq(pParse, pTerm);
         }else{
           pColl = multiSelectCollSeq(pParse, p, aPermute[i]);
-          pTerm->flags |= EP_ExpCollate;
+          if( pColl==0 ) pColl = db->pDfltColl;
-          pTerm->pColl = pColl;
+          pOrderBy->a[i].pExpr =
+             sqlite3ExprAddCollateString(pParse, pTerm, pColl->zName);
         }
         pKeyMerge->aColl[i] = pColl;
         pKeyMerge->aSortOrder[i] = pOrderBy->a[i].sortOrder;
@@ -2541,6 +2542,7 @@ static int multiSelectOrderBy(
   sqlite3VdbeAddOp4(v, OP_Permutation, 0, 0, 0, (char*)aPermute, P4_INTARRAY);
   sqlite3VdbeAddOp4(v, OP_Compare, destA.iSdst, destB.iSdst, nOrderBy,
                          (char*)pKeyMerge, P4_KEYINFO_HANDOFF);
+  sqlite3VdbeChangeP5(v, OPFLAG_PERMUTE);
   sqlite3VdbeAddOp3(v, OP_Jump, addrAltB, addrAeqB, addrAgtB);
 
   /* Release temporary registers
@@ -2608,9 +2610,6 @@ static Expr *substExpr(
       assert( pEList!=0 && pExpr->iColumn<pEList->nExpr );
       assert( pExpr->pLeft==0 && pExpr->pRight==0 );
       pNew = sqlite3ExprDup(db, pEList->a[pExpr->iColumn].pExpr, 0);
-      if( pNew && pExpr->pColl ){
-        pNew->pColl = pExpr->pColl;
-      }
       sqlite3ExprDelete(db, pExpr);
       pExpr = pNew;
     }

src/sqliteInt.h

@@ -1678,7 +1678,6 @@ struct Expr {
     ExprList *pList;     /* Function arguments or in "<expr> IN (<expr-list)" */
     Select *pSelect;     /* Used for sub-selects and "<expr> IN (<select>)" */
   } x;
-  CollSeq *pColl;        /* The collation type of the column or 0 */
 
   /* If the EP_Reduced flag is set in the Expr.flags mask, then no
   ** space is allocated for the fields below this point. An attempt to
@@ -1714,7 +1713,7 @@ struct Expr {
 #define EP_VarSelect  0x0020  /* pSelect is correlated, not constant */
 #define EP_DblQuoted  0x0040  /* token.z was originally in "..." */
 #define EP_InfixFunc  0x0080  /* True for an infix function: LIKE, GLOB, etc */
-#define EP_ExpCollate 0x0100  /* Collating sequence specified explicitly */
+#define EP_Collate    0x0100  /* Tree contains a TK_COLLATE opeartor */
 #define EP_FixedDest  0x0200  /* Result needed in a specific register */
 #define EP_IntValue   0x0400  /* Integer value contained in u.iValue */
 #define EP_xIsSelect  0x0800  /* x.pSelect is valid (otherwise x.pList is) */
@@ -2331,6 +2330,7 @@ struct AuthContext {
 #define OPFLAG_TYPEOFARG     0x80    /* OP_Column only used for typeof() */
 #define OPFLAG_BULKCSR       0x01    /* OP_Open** used to open bulk cursor */
 #define OPFLAG_P2ISREG       0x02    /* P2 to OP_Open** is a register number */
+#define OPFLAG_PERMUTE       0x01    /* OP_Compare: use the permutation */
 
 /*
  * Each trigger present in the database schema is stored as an instance of
@@ -3022,8 +3022,9 @@ int sqlite3ReadSchema(Parse *pParse);
 CollSeq *sqlite3FindCollSeq(sqlite3*,u8 enc, const char*,int);
 CollSeq *sqlite3LocateCollSeq(Parse *pParse, const char*zName);
 CollSeq *sqlite3ExprCollSeq(Parse *pParse, Expr *pExpr);
-Expr *sqlite3ExprSetColl(Expr*, CollSeq*);
+Expr *sqlite3ExprAddCollateToken(Parse *pParse, Expr*, Token*);
-Expr *sqlite3ExprSetCollByToken(Parse *pParse, Expr*, Token*);
+Expr *sqlite3ExprAddCollateString(Parse*,Expr*,const char*);
+Expr *sqlite3ExprSkipCollate(Expr*);
 int sqlite3CheckCollSeq(Parse *, CollSeq *);
 int sqlite3CheckObjectName(Parse *, const char *);
 void sqlite3VdbeSetChanges(sqlite3 *, int);

src/vdbe.c

@@ -422,7 +422,9 @@ void sqlite3VdbeMemPrettyPrint(Mem *pMem, char *zBuf){
 ** Print the value of a register for tracing purposes:
 */
 static void memTracePrint(FILE *out, Mem *p){
-  if( p->flags & MEM_Null ){
+  if( p->flags & MEM_Invalid ){
+    fprintf(out, " undefined");
+  }else if( p->flags & MEM_Null ){
     fprintf(out, " NULL");
   }else if( (p->flags & (MEM_Int|MEM_Str))==(MEM_Int|MEM_Str) ){
     fprintf(out, " si:%lld", p->u.i);
@@ -1077,6 +1079,9 @@ case OP_Copy: {
   while( 1 ){
     sqlite3VdbeMemShallowCopy(pOut, pIn1, MEM_Ephem);
     Deephemeralize(pOut);
+#ifdef SQLITE_DEBUG
+    pOut->pScopyFrom = 0;
+#endif
     REGISTER_TRACE(pOp->p2+pOp->p3-n, pOut);
     if( (n--)==0 ) break;
     pOut++;
@@ -1887,9 +1892,9 @@ case OP_Ge: {             /* same as TK_GE, jump, in1, in3 */
 ** Set the permutation used by the OP_Compare operator to be the array
 ** of integers in P4.
 **
-** The permutation is only valid until the next OP_Permutation, OP_Compare,
+** The permutation is only valid until the next OP_Compare that has
-** OP_Halt, or OP_ResultRow.  Typically the OP_Permutation should occur
+** the OPFLAG_PERMUTE bit set in P5. Typically the OP_Permutation should 
-** immediately prior to the OP_Compare.
+** occur immediately prior to the OP_Compare.
 */
 case OP_Permutation: {
   assert( pOp->p4type==P4_INTARRAY );
@@ -1898,12 +1903,17 @@ case OP_Permutation: {
   break;
 }
 
-/* Opcode: Compare P1 P2 P3 P4 *
+/* Opcode: Compare P1 P2 P3 P4 P5
 **
 ** Compare two vectors of registers in reg(P1)..reg(P1+P3-1) (call this
 ** vector "A") and in reg(P2)..reg(P2+P3-1) ("B").  Save the result of
 ** the comparison for use by the next OP_Jump instruct.
 **
+** If P5 has the OPFLAG_PERMUTE bit set, then the order of comparison is
+** determined by the most recent OP_Permutation operator.  If the
+** OPFLAG_PERMUTE bit is clear, then register are compared in sequential
+** order.
+**
 ** P4 is a KeyInfo structure that defines collating sequences and sort
 ** orders for the comparison.  The permutation applies to registers
 ** only.  The KeyInfo elements are used sequentially.
@@ -1922,6 +1932,7 @@ case OP_Compare: {
   CollSeq *pColl;    /* Collating sequence to use on this term */
   int bRev;          /* True for DESCENDING sort order */
 
+  if( (pOp->p5 & OPFLAG_PERMUTE)==0 ) aPermute = 0;
   n = pOp->p3;
   pKeyInfo = pOp->p4.pKeyInfo;
   assert( n>0 );

src/vdbeaux.c

@@ -866,22 +866,18 @@ static char *displayP4(Op *pOp, char *zTemp, int nTemp){
       i = sqlite3Strlen30(zTemp);
       for(j=0; j<pKeyInfo->nField; j++){
         CollSeq *pColl = pKeyInfo->aColl[j];
-        if( pColl ){
+        const char *zColl = pColl ? pColl->zName : "nil";
-          int n = sqlite3Strlen30(pColl->zName);
+        int n = sqlite3Strlen30(zColl);
-          if( i+n>nTemp-6 ){
+        if( i+n>nTemp-6 ){
-            memcpy(&zTemp[i],",...",4);
+          memcpy(&zTemp[i],",...",4);
-            break;
+          break;
-          }
-          zTemp[i++] = ',';
-          if( pKeyInfo->aSortOrder[j] ){
-            zTemp[i++] = '-';
-          }
-          memcpy(&zTemp[i], pColl->zName,n+1);
-          i += n;
-        }else if( i+4<nTemp-6 ){
-          memcpy(&zTemp[i],",nil",4);
-          i += 4;
         }
+        zTemp[i++] = ',';
+        if( pKeyInfo->aSortOrder[j] ){
+          zTemp[i++] = '-';
+        }
+        memcpy(&zTemp[i], zColl, n+1);
+        i += n;
       }
       zTemp[i++] = ')';
       zTemp[i] = 0;

src/where.c

@@ -563,23 +563,32 @@ static int allowedOp(int op){
 ** Commute a comparison operator.  Expressions of the form "X op Y"
 ** are converted into "Y op X".
 **
-** If a collation sequence is associated with either the left or right
+** If left/right precendence rules come into play when determining the
+** collating
 ** side of the comparison, it remains associated with the same side after
 ** the commutation. So "Y collate NOCASE op X" becomes 
-** "X collate NOCASE op Y". This is because any collation sequence on
+** "X op Y". This is because any collation sequence on
 ** the left hand side of a comparison overrides any collation sequence 
-** attached to the right. For the same reason the EP_ExpCollate flag
+** attached to the right. For the same reason the EP_Collate flag
 ** is not commuted.
 */
 static void exprCommute(Parse *pParse, Expr *pExpr){
-  u16 expRight = (pExpr->pRight->flags & EP_ExpCollate);
+  u16 expRight = (pExpr->pRight->flags & EP_Collate);
-  u16 expLeft = (pExpr->pLeft->flags & EP_ExpCollate);
+  u16 expLeft = (pExpr->pLeft->flags & EP_Collate);
   assert( allowedOp(pExpr->op) && pExpr->op!=TK_IN );
-  pExpr->pRight->pColl = sqlite3ExprCollSeq(pParse, pExpr->pRight);
+  if( expRight==expLeft ){
-  pExpr->pLeft->pColl = sqlite3ExprCollSeq(pParse, pExpr->pLeft);
+    /* Either X and Y both have COLLATE operator or neither do */
-  SWAP(CollSeq*,pExpr->pRight->pColl,pExpr->pLeft->pColl);
+    if( expRight ){
-  pExpr->pRight->flags = (pExpr->pRight->flags & ~EP_ExpCollate) | expLeft;
+      /* Both X and Y have COLLATE operators.  Make sure X is always
-  pExpr->pLeft->flags = (pExpr->pLeft->flags & ~EP_ExpCollate) | expRight;
+      ** used by clearing the EP_Collate flag from Y. */
+      pExpr->pRight->flags &= ~EP_Collate;
+    }else if( sqlite3ExprCollSeq(pParse, pExpr->pLeft)!=0 ){
+      /* Neither X nor Y have COLLATE operators, but X has a non-default
+      ** collating sequence.  So add the EP_Collate marker on X to cause
+      ** it to be searched first. */
+      pExpr->pLeft->flags |= EP_Collate;
+    }
+  }
   SWAP(Expr*,pExpr->pRight,pExpr->pLeft);
   if( pExpr->op>=TK_GT ){
     assert( TK_LT==TK_GT+2 );
@@ -656,12 +665,12 @@ static WhereTerm *findTerm(
           */
           assert(pX->pLeft);
           pColl = sqlite3BinaryCompareCollSeq(pParse, pX->pLeft, pX->pRight);
-          assert(pColl || pParse->nErr);
+          if( pColl==0 ) pColl = pParse->db->pDfltColl;
   
           for(j=0; pIdx->aiColumn[j]!=iColumn; j++){
             if( NEVER(j>=pIdx->nColumn) ) return 0;
           }
-          if( pColl && sqlite3StrICmp(pColl->zName, pIdx->azColl[j]) ) continue;
+          if( sqlite3StrICmp(pColl->zName, pIdx->azColl[j]) ) continue;
         }
         return pTerm;
       }
@@ -1179,7 +1188,7 @@ static void exprAnalyze(
   }
   pTerm = &pWC->a[idxTerm];
   pMaskSet = pWC->pMaskSet;
-  pExpr = pTerm->pExpr;
+  pExpr = sqlite3ExprSkipCollate(pTerm->pExpr);
   prereqLeft = exprTableUsage(pMaskSet, pExpr->pLeft);
   op = pExpr->op;
   if( op==TK_IN ){
@@ -1206,8 +1215,8 @@ static void exprAnalyze(
   pTerm->iParent = -1;
   pTerm->eOperator = 0;
   if( allowedOp(op) && (pTerm->prereqRight & prereqLeft)==0 ){
-    Expr *pLeft = pExpr->pLeft;
+    Expr *pLeft = sqlite3ExprSkipCollate(pExpr->pLeft);
-    Expr *pRight = pExpr->pRight;
+    Expr *pRight = sqlite3ExprSkipCollate(pExpr->pRight);
     if( pLeft->op==TK_COLUMN ){
       pTerm->leftCursor = pLeft->iTable;
       pTerm->u.leftColumn = pLeft->iColumn;
@@ -1235,7 +1244,7 @@ static void exprAnalyze(
         pNew = pTerm;
       }
       exprCommute(pParse, pDup);
-      pLeft = pDup->pLeft;
+      pLeft = sqlite3ExprSkipCollate(pDup->pLeft);
       pNew->leftCursor = pLeft->iTable;
       pNew->u.leftColumn = pLeft->iColumn;
       testcase( (prereqLeft | extraRight) != prereqLeft );
@@ -1314,7 +1323,7 @@ static void exprAnalyze(
     Expr *pNewExpr2;
     int idxNew1;
     int idxNew2;
-    CollSeq *pColl;    /* Collating sequence to use */
+    Token sCollSeqName;  /* Name of collating sequence */
 
     pLeft = pExpr->x.pList->a[1].pExpr;
     pStr2 = sqlite3ExprDup(db, pStr1, 0);
@@ -1336,16 +1345,19 @@ static void exprAnalyze(
       }
       *pC = c + 1;
     }
-    pColl = sqlite3FindCollSeq(db, SQLITE_UTF8, noCase ? "NOCASE" : "BINARY",0);
+    sCollSeqName.z = noCase ? "NOCASE" : "BINARY";
+    sCollSeqName.n = 6;
+    pNewExpr1 = sqlite3ExprDup(db, pLeft, 0);
     pNewExpr1 = sqlite3PExpr(pParse, TK_GE, 
-                     sqlite3ExprSetColl(sqlite3ExprDup(db,pLeft,0), pColl),
+           sqlite3ExprAddCollateToken(pParse,pNewExpr1,&sCollSeqName),
-                     pStr1, 0);
+           pStr1, 0);
     idxNew1 = whereClauseInsert(pWC, pNewExpr1, TERM_VIRTUAL|TERM_DYNAMIC);
     testcase( idxNew1==0 );
     exprAnalyze(pSrc, pWC, idxNew1);
+    pNewExpr2 = sqlite3ExprDup(db, pLeft, 0);
     pNewExpr2 = sqlite3PExpr(pParse, TK_LT,
-                     sqlite3ExprSetColl(sqlite3ExprDup(db,pLeft,0), pColl),
+           sqlite3ExprAddCollateToken(pParse,pNewExpr2,&sCollSeqName),
-                     pStr2, 0);
+           pStr2, 0);
     idxNew2 = whereClauseInsert(pWC, pNewExpr2, TERM_VIRTUAL|TERM_DYNAMIC);
     testcase( idxNew2==0 );
     exprAnalyze(pSrc, pWC, idxNew2);
@@ -1463,12 +1475,12 @@ static int findIndexCol(
   const char *zColl = pIdx->azColl[iCol];
 
   for(i=0; i<pList->nExpr; i++){
-    Expr *p = pList->a[i].pExpr;
+    Expr *p = sqlite3ExprSkipCollate(pList->a[i].pExpr);
     if( p->op==TK_COLUMN
      && p->iColumn==pIdx->aiColumn[iCol]
      && p->iTable==iBase
     ){
-      CollSeq *pColl = sqlite3ExprCollSeq(pParse, p);
+      CollSeq *pColl = sqlite3ExprCollSeq(pParse, pList->a[i].pExpr);
       if( ALWAYS(pColl) && 0==sqlite3StrICmp(pColl->zName, zColl) ){
         return i;
       }
@@ -1515,7 +1527,7 @@ static int isDistinctIndex(
   */
   for(i=0; i<pDistinct->nExpr; i++){
     WhereTerm *pTerm;
-    Expr *p = pDistinct->a[i].pExpr;
+    Expr *p = sqlite3ExprSkipCollate(pDistinct->a[i].pExpr);
     if( p->op!=TK_COLUMN ) return 0;
     pTerm = findTerm(pWC, p->iTable, p->iColumn, ~(Bitmask)0, WO_EQ, 0);
     if( pTerm ){
@@ -1567,7 +1579,7 @@ static int isDistinctRedundant(
   ** current SELECT is a correlated sub-query.
   */
   for(i=0; i<pDistinct->nExpr; i++){
-    Expr *p = pDistinct->a[i].pExpr;
+    Expr *p = sqlite3ExprSkipCollate(pDistinct->a[i].pExpr);
     if( p->op==TK_COLUMN && p->iTable==iBase && p->iColumn<0 ) return 1;
   }
 
@@ -2853,7 +2865,7 @@ static int isSortingIndex(
     /* If the next term of the ORDER BY clause refers to anything other than
     ** a column in the "base" table, then this index will not be of any
     ** further use in handling the ORDER BY. */
-    pOBExpr = pOBItem->pExpr;
+    pOBExpr = sqlite3ExprSkipCollate(pOBItem->pExpr);
     if( pOBExpr->op!=TK_COLUMN || pOBExpr->iTable!=base ){
       break;
     }
@@ -2879,7 +2891,7 @@ static int isSortingIndex(
     ** clause entry.  Set isMatch to 1 if they both match. */
     if( pOBExpr->iColumn==iColumn ){
       if( zColl ){
-        pColl = sqlite3ExprCollSeq(pParse, pOBExpr);
+        pColl = sqlite3ExprCollSeq(pParse, pOBItem->pExpr);
         if( !pColl ) pColl = db->pDfltColl;
         isMatch = sqlite3StrICmp(pColl->zName, zColl)==0;
       }else{

test/collate1.test

@@ -75,6 +75,7 @@ do_test collate1-1.1 {
   }
 } {{} 0x119 0x2D}
 do_test collate1-1.2 {
+breakpoint
   execsql {
     SELECT c2 FROM collate1t1 ORDER BY 1 COLLATE hex;
   }

test/distinct.test

@@ -168,7 +168,7 @@ foreach {tn sql temptables res} {
   6   "b FROM t1"                                          {hash}  {b B}
   7   "a FROM t1"                                          {}      {A a}
   8   "b COLLATE nocase FROM t1"                           {}      {b}
-  9   "b COLLATE nocase FROM t1 ORDER BY b COLLATE nocase" {}      {B}
+  9   "b COLLATE nocase FROM t1 ORDER BY b COLLATE nocase" {}      {b}
 } {
   do_execsql_test    2.$tn.1 "SELECT DISTINCT $sql" $res
   do_temptables_test 2.$tn.2 "SELECT DISTINCT $sql" $temptables

test/interrupt.test

@@ -166,6 +166,8 @@ for {set i 1} {$i<$max_count-5} {incr i 1} {
   } {1 interrupted}
 }
 
+if {0} {  # This doesn't work anymore since the collation factor is
+          # no longer called during schema parsing.
 # Interrupt during parsing
 #
 do_test interrupt-5.1 {
@@ -179,5 +181,5 @@ do_test interrupt-5.1 {
     CREATE INDEX fake ON fake1(a COLLATE fake_collation, b, c DESC);
   }
 } {1 interrupt}
-
+}
 finish_test

test/shared9.test

@@ -136,6 +136,30 @@ do_test 2.4 {
   set ::invoked_mycollate_db1
 } {0}
 
+forcedelete test.db test.db2
+sqlite3 db1 test.db
+sqlite3 db2 test.db
+db1 collate mycollate mycollate_db1
+db2 collate mycollate mycollate_db2
+
+do_test 2.13 {
+  set ::invoked_mycollate_db1 0
+  db1 eval {
+    CREATE TABLE t1(a, CHECK (a COLLATE mycollate IN ('one', 'two', 'three')));
+    INSERT INTO t1 VALUES('one');
+  }
+  db1 close
+  set ::invoked_mycollate_db1
+} {1}
+do_test 2.14 {
+  set ::invoked_mycollate_db1 0
+  db2 eval {
+    INSERT INTO t1 VALUES('two');
+  }
+  db2 close
+  set ::invoked_mycollate_db1
+} {0}
+
 #-------------------------------------------------------------------------
 # This test verifies that a bug causing a busy-handler belonging to one
 # shared-cache connection to be executed as a result of an sqlite3_step()
@@ -204,4 +228,3 @@ db2 close
 
 sqlite3_enable_shared_cache $::enable_shared_cache
 finish_test
-

test/tkt2822.test

@@ -273,11 +273,21 @@ do_test tkt2822-7.1 {
     SELECT * FROM t7 ORDER BY 0;
   }
 } {1 {1st ORDER BY term out of range - should be between 1 and 25}}
-do_test tkt2822-7.2 {
+do_test tkt2822-7.2.1 {
   catchsql {
     SELECT * FROM t7 ORDER BY 1, 0;
   }
 } {1 {2nd ORDER BY term out of range - should be between 1 and 25}}
+do_test tkt2822-7.2.2 {
+  catchsql {
+    SELECT * FROM t7 ORDER BY 1, 26;
+  }
+} {1 {2nd ORDER BY term out of range - should be between 1 and 25}}
+do_test tkt2822-7.2.3 {
+  catchsql {
+    SELECT * FROM t7 ORDER BY 1, 65536;
+  }
+} {1 {2nd ORDER BY term out of range - should be between 1 and 25}}
 do_test tkt2822-7.3 {
   catchsql {
     SELECT * FROM t7 ORDER BY 1, 2, 0;