Fix the sqlite3_bind_value() interface so that it does not remove a

byte-order mark on the value being bound.

FossilOrigin-Name: 56e268312d038608b5fb2a8ed24002f478ad5a22eb0214b1a064f2926aada66c

manifest

@@ -1,5 +1,5 @@
-C Follow\sdocumented\sbehavior\swhen\scasting\sa\szeroblob()\sto\stext\son\sa\sUTF-16\ndatabase.\s[bugs:/info/2026-10-23T12:45:59Z|bug\s2026-10-23T12:45:59Z]
-D 2026-10-23T13:06:30.822
+C Fix\sthe\ssqlite3_bind_value()\sinterface\sso\sthat\sit\sdoes\snot\sremove\sa\nbyte-order\smark\son\sthe\svalue\sbeing\sbound.
+D 2026-10-23T15:30:27.214
 F .fossil-settings/binary-glob 61195414528fb3ea9693577e1980230d78a1f8b0a54c78cf1b9b24d0a409ed6a x
 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1
 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea
@@ -809,10 +809,10 @@ F src/resolve.c 658d5d8f67b7565cbed56394d0fdbb124561a92c7455253186189db5e090e661
 F src/rowset.c 8432130e6c344b3401a8874c3cb49fefe6873fec593294de077afea2dce5ec97
 F src/select.c 114c5874d9c61bb4175831067b6f7c2542da72c9e5fba10ff944c9ebded8bcea
 F src/shell.c.in 808c0bd4a593a3552777c059e90c9c47e459f507fa25d2afd298edf09d3140f5
-F src/sqlite.h.in 59fcb8ca48dd0477770c32b653ac67f910a4eefadfe0c98868b7a9cbcff95019
+F src/sqlite.h.in 5588081b98e60a771c6ee3512a9f5837ee3268130e524d5dc956601eafc737ae
 F src/sqlite3.rc 015537e6ac1eec6c7050e17b616c2ffe6f70fca241835a84a4f0d5937383c479
 F src/sqlite3ext.h 5e113c4c0442f2f1b3345d44cbe4b82fad45f2fc0c35f0ed4eef27fb8f1cb394
-F src/sqliteInt.h 97293a49107ea79c7ebe72f9a5ffc262a3ecb8bf59ba0002c1f4753669bd2e3d
+F src/sqliteInt.h 5a4faf7d7b6103df8797240cad67f9f63b77c8043487f23c6eac7dd62a0667a0
 F src/sqliteLimit.h a905209cea2a167391382f64ecd42f944863ae44eabd540ef24a70361182f5ae
 F src/status.c 7565d63a79aa2f326339a24a0461a60096d0bd2bce711fefb50b5c89335f3592
 F src/table.c 0f141b58a16de7e2fbe81c308379e7279f4c6b50eb08efeec5892794a0ba30d1
@@ -874,13 +874,13 @@ F src/upsert.c dd9f0fcccbfb4f20e1026a21a7254ba3f2c08e9cfa92affaff5b5ec3b00ea549
 F src/utf.c c5bec56770b71fe75e12f1f7b0a512032e1fd3d1f83c50c4ab4474ced20347a4
 F src/util.c 5cda82abda6cc880f67749af782ebd79ac1fad75edf9ab8cdb568482a71e9cd4
 F src/vacuum.c d3d35d8ae893d419ade5fa196d761a83bddcbb62137a1a157ae751ef38b26e82
-F src/vdbe.c 7bb9e161f05986446186513a5c9498455930069c2177b12e074a46fa006890c5
+F src/vdbe.c 3ec0ab7a9eff7e7ad32e43a5178c35030e1cb505e4866b707897e87fad9bb8ca
 F src/vdbe.h b6ffb28da04363fd5d439b8204118fd3f2e751abe4f8e3edb089155fb90d4607
 F src/vdbeInt.h 581b112c6a763a5871ae1d2b948c5288e749d03c7e3829ebce2738402fe21074
-F src/vdbeapi.c 4b38eb061fe6a8e96055e74d9301f0f36e453855e650a3e5fdd25ad45c7d0d7e
+F src/vdbeapi.c b1501e7184c5286cf2507e80bd437e8ea749997376bacaf199160c8563ef1d16
 F src/vdbeaux.c 7e3975222531284ff14cb1be93b5751f6a3392a56d86d6329fcaa120471f56f3
 F src/vdbeblob.c b3f0640db9642fbdc88bd6ebcc83d6009514cafc98f062f675f2c8d505d82692
-F src/vdbemem.c cf4000a8b355be2028b290516898ff99d611c0d33397ac6c4f6d77f84dcd39fb
+F src/vdbemem.c 29f9bf1f900c85415182e7f97cfcee296742297b3f93113927f3767eeedd8df2
 F src/vdbesort.c a72ad2a629df2901fde4aa588e38590d0aa27b7d5bc3b2a416e5f295de839cc1
 F src/vdbetrace.c 49e689f751505839742f4a243a1a566e57d5c9eaf0d33bbaa26e2de3febf7b41
 F src/vdbevtab.c fc46b9cbd759dc013f0b3724549cc0d71379183c667df3a5988f7e2f1bd485f3
@@ -2297,8 +2297,11 @@ F tool/warnings-clang.sh bbf6a1e685e534c92ec2bfba5b1745f34fb6f0bc2a362850723a9ee
 F tool/warnings.sh a554d13f6e5cf3760f041b87939e3d616ec6961859c3245e8ef701d1eafc2ca2
 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f
 F tool/winmain.c 00c8fb88e365c9017db14c73d3c78af62194d9644feaf60e220ab0f411f3604c
-P 320cb1f51382868ed576d8fa52c75672a2c3e4cee99f868385645bd9a370457a
-R 0517e9b64f9f1be95cb3761b01d34e64
+P 8a0296aa4db20cdcbbfaa0e4c61b5fe339b0514cf73832253ff0abd1ce59d31c
+R 1dd3a5c88fdf1dc284f23dd932cd9767
+T *branch * bom-handling
+T *sym-bom-handling *
+T -sym-trunk *
 U drh
-Z fe678a626872621c9456ec3e5b057ad7
+Z e0a3b639a8ffa1ff10e4b189ad45981e
 # Remove this line to create a well-formed Fossil manifest.

manifest.tags

@@ -1,2 +1,2 @@
-branch trunk
-tag trunk
+branch bom-handling
+tag bom-handling

manifest.uuid

@@ -1 +1 @@
-8a0296aa4db20cdcbbfaa0e4c61b5fe339b0514cf73832253ff0abd1ce59d31c
+56e268312d038608b5fb2a8ed24002f478ad5a22eb0214b1a064f2926aada66c

src/sqlite.h.in

@@ -5888,6 +5888,10 @@ int sqlite3_create_window_function(
 ** These constants define integer codes that represent the various
 ** text encodings supported by SQLite.
 **
+** If any value other than the ones defined here are passed into the
+** "encoding" argument of an API function, then the results are
+** unspecified and subject to change.
+**
 ** <dl>
 ** [[SQLITE_UTF8]] <dt>SQLITE_UTF8</dt><dd>Text is encoding as UTF-8</dd>
 **

src/sqliteInt.h

@@ -1843,10 +1843,13 @@ struct sqlite3 {
 };
 
 /*
-** A macro to discover the encoding of a database.
+** A macro to discover the encoding of a database or for managing
+** encoding values.
 */
 #define SCHEMA_ENC(db) ((db)->aDb[0].pSchema->enc)
 #define ENC(db)        ((db)->enc)
+#define ENC_MASK       0x1f           /* Usable encoding bits */
+#define ENC_KEEPBOM    0x80           /* Do not remove BOM */
 
 /*
 ** A u64 constant where the lower 32 bits are all zeros.  Only the

src/vdbe.c

@@ -796,18 +796,14 @@ static SQLITE_NOINLINE int vdbeColumnFromOverflow(
     assert( t>=12 );
     sqlite3RCStrRef(pBuf);
     if( t&1 ){
-      /* Prevents the removal of a BOM in UTF16 text.  Otherwise
-      ** it causes no harm as long as len>=0, and pDest->enc is set
-      ** to the correct encoding afterwards: ----
-      ** (tag-20261016-1)                        `vvvvvvvvvvv    */
-      rc = sqlite3VdbeMemSetStr(pDest, pBuf, len, SQLITE_UTF8,
+      rc = sqlite3VdbeMemSetStr(pDest, pBuf, len, encoding|ENC_KEEPBOM,
                                 sqlite3RCStrUnref);
       pDest->flags |= MEM_Term;
     }else{
       rc = sqlite3VdbeMemSetStr(pDest, pBuf, len, 0,
                                 sqlite3RCStrUnref);
-     }
-    pDest->enc = encoding;
+      pDest->enc = encoding;
+    }
   }else{
     rc = sqlite3VdbeMemFromBtree(pC->uc.pCursor, iOffset, len, pDest);
     if( rc ) return rc;

src/vdbeapi.c

@@ -1784,12 +1784,13 @@ static int bindText(
   const void *zData,     /* Pointer to the data to be bound */
   i64 nData,             /* Number of bytes of data to be bound */
   void (*xDel)(void*),   /* Destructor for the data */
-  u8 encoding            /* Encoding for the data */
+  u8 encoding            /* Encoding for the data + optional ENC_KEEPBOM */
 ){
   Vdbe *p = (Vdbe *)pStmt;
   Mem *pVar;
   int rc;
 
+  assert( (encoding&ENC_KEEPBOM)==0 || (encoding&ENC_MASK)>SQLITE_UTF8 );
   rc = vdbeUnbind(p, (u32)(i-1));
   if( rc==SQLITE_OK ){
     assert( p!=0 && p->aVar!=0 && i>0 && i<=p->nVar ); /* tag-20240917-01 */
@@ -1996,8 +1997,9 @@ int sqlite3_bind_value(sqlite3_stmt *pStmt, int i, const sqlite3_value *pValue){
       break;
     }
     case SQLITE_TEXT: {
+      assert( pValue->enc!=0 );
       rc = bindText(pStmt,i,  pValue->z, pValue->n, SQLITE_TRANSIENT,
-                              pValue->enc);
+              pValue->enc<=SQLITE_UTF8 ? pValue->enc : pValue->enc|ENC_KEEPBOM);
       break;
     }
     default: {

src/vdbemem.c

@@ -1307,11 +1307,13 @@ int sqlite3VdbeMemSetStr(
   i64 nByte = n;      /* New value for pMem->n */
   int iLimit;         /* Maximum allowed string or blob size */
   u16 flags;          /* New value for pMem->flags */
+  u8 xenc;            /* Encoding without the ENC_KEEPBOM bit */
 
   assert( pMem!=0 );
   assert( pMem->db==0 || sqlite3_mutex_held(pMem->db->mutex) );
   assert( !sqlite3VdbeMemIsRowSet(pMem) );
-  assert( enc!=0 || n>=0 );
+  xenc = enc & ENC_MASK;
+  assert( xenc!=0 || n>=0 );
 
   /* If z is a NULL pointer, set pMem to contain an SQL NULL. */
   if( !z ){
@@ -1325,16 +1327,16 @@ int sqlite3VdbeMemSetStr(
     iLimit = SQLITE_MAX_LENGTH;
   }
   if( nByte<0 ){
-    assert( enc!=0 );
-    if( enc==SQLITE_UTF8 ){
+    assert( xenc!=0 );
+    if( xenc==SQLITE_UTF8 ){
       nByte = strlen(z);
     }else{
       for(nByte=0; nByte<=iLimit && (z[nByte] | z[nByte+1]); nByte+=2){}
     }
     flags= MEM_Str|MEM_Term;
-  }else if( enc==0 ){
+  }else if( xenc==0 ){
     flags = MEM_Blob;
-    enc = SQLITE_UTF8;
+    xenc = SQLITE_UTF8;
   }else{
     flags = MEM_Str;
   }
@@ -1357,7 +1359,7 @@ int sqlite3VdbeMemSetStr(
   if( xDel==SQLITE_TRANSIENT ){
     i64 nAlloc = nByte;
     if( flags&MEM_Term ){
-      nAlloc += (enc==SQLITE_UTF8?1:2);
+      nAlloc += (xenc==SQLITE_UTF8?1:2);
     }
     testcase( nAlloc==0 );
     testcase( nAlloc==31 );
@@ -1381,14 +1383,13 @@ int sqlite3VdbeMemSetStr(
 
   pMem->n = (int)(nByte & 0x7fffffff);
   pMem->flags = flags;
-  pMem->enc = enc;
+  pMem->enc = xenc;
 
 #ifndef SQLITE_OMIT_UTF16
-  /* tag-20261016-1:  To retain the BOM, use enc==SQLITE_UTF8, even
-  ** for UTF-16 text, and then set pMem->enc correctly after this
-  ** function returns.  Note, however, that n must be non-negative
-  ** for this trick to work. */
-  if( enc>SQLITE_UTF8 && sqlite3VdbeMemHandleBom(pMem) ){
+  if( xenc>SQLITE_UTF8
+   && (enc & ENC_KEEPBOM)==0
+   && sqlite3VdbeMemHandleBom(pMem)
+  ){
     return SQLITE_NOMEM_BKPT;
   }
 #endif