More efficient implementation of a lookaside allocator that supports mini (in this case, harcoded to 128B) slots.

FossilOrigin-Name: b02fdc09c838f355d9efce57f817d6a86153153b4a1f2b763914f4aaa34cb76e

src/main.c

@@ -683,6 +683,7 @@ int sqlite3_config(int op, ...){
 static int setupLookaside(sqlite3 *db, void *pBuf, int sz, int cnt){
 #ifndef SQLITE_OMIT_LOOKASIDE
   void *pStart;
+  sqlite3_int64 szAlloc = sz*(sqlite3_int64)cnt;
   
   if( sqlite3LookasideUsed(db,0)>0 ){
     return SQLITE_BUSY;
@@ -705,7 +706,7 @@ static int setupLookaside(sqlite3 *db, void *pBuf, int sz, int cnt){
     pStart = 0;
   }else if( pBuf==0 ){
     sqlite3BeginBenignMalloc();
-    pStart = sqlite3Malloc( sz*(sqlite3_int64)cnt );  /* IMP: R-61949-35727 */
+    pStart = sqlite3Malloc( szAlloc );  /* IMP: R-61949-35727 */
     sqlite3EndBenignMalloc();
     if( pStart ) cnt = sqlite3MallocSize(pStart)/sz;
   }else{
@@ -714,6 +715,15 @@ static int setupLookaside(sqlite3 *db, void *pBuf, int sz, int cnt){
   db->lookaside.pStart = pStart;
   db->lookaside.pInit = 0;
   db->lookaside.pFree = 0;
+#ifndef SQLITE_OMIT_MINI_LOOKASIDE
+  db->lookaside.pMiniInit = 0;
+  db->lookaside.pMiniFree = 0;
+  /* The arithmetic below causes the number of large lookaside slots to be 1/3
+  ** the number of mini slots, based on the observation that 75% of allocations
+  ** are <=128B.
+  */
+  cnt = szAlloc/(3*128+sz);
+#endif
   db->lookaside.sz = (u16)sz;
   db->lookaside.szTrue = (u16)sz;
   if( pStart ){
@@ -727,6 +737,16 @@ static int setupLookaside(sqlite3 *db, void *pBuf, int sz, int cnt){
       db->lookaside.pInit = p;
       p = (LookasideSlot*)&((u8*)p)[sz];
     }
+#ifndef SQLITE_OMIT_MINI_LOOKASIDE
+    db->lookaside.pMiddle = p;
+    /* Fill the remainder of the buffer with mini slots */
+    while(p<=(LookasideSlot*)&((u8*)pStart)[szAlloc-128]){
+      p->pNext = db->lookaside.pMiniInit;
+      db->lookaside.pMiniInit = p;
+      db->lookaside.nSlot++;
+      p = (LookasideSlot*)&((u8*)p)[128];
+    }
+#endif
     db->lookaside.pEnd = p;
     db->lookaside.bDisable = 0;
     db->lookaside.bMalloced = pBuf==0 ?1:0;

src/malloc.c

@@ -332,6 +332,13 @@ int sqlite3MallocSize(void *p){
   assert( sqlite3MemdebugHasType(p, MEMTYPE_HEAP) );
   return sqlite3GlobalConfig.m.xSize(p);
 }
+static int lookasideMallocSize(sqlite3 *db, void *p){
+#ifndef SQLITE_OMIT_MINI_LOOKASIDE    
+  return p<db->lookaside.pMiddle ? db->lookaside.szTrue : 128;
+#else
+  return db->lookaside.szTrue;
+#endif  
+}
 int sqlite3DbMallocSize(sqlite3 *db, void *p){
   assert( p!=0 );
   if( db==0 || !isLookaside(db,p) ){
@@ -347,7 +354,7 @@ int sqlite3DbMallocSize(sqlite3 *db, void *p){
     return sqlite3GlobalConfig.m.xSize(p);
   }else{
     assert( sqlite3_mutex_held(db->mutex) );
-    return db->lookaside.szTrue;
+    return lookasideMallocSize(db, p);
   }
 }
 sqlite3_uint64 sqlite3_msize(void *p){
@@ -397,6 +404,17 @@ void sqlite3DbFreeNN(sqlite3 *db, void *p){
     }
     if( isLookaside(db, p) ){
       LookasideSlot *pBuf = (LookasideSlot*)p;
+#ifndef SQLITE_OMIT_MINI_LOOKASIDE
+      if( p>=db->lookaside.pMiddle ){
+# ifdef SQLITE_DEBUG
+        /* Trash all content in the buffer being freed */
+        memset(p, 0xaa, 128);
+# endif
+        pBuf->pNext = db->lookaside.pMiniFree;
+        db->lookaside.pMiniFree = pBuf;
+        return;
+      }
+#endif
 #ifdef SQLITE_DEBUG
       /* Trash all content in the buffer being freed */
       memset(p, 0xaa, db->lookaside.szTrue);
@@ -564,6 +582,18 @@ void *sqlite3DbMallocRawNN(sqlite3 *db, u64 n){
       return db->mallocFailed ? 0 : dbMallocRawFinish(db, n);
     }
     db->lookaside.anStat[1]++;
+# ifndef SQLITE_OMIT_MINI_LOOKASIDE
+  }else if( n<=128 ){
+    if( (pBuf = db->lookaside.pMiniFree)!=0 ){
+      db->lookaside.pMiniFree = pBuf->pNext;
+      db->lookaside.anStat[0]++;
+      return (void*)pBuf;
+    }else if( (pBuf = db->lookaside.pMiniInit)!=0 ){
+      db->lookaside.pMiniInit = pBuf->pNext;
+      db->lookaside.anStat[0]++;
+      return (void*)pBuf;
+    }
+# endif
   }else if( (pBuf = db->lookaside.pFree)!=0 ){
     db->lookaside.pFree = pBuf->pNext;
     db->lookaside.anStat[0]++;
@@ -597,7 +627,7 @@ void *sqlite3DbRealloc(sqlite3 *db, void *p, u64 n){
   assert( db!=0 );
   if( p==0 ) return sqlite3DbMallocRawNN(db, n);
   assert( sqlite3_mutex_held(db->mutex) );
-  if( isLookaside(db,p) && n<=db->lookaside.szTrue ) return p;
+  if( isLookaside(db,p) && n<lookasideMallocSize(db, p) ) return p;
   return dbReallocFinish(db, p, n);
 }
 static SQLITE_NOINLINE void *dbReallocFinish(sqlite3 *db, void *p, u64 n){
@@ -608,7 +638,7 @@ static SQLITE_NOINLINE void *dbReallocFinish(sqlite3 *db, void *p, u64 n){
     if( isLookaside(db, p) ){
       pNew = sqlite3DbMallocRawNN(db, n);
       if( pNew ){
-        memcpy(pNew, p, db->lookaside.szTrue);
+        memcpy(pNew, p, lookasideMallocSize(db, p));
         sqlite3DbFree(db, p);
       }
     }else{

src/sqliteInt.h

@@ -1287,6 +1287,11 @@ struct Lookaside {
   u32 anStat[3];          /* 0: hits.  1: size misses.  2: full misses */
   LookasideSlot *pInit;   /* List of buffers not previously used */
   LookasideSlot *pFree;   /* List of available buffers */
+#ifndef SQLITE_OMIT_MINI_LOOKASIDE
+  LookasideSlot *pMiniInit; /* List of mini buffers not prediously used */
+  LookasideSlot *pMiniFree; /* List of available mini buffers */
+  void *pMiddle;          /* An address between the fullsize and mini buffers */
+#endif
   void *pStart;           /* First byte of available memory space */
   void *pEnd;             /* First byte past end of available space */
 };

src/status.c

@@ -188,6 +188,10 @@ static u32 countLookasideSlots(LookasideSlot *p){
 int sqlite3LookasideUsed(sqlite3 *db, int *pHighwater){
   u32 nInit = countLookasideSlots(db->lookaside.pInit);
   u32 nFree = countLookasideSlots(db->lookaside.pFree);
+#ifndef SQLITE_OMIT_MINI_LOOKASIDE
+  nInit += countLookasideSlots(db->lookaside.pMiniInit);
+  nFree += countLookasideSlots(db->lookaside.pMiniFree);
+#endif
   if( pHighwater ) *pHighwater = db->lookaside.nSlot - nInit;
   return db->lookaside.nSlot - (nInit+nFree);
 }
@@ -220,6 +224,15 @@ int sqlite3_db_status(
           db->lookaside.pInit = db->lookaside.pFree;
           db->lookaside.pFree = 0;
         }
+#ifndef SQLITE_OMIT_MINI_LOOKASIDE
+        p = db->lookaside.pMiniFree;
+        if( p ){
+          while( p->pNext ) p = p->pNext;
+          p->pNext = db->lookaside.pMiniInit;
+          db->lookaside.pMiniInit = db->lookaside.pMiniFree;
+          db->lookaside.pMiniFree = 0;
+        }
+#endif
       }
       break;
     }