mariadb-columnstore-engine/versioning/BRM/vbbm.h

/* Copyright (C) 2014 InfiniDB, Inc.

   This program is free software; you can redistribute it and/or
   modify it under the terms of the GNU General Public License
   as published by the Free Software Foundation; version 2 of
   the License.

   This program is distributed in the hope that it will be useful,
   but WITHOUT ANY WARRANTY; without even the implied warranty of
   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
   GNU General Public License for more details.

   You should have received a copy of the GNU General Public License
   along with this program; if not, write to the Free Software
   Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
   MA 02110-1301, USA. */

/******************************************************************************
 * $Id: vbbm.h 1926 2013-06-30 21:18:14Z wweeks $
 *
 *****************************************************************************/

/** @file
 * class XXX interface
 */

#pragma once

#include <vector>
//#define NDEBUG
#include <cassert>
#include <boost/thread.hpp>

#include "brmshmimpl.h"

#include "shmkeys.h"
#include "brmtypes.h"
#include "undoable.h"
#include "mastersegmenttable.h"

// These config parameters need to be loaded

// will get a small hash function performance boost by using powers of 2
#define VBSTORAGE_INITIAL_COUNT 100000
#define VBSTORAGE_INITIAL_SIZE (VBSTORAGE_INITIAL_COUNT * sizeof(VBBMEntry))
#define VBSTORAGE_INCREMENT_COUNT 10000
#define VBSTORAGE_INCREMENT (VBSTORAGE_INCREMENT_COUNT * sizeof(VBBMEntry))

// (average list length = 4)
#define VBTABLE_INITIAL_SIZE (25000 * sizeof(int))
#define VBTABLE_INCREMENT (2500 * sizeof(int))

#define VBBM_INCREMENT (VBTABLE_INCREMENT + VBSTORAGE_INCREMENT)

#define VBBM_SIZE(files, entries)                                                                   \
  ((entries * sizeof(VBBMEntry)) + (entries / 4 * sizeof(int)) + (files * sizeof(VBFileMetadata)) + \
   sizeof(VBShmsegHeader))

#define EXPORT

namespace idbdatafile
{
class IDBDataFile;
}

namespace BRM
{
class VSS;

struct VBFileMetadata
{
  OID_t OID;
  uint64_t fileSize;
  uint64_t nextOffset;
};

struct VBBMEntry
{
  LBID_t lbid;
  VER_t verID;
  OID_t vbOID;
  uint32_t vbFBO;
  int next;
  EXPORT VBBMEntry();
};

struct VBShmsegHeader
{
  int nFiles;
  int vbCapacity;
  int vbCurrentSize;
  int vbLWM;
  int numHashBuckets;

  // the rest of the overlay looks like this
  // 	VBFileMetadata files[nFiles];
  // 	int hashBuckets[numHashBuckets];
  // 	VBBMEntry storage[vbCapacity];
};

class VBBMImpl
{
 public:
  static VBBMImpl* makeVBBMImpl(unsigned key, off_t size, bool readOnly = false);

  inline void grow(unsigned key, off_t size)
#ifdef NDEBUG
  {
    fVBBM.grow(key, size);
  }
#else
  {
    int rc = fVBBM.grow(key, size);
    idbassert(rc == 0);
  }
#endif
  inline void makeReadOnly()
  {
    fVBBM.setReadOnly();
  }
  inline void clear(unsigned key, off_t size)
  {
    fVBBM.clear(key, size);
  }
  inline void swapout(BRMShmImpl& rhs)
  {
    fVBBM.swap(rhs);
    rhs.destroy();
  }
  inline unsigned key() const
  {
    return fVBBM.key();
  }

  inline VBShmsegHeader* get() const
  {
    return reinterpret_cast<VBShmsegHeader*>(fVBBM.fMapreg.get_address());
  }

 private:
  VBBMImpl(unsigned key, off_t size, bool readOnly = false);
  ~VBBMImpl();
  VBBMImpl(const VBBMImpl& rhs);
  VBBMImpl& operator=(const VBBMImpl& rhs);

  BRMShmImpl fVBBM;

  static boost::mutex fInstanceMutex;
  static VBBMImpl* fInstance;
};

/** @brief The Version Buffer Block Map (VBBM)
 *
 * At a high level, the VBBM maintains a table describing the contents of
 * the Version Buffer.  For every entry in the Version Buffer, it associates its
 * <LBID, VerID> identifier with the OID and offset it is stored at.
 *
 * As implemented, it is a hash table and a set of lists that exist in
 * shared memory.  The hash table is keyed by <LBID, VerID>, and
 * each valid entry points to the head of a unique list.  Each list element
 * contains the LBID, VerID, VB OID, and VB offset that encapsulate "an entry in the
 * VBBM table".  Every list contains all elements that collide on that hash table
 * entry that points to it, "load factor" has no bearing on performance,
 * and lists can grow arbitrarily large.
 * Technically lookups are O(n), but in normal circumstances it'll
 * be constant time.  As things are right now, we expect there to be about
 * 100k VBBM entries.  The hash table is sized such that on average there will be 4
 * entries per list when it's at capacity.
 *
 * The memory management & structure manipulation code is nearly identical
 * to that in the VSS, so any bugs found here are likely there as well.
 *
 * Shared memory is managed using code similar to the ExtentMap & VSS.  When
 * the shared memory segment needs to grow, it is write-locked, a new one
 * is created, the contents are reinserted to the new one, the key is
 * registered, and the old segment is destroyed when the last reference to it
 * is detached.
 */

class VBBM : public Undoable
{
 public:
  enum OPS
  {
    NONE,
    READ,
    WRITE
  };

  EXPORT VBBM();
  EXPORT ~VBBM();

  EXPORT void lock(OPS op);
  EXPORT void release(OPS op);
  EXPORT int lookup(LBID_t lbid, VER_t ver, OID_t& oid, uint32_t& fbo) const;
  EXPORT void insert(LBID_t lbid, VER_t ver, OID_t oid, uint32_t fbo, bool loading = false);
  EXPORT void getBlocks(int num, OID_t vbOID, std::vector<VBRange>& vbRanges, VSS& vss, bool flushPMCache);
  EXPORT void removeEntry(LBID_t, VER_t ver);

  EXPORT int size() const;
  EXPORT bool hashEmpty() const;
  EXPORT int checkConsistency() const;
  EXPORT void setReadOnly();

  EXPORT void clear();
  EXPORT void load(std::string filename);
  EXPORT void loadVersion2(idbdatafile::IDBDataFile* in);
  EXPORT void save(std::string filename);

#ifdef BRM_DEBUG
  EXPORT int getShmid() const;
#endif

 private:
  VBBM(const VBBM&);
  VBBM& operator=(const VBBM&);

  VBShmsegHeader* vbbm;
  VBFileMetadata* files;
  int* hashBuckets;
  VBBMEntry* storage;

  key_t currentVBBMShmkey;
  int vbbmShmid;
  bool r_only;
  MSTEntry* vbbmShminfo;
  MasterSegmentTable mst;
  static boost::mutex mutex;  // @bug5355 - made mutex static
  static const int MAX_IO_RETRIES = 10;

  key_t chooseShmkey() const;
  void growVBBM(bool addAFile = false);
  void growForLoad(int count);
  void copyVBBM(VBShmsegHeader* dest);
  void initShmseg(int nFiles);

  void _insert(VBBMEntry& e, VBShmsegHeader* dest, int* destTable, VBBMEntry* destStorage,
               bool loading = false);
  int getIndex(LBID_t lbid, VER_t verID, int& prev, int& bucket) const;
  ShmKeys fShmKeys;
  VBBMImpl* fPVBBMImpl;

  /* Shared nothing mods */
  uint64_t currentFileSize;
  void setCurrentFileSize();
  uint32_t addVBFileIfNotExists(OID_t vbOID);
};

}  // namespace BRM

#undef EXPORT