Revert "No boost condition (#2822)" (#2828)

This reverts commit f916e64927.

utils/rwlock/CMakeLists.txt

@@ -5,7 +5,7 @@ include_directories( ${ENGINE_COMMON_INCLUDES} )
 
 ########### next target ###############
 
-set(rwlock_LIB_SRCS rwlock.cpp)
+set(rwlock_LIB_SRCS rwlock.cpp rwlock_local.cpp)
 
 add_library(rwlock SHARED ${rwlock_LIB_SRCS})
 add_dependencies(rwlock external_boost)

utils/rwlock/rwlock.cpp

@@ -58,7 +58,7 @@ using namespace rwlock;
 
 // This mutex needs to be fully instantiated by the runtime static object
 // init mechanism or the lock in makeRWLockShmImpl() will fail
-std::mutex instanceMapMutex;
+boost::mutex instanceMapMutex;
 typedef std::tr1::unordered_map<int, RWLockShmImpl*> LockMap_t;
 // Windows doesn't init static objects the same as Linux, so make this a ptr
 LockMap_t* lockMapPtr = 0;
@@ -113,7 +113,7 @@ namespace rwlock
 /*static*/
 RWLockShmImpl* RWLockShmImpl::makeRWLockShmImpl(int key, bool* excl)
 {
-  std::unique_lock lk(instanceMapMutex);
+  boost::mutex::scoped_lock lk(instanceMapMutex);
   LockMap_t::iterator iter;
 
   if (!lockMapPtr)

utils/rwlock/rwlock_local.cpp

@@ -0,0 +1,323 @@
+/* Copyright (C) 2014 InfiniDB, Inc.
+
+   This program is free software; you can redistribute it and/or
+   modify it under the terms of the GNU General Public License
+   as published by the Free Software Foundation; version 2 of
+   the License.
+
+   This program is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+   GNU General Public License for more details.
+
+   You should have received a copy of the GNU General Public License
+   along with this program; if not, write to the Free Software
+   Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
+   MA 02110-1301, USA. */
+
+/*****************************************************************************
+ * $Id$
+ *
+ ****************************************************************************/
+
+/*
+ * Brief description of the file contents
+ *
+ * More detailed description
+ */
+
+#include <iostream>
+using namespace std;
+
+#include <boost/thread.hpp>
+#include <boost/thread/condition.hpp>
+using namespace boost;
+
+#define RWLOCK_LOCAL_DLLEXPORT
+#include "rwlock_local.h"
+#undef RWLOCK_LOCAL_DLLEXPORT
+
+// semaphore numbers
+#define MUTEX 0
+#define READERS 1
+#define WRITERS 2
+
+#ifdef DEBUG
+using namespace std;
+#define PRINTSTATE()                                            \
+  cerr << "  reading = " << state.reading << endl               \
+       << "  writing = " << state.writing << endl               \
+       << "  readerswaiting = " << state.readerswaiting << endl \
+       << "  writerswaiting = " << state.writerswaiting << endl;
+
+#define CHECKSAFETY()                                                               \
+  if (!((state.reading == 0 && (state.writing == 0 || state.writing == 1)) ||       \
+        (state.reading > 0 && state.writing == 0)))                                 \
+  {                                                                                 \
+    cerr << "RWLock_local::" << __func__ << ": safety invariant violation" << endl; \
+    PRINTSTATE();                                                                   \
+    throw std::logic_error("RWLock_local: safety invariant violation");             \
+  }
+
+#define CHECKLIVENESS()                                                               \
+  if (!((!(state.readerswaiting > 0 || state.writerswaiting > 0) ||                   \
+         (state.reading > 0 || state.writing > 0)) ||                                 \
+        (!(state.reading == 0 && state.writing == 0) ||                               \
+         (state.readerswaiting == 0 && state.writerswaiting == 0))))                  \
+  {                                                                                   \
+    cerr << "RWLock_local::" << __func__ << ": liveness invariant violation" << endl; \
+    PRINTSTATE();                                                                     \
+    throw std::logic_error("RWLock_local: liveness invariant violation");             \
+  }
+
+#undef CHECKLIVENESS
+#define CHECKLIVENESS()
+
+#endif
+
+namespace rwlock
+{
+RWLock_local::RWLock_local()
+{
+  state.reading = 0;
+  state.readerswaiting = 0;
+  state.writing = 0;
+  state.writerswaiting = 0;
+}
+
+RWLock_local::~RWLock_local()
+{
+}
+
+void RWLock_local::read_lock()
+{
+  mutex.lock();
+#ifdef DEBUG
+  CHECKSAFETY();
+  CHECKLIVENESS();
+#endif
+
+  if (state.writerswaiting > 0 || state.writing > 0)
+  {
+    state.readerswaiting++;
+#ifdef DEBUG
+    CHECKSAFETY();
+    CHECKLIVENESS();
+#endif
+
+    while (state.writerswaiting > 0 || state.writing > 0)
+      okToRead.wait(mutex);
+
+    state.readerswaiting--;
+  }
+
+  state.reading++;
+
+#ifdef DEBUG
+  CHECKSAFETY();
+  CHECKLIVENESS();
+#endif
+
+  mutex.unlock();
+}
+
+void RWLock_local::read_unlock()
+{
+  mutex.lock();
+#ifdef DEBUG
+  CHECKSAFETY();
+  CHECKLIVENESS();
+#endif
+
+  state.reading--;
+
+  if (state.writerswaiting > 0 && state.reading == 0)
+    okToWrite.notify_one();
+
+#ifdef DEBUG
+  CHECKSAFETY();
+  CHECKLIVENESS();
+#endif
+  mutex.unlock();
+}
+
+void RWLock_local::write_lock()
+{
+  mutex.lock();
+#ifdef DEBUG
+  CHECKSAFETY();
+  CHECKLIVENESS();
+#endif
+
+  if (state.writing > 0 || state.reading > 0)
+  {
+    state.writerswaiting++;
+
+#ifdef DEBUG
+    CHECKSAFETY();
+    CHECKLIVENESS();
+#endif
+
+    while (state.writing > 0 || state.reading > 0)
+      okToWrite.wait(mutex);
+
+    state.writerswaiting--;
+  }
+
+  state.writing++;
+
+#ifdef DEBUG
+  CHECKSAFETY();
+  CHECKLIVENESS();
+#endif
+}
+
+void RWLock_local::write_unlock()
+{
+#ifdef DEBUG
+  CHECKSAFETY();
+  CHECKLIVENESS();
+#endif
+
+  state.writing--;
+
+  if (state.writerswaiting > 0)
+    okToWrite.notify_one();
+  else if (state.readerswaiting > 0)
+    okToRead.notify_all();
+
+#ifdef DEBUG
+  CHECKSAFETY();
+  CHECKLIVENESS();
+#endif
+  mutex.unlock();
+}
+
+void RWLock_local::upgrade_to_write()
+{
+  mutex.lock();
+#ifdef DEBUG
+  CHECKSAFETY();
+  CHECKLIVENESS();
+#endif
+  state.reading--;
+
+  // try to cut in line
+  if (state.reading == 0)
+  {
+    state.writing++;
+#ifdef DEBUG
+    CHECKSAFETY();
+    CHECKLIVENESS();
+#endif
+    return;
+  }
+
+  // cut & paste from write_lock()
+  if (state.writing > 0 || state.reading > 0)
+  {
+    state.writerswaiting++;
+
+#ifdef DEBUG
+    CHECKSAFETY();
+    CHECKLIVENESS();
+#endif
+
+    while (state.writing > 0 || state.reading > 0)
+      okToWrite.wait(mutex);
+
+    state.writerswaiting--;
+  }
+
+  state.writing++;
+}
+
+/* It's safe (and necessary) to simply convert this writer to a reader without
+ blocking */
+void RWLock_local::downgrade_to_read()
+{
+#ifdef DEBUG
+  CHECKSAFETY();
+  CHECKLIVENESS();
+#endif
+
+  state.writing--;
+
+  if (state.readerswaiting > 0)
+    okToRead.notify_all();
+
+  state.reading++;
+
+#ifdef DEBUG
+  CHECKSAFETY();
+  CHECKLIVENESS();
+#endif
+  mutex.unlock();
+}
+
+void RWLock_local::lock()
+{
+  mutex.lock();
+}
+
+void RWLock_local::unlock()
+{
+  mutex.unlock();
+}
+
+int RWLock_local::getWriting()
+{
+  return state.writing;
+}
+
+int RWLock_local::getReading()
+{
+  return state.reading;
+}
+
+int RWLock_local::getWritersWaiting()
+{
+  return state.writerswaiting;
+}
+
+int RWLock_local::getReadersWaiting()
+{
+  return state.readerswaiting;
+}
+
+ScopedRWLock_local::ScopedRWLock_local(RWLock_local* l, rwlock_mode m)
+{
+  thelock = l;
+  mode = m;
+  assert(m == R || m == W);
+  locked = false;
+  lock();
+}
+
+ScopedRWLock_local::~ScopedRWLock_local()
+{
+  if (locked)
+    unlock();
+}
+
+void ScopedRWLock_local::lock()
+{
+  if (mode == R)
+    thelock->read_lock();
+  else
+    thelock->write_lock();
+
+  locked = true;
+}
+
+void ScopedRWLock_local::unlock()
+{
+  if (mode == R)
+    thelock->read_unlock();
+  else
+    thelock->write_unlock();
+
+  locked = false;
+}
+
+}  // namespace rwlock

utils/rwlock/rwlock_local.h

@@ -0,0 +1,192 @@
+/* Copyright (C) 2014 InfiniDB, Inc.
+
+   This program is free software; you can redistribute it and/or
+   modify it under the terms of the GNU General Public License
+   as published by the Free Software Foundation; version 2 of
+   the License.
+
+   This program is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+   GNU General Public License for more details.
+
+   You should have received a copy of the GNU General Public License
+   along with this program; if not, write to the Free Software
+   Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
+   MA 02110-1301, USA. */
+
+/******************************************************************************
+ * $Id$
+ *
+ *****************************************************************************/
+
+/** @file
+ * class RWLock_local interface
+ */
+
+#pragma once
+
+#include <boost/thread.hpp>
+#include <boost/thread/condition.hpp>
+
+#define EXPORT
+
+namespace rwlock
+{
+/** @brief Implements RW locks for use across threads & processes
+ *
+ * Implements RW locks for use across threads & processes.  Every
+ * instance that shares a lock must be instantiated using the same
+ * key.  There is 'no limit' on the number of RW locks that can
+ * exist on the system at any one time.
+ *
+ * Summary of operation:
+ * 		- readers can work concurrently
+ *		- writers get exclusive access
+ * 		- writers have priority
+ *		- all state persists across all invocations sharing a given key
+ *
+ * Note: because state has to persist, it will have to be cleaned
+ * up somewhere else.  Crashes while holding a read or write lock will
+ * eventually deadlock the set of processes that share the same key obviously.
+ */
+
+class RWLock_local
+{
+ public:
+  class not_excl : public std::exception
+  {
+   public:
+    virtual const char* what() const throw()
+    {
+      return "not_excl";
+    }
+  };
+
+  class wouldblock : public std::exception
+  {
+   public:
+    virtual const char* what() const throw()
+    {
+      return "wouldblock";
+    }
+  };
+
+  /** @brief Keyed constructor.
+   *
+   * Instantiate an RWLock_local with the given key.  All instances that
+   * share a key share the same lock.
+   *
+   * @param key The key
+   * @param excl If true and this is the first instance with the
+   * supplied key, it will return holding the write lock.  If true and
+   * this is not the first instance, it will throw not_excl.  The intent
+   * is similar to the IPC_EXCL flag in the sem/shm implementations.
+   */
+  EXPORT RWLock_local();
+
+  EXPORT ~RWLock_local();
+
+  /** @brief Grab a read lock
+   *
+   * Grab a read lock.  This will block iff writers are waiting or
+   * a writer is active.
+   *
+   * @param block (For testing only) If false, will throw
+   * wouldblock instead of blocking
+   */
+  EXPORT void read_lock();
+
+  /** @brief Release a read lock.
+   *
+   * Release a read lock.
+   */
+  EXPORT void read_unlock();
+
+  /** @brief Grab a write lock
+   *
+   * Grab a write lock.  This will block while another writer or reader is
+   * active and will have exclusive access on waking.
+   *
+   * @param block (For testing only) If false, will throw
+   * wouldblock instead of blocking
+   */
+  EXPORT void write_lock();
+
+  /** @brief Release a write lock.
+   *
+   * Release a write lock.
+   */
+  EXPORT void write_unlock();
+
+  /** @brief Upgrade a read lock to a write lock
+   *
+   * Upgrade a read lock to a write lock.  It may have to block
+   * if there are other readers currently reading.  No guarantees of atomicity.
+   */
+  EXPORT void upgrade_to_write();
+
+  /** @brief Downgrade a write lock to a read lock
+   *
+   * Downgrade a write lock to a read lock.  The conversion happens
+   * atomically.
+   */
+  EXPORT void downgrade_to_read();
+
+  /* These are for white box testing only */
+  EXPORT void lock();
+  EXPORT void unlock();
+  EXPORT int getWriting();
+  EXPORT int getReading();
+  EXPORT int getWritersWaiting();
+  EXPORT int getReadersWaiting();
+
+ private:
+  // Not copyable
+  RWLock_local(const RWLock_local& rwl);
+  RWLock_local& operator=(const RWLock_local& rwl);
+
+  /// the layout of the shmseg
+  struct State
+  {
+    int writerswaiting, writing, readerswaiting, reading;
+  } state;
+  boost::mutex mutex;
+  boost::condition okToRead;
+  boost::condition okToWrite;
+};
+
+enum rwlock_mode
+{
+  R,
+  W
+};
+
+class ScopedRWLock_local
+{
+ public:
+  ScopedRWLock_local(RWLock_local*, rwlock_mode);
+  ~ScopedRWLock_local();
+
+  void lock();
+  void unlock();
+
+ private:
+  explicit ScopedRWLock_local()
+  {
+  }
+  explicit ScopedRWLock_local(const ScopedRWLock_local&)
+  {
+  }
+  ScopedRWLock_local& operator=(const ScopedRWLock_local&)
+  {
+    return *this;
+  }
+  RWLock_local* thelock;
+  rwlock_mode mode;
+  bool locked;
+};
+
+#undef EXPORT
+
+}  // namespace rwlock

utils/rwlock/tdriver-rw.cpp

@@ -0,0 +1,296 @@
+/* Copyright (C) 2014 InfiniDB, Inc.
+
+   This program is free software; you can redistribute it and/or
+   modify it under the terms of the GNU General Public License
+   as published by the Free Software Foundation; version 2 of
+   the License.
+
+   This program is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+   GNU General Public License for more details.
+
+   You should have received a copy of the GNU General Public License
+   along with this program; if not, write to the Free Software
+   Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
+   MA 02110-1301, USA. */
+
+/*****************************************************************************
+ * $Id$
+ *
+ ****************************************************************************/
+
+/** @file
+ * Brief description of the file contents
+ *
+ * More detailed description
+ */
+
+#include <iostream>
+#include <sys/types.h>
+#include <stdexcept>
+#include <stdlib.h>
+#include <sys/time.h>
+#include <unistd.h>
+#include <pthread.h>
+
+#include <cppunit/extensions/HelperMacros.h>
+
+#include "rwlock.h"
+#include "rwlock_local.h"
+
+using namespace std;
+using namespace rwlock;
+
+int threadStop;
+
+static void* RWRunner(void* arg)
+{
+  struct timeval tv;
+  int op, op2, interval;
+  RWLock* rwlock;
+
+  gettimeofday(&tv, NULL);
+  rwlock = new RWLock(reinterpret_cast<int64_t>(arg));
+
+  while (!threadStop)
+  {
+    op = rand_r(reinterpret_cast<uint32_t*>(&tv.tv_usec)) % 10;
+
+    if (op < 8)  // read
+    {
+      interval = rand_r(reinterpret_cast<uint32_t*>(&tv.tv_usec)) % 100000;
+      rwlock->read_lock();
+      rwlock->lock();
+      CPPUNIT_ASSERT(rwlock->getReading() > 0);
+      CPPUNIT_ASSERT(rwlock->getWriting() == 0);
+      rwlock->unlock();
+      usleep(interval);
+
+      op2 = rand_r(reinterpret_cast<uint32_t*>(&tv.tv_usec)) % 2;
+
+      if (op2)
+      {
+        rwlock->upgrade_to_write();
+        rwlock->lock();
+        CPPUNIT_ASSERT(rwlock->getReading() == 0);
+        CPPUNIT_ASSERT(rwlock->getWriting() == 1);
+        rwlock->unlock();
+        usleep(interval);
+        rwlock->write_unlock();
+      }
+      else
+      {
+        /* For testing the lock recovery code in the BRM workernodes */
+        /*
+        int crash = rand_r((uint32_t *) &tv.tv_usec) % 100;
+        if (crash > 0)   // 1% chance of crashing
+                rwlock->read_unlock();
+        */
+      }
+    }
+    else if (op < 9)  // write
+    {
+      interval = rand_r(reinterpret_cast<uint32_t*>(&tv.tv_usec)) % 100000;
+      rwlock->write_lock();
+      rwlock->lock();
+      CPPUNIT_ASSERT(rwlock->getReading() == 0);
+      CPPUNIT_ASSERT(rwlock->getWriting() == 1);
+      rwlock->unlock();
+      usleep(interval);
+
+      op2 = rand_r(reinterpret_cast<uint32_t*>(&tv.tv_usec)) % 2;
+
+      if (op2)
+      {
+        rwlock->downgrade_to_read();
+        rwlock->lock();
+        CPPUNIT_ASSERT(rwlock->getReading() > 0);
+        CPPUNIT_ASSERT(rwlock->getWriting() == 0);
+        rwlock->unlock();
+        usleep(interval);
+        rwlock->read_unlock();
+      }
+      else
+
+        rwlock->write_unlock();
+    }
+    else if (op == 9)  // delete
+    {
+      delete rwlock;
+      rwlock = new RWLock(reinterpret_cast<int64_t>(arg));
+    }
+  }
+
+  delete rwlock;
+  pthread_exit(0);
+}
+
+static void* RWRunner_local(void* arg)
+{
+  struct timeval tv;
+  int op, op2, interval;
+  RWLock_local* rwlock = reinterpret_cast<RWLock_local*>(arg);
+
+  gettimeofday(&tv, NULL);
+
+  while (!threadStop)
+  {
+    op = rand_r(reinterpret_cast<uint32_t*>(&tv.tv_usec)) % 10;
+
+    //    		cout << "doing op " << op << endl;
+    switch (op)
+    {
+      case 0:  // read
+      case 1:
+      case 2:
+      case 3:
+      case 4:
+      case 5:
+      case 6:
+      case 7:
+      case 8:
+      {
+        interval = rand_r(reinterpret_cast<uint32_t*>(&tv.tv_usec)) % 100000;
+        rwlock->read_lock();
+        rwlock->lock();
+        CPPUNIT_ASSERT(rwlock->getReading() > 0);
+        CPPUNIT_ASSERT(rwlock->getWriting() == 0);
+        rwlock->unlock();
+        usleep(interval);
+        op2 = rand_r(reinterpret_cast<uint32_t*>(&tv.tv_usec)) % 2;
+
+        if (op2)
+        {
+          rwlock->upgrade_to_write();
+          // 					rwlock->lock();
+          CPPUNIT_ASSERT(rwlock->getReading() == 0);
+          CPPUNIT_ASSERT(rwlock->getWriting() == 1);
+          // 					rwlock->unlock();
+          usleep(interval);
+          rwlock->write_unlock();
+        }
+
+        break;
+      }
+
+      case 9:  // write
+      {
+        interval = rand_r(reinterpret_cast<uint32_t*>(&tv.tv_usec)) % 100000;
+        rwlock->write_lock();
+        // 				rwlock->lock();
+        CPPUNIT_ASSERT(rwlock->getReading() == 0);
+        CPPUNIT_ASSERT(rwlock->getWriting() == 1);
+        // 				rwlock->unlock();
+        usleep(interval);
+        op2 = rand_r(reinterpret_cast<uint32_t*>(&tv.tv_usec)) % 2;
+
+        if (op2)
+        {
+          rwlock->downgrade_to_read();
+          rwlock->lock();
+          CPPUNIT_ASSERT(rwlock->getReading() > 0);
+          CPPUNIT_ASSERT(rwlock->getWriting() == 0);
+          rwlock->unlock();
+          usleep(interval);
+          rwlock->read_unlock();
+        }
+        else
+          rwlock->write_unlock();
+
+        break;
+      }
+
+      default: break;
+    }
+  }
+
+  pthread_exit(0);
+}
+
+class RWLockTest : public CppUnit::TestFixture
+{
+  CPPUNIT_TEST_SUITE(RWLockTest);
+
+  CPPUNIT_TEST(LongRWTest_1);
+  // CPPUNIT_TEST(LongRWLocalTest_1);
+
+  CPPUNIT_TEST_SUITE_END();
+
+ private:
+ public:
+  void LongRWTest_1()
+  {
+    int key = 0x20000;  // the extentmap key
+
+    const int threadCount = 30;
+    int i;
+    pthread_t threads[threadCount];
+
+    cerr << endl
+         << "Multithreaded RWLock test.  "
+            "This runs for 60 minutes."
+         << endl;
+
+    threadStop = 0;
+
+    for (i = 0; i < threadCount; i++)
+    {
+      if (pthread_create(&threads[i], NULL, RWRunner, reinterpret_cast<void*>(key)) < 0)
+        throw logic_error("Error creating threads for the ipc test");
+    }
+
+    sleep(3600);
+    threadStop = 1;
+
+    for (i = 0; i < threadCount; i++)
+    {
+      cerr << "Waiting for thread #" << i << endl;
+      pthread_join(threads[i], NULL);
+    }
+  }
+
+  void LongRWLocalTest_1()
+  {
+    const int threadCount = 40;
+    int i;
+    pthread_t threads[threadCount];
+    RWLock_local rwlock;
+
+    cerr << endl
+         << "Multithreaded RWLock_local test.  "
+            "This runs for 30-60 seconds."
+         << endl;
+
+    threadStop = 0;
+
+    for (i = 0; i < threadCount; i++)
+    {
+      if (pthread_create(&threads[i], NULL, RWRunner_local, reinterpret_cast<void*>(&rwlock)) < 0)
+        throw logic_error("Error creating threads for the local test");
+    }
+
+    sleep(30);
+    threadStop = 1;
+
+    for (i = 0; i < threadCount; i++)
+    {
+      cerr << "Waiting for thread #" << i << endl;
+      pthread_join(threads[i], NULL);
+    }
+  }
+};
+
+CPPUNIT_TEST_SUITE_REGISTRATION(RWLockTest);
+
+#include <cppunit/extensions/TestFactoryRegistry.h>
+#include <cppunit/ui/text/TestRunner.h>
+
+int main(int argc, char** argv)
+{
+  CppUnit::TextUi::TestRunner runner;
+  CppUnit::TestFactoryRegistry& registry = CppUnit::TestFactoryRegistry::getRegistry();
+  runner.addTest(registry.makeTest());
+  bool wasSuccessful = runner.run("", false);
+  return (wasSuccessful ? 0 : 1);
+}