Add application defined sequential consistency for certification

Client state methods before_prepare() and before_commit() accept
a callback which is called by the provider after it can guarantee
sequential consistency. This allows application threads which wish to
maintain sequential consistency to enter before_prepare() and
before_commit() calls concurrently without waiting the prior call
to finish.

include/wsrep/client_state.hpp

@@ -413,11 +413,49 @@ namespace wsrep
 
         /** @name Commit ordering interface */
         /** @{ */
-        int before_prepare();
+
+        /**
+         * This method should be called before the transaction
+         * is prepared. This call certifies the transaction and
+         * assigns write set meta data.
+         *
+         * @param seq_cb Callback which is passed to underlying
+         *               certify() call. See wsrep::provider::certify().
+         *
+         * @return Zero on success, non-zero on failure.
+         */
+        int before_prepare(const wsrep::provider::seq_cb_t* seq_cb);
+
+        /** Same as before_prepare() above, but nullptr is passed
+         * to seq_cb. */
+        int before_prepare()
+        {
+            return before_prepare(nullptr);
+        }
 
         int after_prepare();
 
-        int before_commit();
+        /**
+         * This method should be called before transaction is committed.
+         * This call makes the transaction to enter commit time
+         * critical section. The critical section is left by calling
+         * ordered_commit().
+         *
+         * If before_prepare() is not called before this call, the
+         * before_prepare() is called internally.
+         *
+         * @param seq_cb Callback which is passed to underlying
+         *               before_prepare() call.
+         *
+         * @return Zero on success, non-zero on failure.
+         */
+        int before_commit(const wsrep::provider::seq_cb_t* seq_cb);
+
+        /** Same as before_commit(), but nullptr is passed to seq_cb. */
+        int before_commit()
+        {
+            return before_commit(nullptr);
+        }
 
         int ordered_commit();
 

include/wsrep/provider.hpp

@@ -332,10 +332,37 @@ namespace wsrep
         virtual int append_key(wsrep::ws_handle&, const wsrep::key&) = 0;
         virtual enum status append_data(
             wsrep::ws_handle&, const wsrep::const_buffer&) = 0;
+
+        /**
+         * Callback for application defined sequential consistency.
+         * The provider will call
+         * the callback once it can guarantee sequential consistency. */
+        typedef struct seq_cb {
+            /** Opaque caller context */
+            void *ctx;
+            /** Function to be called by the provider when sequential
+             * consistency is guaranteed. */
+            void (*fn)(void *ctx);
+        } seq_cb_t;
+
+        /**
+         * Certify the write set. 
+         *
+         * @param client_id[in] Id of the client session.
+         * @param ws_handle[in,out] Write set handle associated to the current
+         *                 transaction.
+         * @param flags[in] Flags associated to the write set (see struct flag).
+         * @param ws_meta[out] Write set meta data associated to the
+         *                     replicated write set.
+         * @param seq_cb[in] Optional callback for application defined
+         *                   sequential consistency.
+         *
+         * @return Status code defined in struct status.
+         */
         virtual enum status
-        certify(wsrep::client_id, wsrep::ws_handle&,
-                int,
-                wsrep::ws_meta&) = 0;
+        certify(wsrep::client_id client_id, wsrep::ws_handle& ws_handle,
+                int flags, wsrep::ws_meta& ws_meta, const seq_cb_t* seq_cb)
+            = 0;
         /**
          * BF abort a transaction inside provider.
          *

include/wsrep/transaction.hpp

@@ -175,11 +175,12 @@ namespace wsrep
 
         int after_row();
 
-        int before_prepare(wsrep::unique_lock<wsrep::mutex>&);
+        int before_prepare(wsrep::unique_lock<wsrep::mutex>&,
+                           const wsrep::provider::seq_cb_t*);
 
         int after_prepare(wsrep::unique_lock<wsrep::mutex>&);
 
-        int before_commit();
+        int before_commit(const wsrep::provider::seq_cb_t*);
 
         int ordered_commit();
 
@@ -248,7 +249,8 @@ namespace wsrep
         bool abort_or_interrupt(wsrep::unique_lock<wsrep::mutex>&);
         int streaming_step(wsrep::unique_lock<wsrep::mutex>&, bool force = false);
         int certify_fragment(wsrep::unique_lock<wsrep::mutex>&);
-        int certify_commit(wsrep::unique_lock<wsrep::mutex>&);
+        int certify_commit(wsrep::unique_lock<wsrep::mutex>&,
+                           const wsrep::provider::seq_cb_t*);
         int append_sr_keys_for_commit();
         int release_commit_order(wsrep::unique_lock<wsrep::mutex>&);
         void remove_fragments_in_storage_service_scope(