Post-review fixes + some bugs fixed + several minor features

server-tools/instance-manager/Makefile.am

@@ -74,7 +74,7 @@ mysqlmanager_SOURCES=	command.cc command.h mysqlmanager.cc \
 			buffer.h buffer.cc parse.cc parse.h \
 			guardian.cc guardian.h \
 			parse_output.cc parse_output.h \
-                        mysql_manager_error.h client_func.c
+                        mysql_manager_error.h
 
 mysqlmanager_LDADD=	liboptions.a \
 			libnet.a \

server-tools/instance-manager/buffer.cc

@@ -40,7 +40,7 @@
 
   RETURN
     0 - ok
-    1 - The buffer came to 16Mb barrier
+    1 - got an error in reserve()
 */
 
 int Buffer::append(uint position, const char *string, uint len_arg)
@@ -71,7 +71,7 @@ int Buffer::append(uint position, const char *string, uint len_arg)
 
   RETURN
     0 - ok
-    1 - The buffer came to 16Mb barrier
+    1 - realloc error or we have come to the 16Mb barrier
 */
 
 int Buffer::reserve(uint position, uint len_arg)
@@ -92,6 +92,18 @@ int Buffer::reserve(uint position, uint len_arg)
   return 0;
 
 err:
+  error= 1;
   return 1;
 }
 
+
+int Buffer::get_size()
+{
+  return buffer_size;
+}
+
+
+int Buffer::is_error()
+{
+  return error;
+}

server-tools/instance-manager/buffer.h

@@ -36,11 +36,17 @@ private:
   /* maximum buffer size is 16Mb */
   enum { MAX_BUFFER_SIZE= 16777216 };
   size_t buffer_size;
+  /* Error flag. Triggered if we get an error of some kind */
+  int error;
 public:
-  Buffer()
+  Buffer(size_t buffer_size_arg= BUFFER_INITIAL_SIZE)
+    :buffer_size(BUFFER_INITIAL_SIZE), error(0)
   {
-    buffer=(char *) malloc(BUFFER_INITIAL_SIZE);
+    /*
-    buffer_size= BUFFER_INITIAL_SIZE;
+      As append() will invokes realloc() anyway, it's ok if malloc returns 0
+    */
+    if (!(buffer= (char*) malloc(buffer_size)))
+        buffer_size= 0;
   }
 
   ~Buffer()
@@ -50,6 +56,8 @@ public:
 
 public:
   char *buffer;
+  int get_size();
+  int is_error();
   int append(uint position, const char *string, uint len_arg);
   int reserve(uint position, uint len_arg);
 };

server-tools/instance-manager/client_func.c

@@ -1,32 +0,0 @@
-#include <my_global.h>
-#include <my_sys.h>
-#include <mysql.h>
-
-/*
-  Currently we cannot use libmysqlclient directly because of the linking
-  issues. Here we provide needed libmysqlclient functions.
-  TODO: to think how to use libmysqlclient code instead of copy&paste.
-  The other possible solution is to use simple_command directly.
-*/
-
-const char * STDCALL
-mysql_get_server_info(MYSQL *mysql)
-{
-  return((char*) mysql->server_version);
-}
-
-int STDCALL
-mysql_ping(MYSQL *mysql)
-{
-  DBUG_ENTER("mysql_ping");
-  DBUG_RETURN(simple_command(mysql,COM_PING,0,0,0));
-}
-
-int STDCALL
-mysql_shutdown(MYSQL *mysql, enum mysql_enum_shutdown_level shutdown_level)
-{
-  uchar level[1];
-  DBUG_ENTER("mysql_shutdown");
-  level[0]= (uchar) shutdown_level;
-  DBUG_RETURN(simple_command(mysql, COM_SHUTDOWN, (char *)level, 1, 0));
-}

server-tools/instance-manager/commands.cc

@@ -184,7 +184,8 @@ int Show_instance_status::do_command(struct st_net *net,
     }
 
 
-    if (my_net_write(net, send_buff.buffer, (uint) position))
+    if (my_net_write(net, send_buff.buffer, (uint) position) ||
+        send_buff.is_error())
       goto err;
   }
 
@@ -270,7 +271,8 @@ int Show_instance_options::do_command(struct st_net *net,
       store_to_string(&send_buff,
                      (char *) instance->options.mysqld_path,
                      &position);
-      if (my_net_write(net, send_buff.buffer, (uint) position))
+      if (my_net_write(net, send_buff.buffer, (uint) position) ||
+          send_buff.is_error())
         goto err;
     }
 
@@ -279,7 +281,8 @@ int Show_instance_options::do_command(struct st_net *net,
       position= 0;
       store_to_string(&send_buff, (char *) "nonguarded", &position);
       store_to_string(&send_buff, "", &position);
-      if (my_net_write(net, send_buff.buffer, (uint) position))
+      if (my_net_write(net, send_buff.buffer, (uint) position) ||
+          send_buff.is_error())
         goto err;
     }
 
@@ -296,7 +299,8 @@ int Show_instance_options::do_command(struct st_net *net,
       store_to_string(&send_buff, option_value + 1, &position);
       /* join name and the value into the same option again */
       *option_value= '=';
-      if (my_net_write(net, send_buff.buffer, (uint) position))
+      if (my_net_write(net, send_buff.buffer, (uint) position) ||
+          send_buff.is_error())
         goto err;
     }
   }

server-tools/instance-manager/guardian.cc

@@ -21,9 +21,32 @@
 
 #include "guardian.h"
 #include "instance_map.h"
+#include "instance.h"
 #include "mysql_manager_error.h"
 #include "log.h"
 #include <string.h>
+#include <sys/types.h>
+#include <signal.h>
+
+
+/*
+  The Guardian list node structure. Guardian utilizes it to store
+  guarded instances plus some additional info.
+*/
+
+struct GUARD_NODE
+{
+  Instance *instance;
+  /* state of an instance (i.e. STARTED, CRASHED, etc.) */
+  int state;
+  /* the amount of attemts to restart instance (cleaned up at success) */
+  int restart_counter;
+  /* triggered at a crash */
+  time_t crash_moment;
+  /* General time field. Used to provide timeouts (at shutdown and restart) */
+  time_t last_checked;
+};
+
 
 C_MODE_START
 
@@ -42,15 +65,13 @@ Guardian_thread::Guardian_thread(Thread_registry &thread_registry_arg,
                                  uint monitoring_interval_arg) :
   Guardian_thread_args(thread_registry_arg, instance_map_arg,
                        monitoring_interval_arg),
-  thread_info(pthread_self())
+  thread_info(pthread_self()), guarded_instances(0)
 {
   pthread_mutex_init(&LOCK_guardian, 0);
   pthread_cond_init(&COND_guardian, 0);
-  shutdown_guardian= FALSE;
+  shutdown_requested= FALSE;
-  is_stopped= FALSE;
+  stopped= FALSE;
   init_alloc_root(&alloc, MEM_ROOT_BLOCK_SIZE, 0);
-  guarded_instances= NULL;
-  starting_instances= NULL;
 }
 
 
@@ -65,19 +86,107 @@ Guardian_thread::~Guardian_thread()
 }
 
 
-void Guardian_thread::shutdown()
+void Guardian_thread::request_shutdown(bool stop_instances_arg)
 {
   pthread_mutex_lock(&LOCK_guardian);
-  shutdown_guardian= TRUE;
+  /* stop instances or just clean up Guardian repository */
+  stop_instances(stop_instances_arg);
+  shutdown_requested= TRUE;
   pthread_mutex_unlock(&LOCK_guardian);
 }
 
 
-void Guardian_thread::request_stop_instances()
+void Guardian_thread::process_instance(Instance *instance,
+                                       GUARD_NODE *current_node,
+                                       LIST **guarded_instances,
+                                       LIST *elem)
 {
-  pthread_mutex_lock(&LOCK_guardian);
+  int waitchild= Instance::DEFAULT_SHUTDOWN_DELAY;
-  request_stop= TRUE;
+  /* The amount of times, Guardian attempts to restart an instance */
-  pthread_mutex_unlock(&LOCK_guardian);
+  int restart_retry= 100;
+  time_t current_time= time(NULL);
+
+  if (current_node->state == STOPPING)
+  {
+    /* this brach is executed during shutdown */
+    if (instance->options.shutdown_delay != NULL)
+      waitchild= atoi(instance->options.shutdown_delay);
+
+    /* this returns true if and only if an instance was stopped for shure */
+    if (instance->is_crashed())
+      *guarded_instances= list_delete(*guarded_instances, elem);
+    else if (current_time - current_node->last_checked > waitchild)
+      {
+        instance->kill_instance(SIGKILL);
+        /*
+          Later we do elem= elem->next. This is ok, as we are only removing
+          the node from the list. The pointer to the next one is still valid.
+        */
+        *guarded_instances= list_delete(*guarded_instances, elem);
+      }
+
+    return;
+  }
+
+  if (instance->is_running())
+  {
+    /* clear status fields */
+    current_node->restart_counter= 0;
+    current_node->crash_moment= 0;
+    current_node->state= STARTED;
+  }
+  else
+  {
+    switch (current_node->state)
+    {
+    case NOT_STARTED:
+      instance->start();
+      current_node->last_checked= current_time;
+      log_info("guardian: starting instance %s",
+               instance->options.instance_name);
+      current_node->state= STARTING;
+      break;
+    case STARTED:     /* fallthrough */
+    case STARTING:    /* let the instance start or crash */
+      if (instance->is_crashed())
+      {
+        current_node->crash_moment= current_time;
+        current_node->last_checked= current_time;
+        current_node->state= JUST_CRASHED;
+        /* fallthrough -- restart an instance immediately */
+      }
+      else
+        break;
+    case JUST_CRASHED:
+      if (current_time - current_node->crash_moment <= 2)
+      {
+        instance->start();
+        log_info("guardian: starting instance %s",
+                 instance->options.instance_name);
+      }
+      else current_node->state= CRASHED;
+      break;
+    case CRASHED:    /* just regular restarts */
+      if (current_time - current_node->last_checked >
+          monitoring_interval)
+      {
+        if ((current_node->restart_counter < restart_retry))
+        {
+          instance->start();
+          current_node->last_checked= current_time;
+          ((GUARD_NODE *) elem->data)->restart_counter++;
+          log_info("guardian: starting instance %s",
+                   instance->options.instance_name);
+        }
+        else current_node->state= CRASHED_AND_ABANDONED;
+      }
+      break;
+    case CRASHED_AND_ABANDONED:
+      break; /* do nothing */
+    default:
+      DBUG_ASSERT(0);
+    }
+  }
 }
 
 
@@ -96,8 +205,7 @@ void Guardian_thread::request_stop_instances()
 void Guardian_thread::run()
 {
   Instance *instance;
-  int restart_retry= 100;
+  LIST *elem;
-  LIST *loop;
   struct timespec timeout;
 
   thread_registry.register_thread(&thread_info);
@@ -105,68 +213,31 @@ void Guardian_thread::run()
   my_thread_init();
   pthread_mutex_lock(&LOCK_guardian);
 
+  /* loop, until all instances were shut down at the end */
+  while (!(shutdown_requested && (guarded_instances == NULL)))
+  {
+    elem= guarded_instances;
 
-  while (!shutdown_guardian)
+    while (elem != NULL)
     {
-    int status= 0;
+      struct timespec timeout;
-    loop= guarded_instances;
 
-    while (loop != NULL)
+      GUARD_NODE *current_node= (GUARD_NODE *) elem->data;
-    {
+      instance= ((GUARD_NODE *) elem->data)->instance;
-      instance= ((GUARD_NODE *) loop->data)->instance;
+      process_instance(instance, current_node, &guarded_instances, elem);
-      if (!instance->is_running())
-      {
-        int state= 0;                           /* state of guardian */
 
-        if ((((GUARD_NODE *) loop->data)->crash_moment == 0))
+      elem= elem->next;
-          state= 1;                             /* an instance just crashed */
-        else
-           if (time(NULL) - ((GUARD_NODE *) loop->data)->crash_moment <= 2)
-             /* try to restart an instance immediately */
-             state= 2;
-           else
-             state= 3;                          /* try to restart it */
-
-        if (state == 1)
-          ((GUARD_NODE *) loop->data)->crash_moment= time(NULL);
-
-        if ((state == 1) || (state == 2))
-        {
-          instance->start();
-          ((GUARD_NODE *) loop->data)->restart_counter++;
-          log_info("guardian: starting instance %s",
-                   instance->options.instance_name);
     }
-        else
-        {
-          if ((status == ETIMEDOUT) &&
-             (((GUARD_NODE *) loop->data)->restart_counter < restart_retry))
-          {
-           instance->start();
-           ((GUARD_NODE *) loop->data)->restart_counter++;
-           log_info("guardian: starting instance %s",
-                    instance->options.instance_name);
-          }
-        }
-      }
-      else /* clear status fields */
-      {
-        ((GUARD_NODE *) loop->data)->restart_counter= 0;
-        ((GUARD_NODE *) loop->data)->crash_moment= 0;
-      }
-      loop= loop->next;
-    }
-    move_to_list(&starting_instances, &guarded_instances);
     timeout.tv_sec= time(NULL) + monitoring_interval;
     timeout.tv_nsec= 0;
 
-    status= pthread_cond_timedwait(&COND_guardian, &LOCK_guardian, &timeout);
+    /* check the loop predicate before sleeping */
+    if (!(shutdown_requested && (guarded_instances == NULL)))
+      pthread_cond_timedwait(&COND_guardian, &LOCK_guardian, &timeout);
   }
 
+  stopped= TRUE;
   pthread_mutex_unlock(&LOCK_guardian);
-  if (request_stop)
-    stop_instances();
-  is_stopped= TRUE;
   /* now, when the Guardian is stopped we can stop the IM */
   thread_registry.unregister_thread(&thread_info);
   thread_registry.request_shutdown();
@@ -174,7 +245,29 @@ void Guardian_thread::run()
 }
 
 
-int Guardian_thread::start()
+int Guardian_thread::is_stopped()
+{
+  int var;
+  pthread_mutex_lock(&LOCK_guardian);
+  var= stopped;
+  pthread_mutex_unlock(&LOCK_guardian);
+  return var;
+}
+
+
+/*
+  Initialize the list of guarded instances: loop through the Instance_map and
+  add all of the instances, which don't have 'nonguarded' option specified.
+
+  SYNOPSYS
+    Guardian_thread::init()
+
+  RETURN
+    0 - ok
+    1 - error occured
+*/
+
+int Guardian_thread::init()
 {
   Instance *instance;
   Instance_map::Iterator iterator(instance_map);
@@ -183,7 +276,7 @@ int Guardian_thread::start()
   while ((instance= iterator.next()))
   {
     if ((instance->options.nonguarded == NULL))
-      if (add_instance_to_list(instance, &guarded_instances))
+      if (guard(instance))
         return 1;
   }
   instance_map->unlock();
@@ -193,7 +286,7 @@ int Guardian_thread::start()
 
 
 /*
-  Start instance guarding
+  Add instance to the Guardian list
 
   SYNOPSYS
     guard()
@@ -201,36 +294,15 @@ int Guardian_thread::start()
 
   DESCRIPTION
 
-    The instance is added to the list of starting instances. Then after one guardian
+    The instance is added to the guarded instances list. Usually guard() is
-    loop it is moved to the guarded instances list. Usually guard() is called after we
+    called after we start an instance.
-    start an instance, so we need to give some time to the instance to start.
 
   RETURN
     0 - ok
     1 - error occured
 */
 
-
 int Guardian_thread::guard(Instance *instance)
-{
-  return add_instance_to_list(instance, &starting_instances);
-}
-
-
-void Guardian_thread::move_to_list(LIST **from, LIST **to)
-{
-  LIST *tmp;
-
-  while (*from)
-  {
-    tmp= rest(*from);
-    *to= list_add(*to, *from);
-    *from= tmp;
-  }
-}
-
-
-int Guardian_thread::add_instance_to_list(Instance *instance, LIST **list)
 {
   LIST *node;
   GUARD_NODE *content;
@@ -244,10 +316,11 @@ int Guardian_thread::add_instance_to_list(Instance *instance, LIST **list)
   content->instance= instance;
   content->restart_counter= 0;
   content->crash_moment= 0;
+  content->state= NOT_STARTED;
   node->data= (void *) content;
 
   pthread_mutex_lock(&LOCK_guardian);
-  *list= list_add(*list, node);
+  guarded_instances= list_add(guarded_instances, node);
   pthread_mutex_unlock(&LOCK_guardian);
 
   return 0;
@@ -256,7 +329,7 @@ int Guardian_thread::add_instance_to_list(Instance *instance, LIST **list)
 
 /*
   TODO: perhaps it would make sense to create a pool of the LIST elements
-  elements and give them upon request. Now we are loosing a bit of memory when
+  and give them upon request. Now we are loosing a bit of memory when
   guarded instance was stopped and then restarted (since we cannot free just
   a piece of the MEM_ROOT).
 */
@@ -288,21 +361,61 @@ int Guardian_thread::stop_guard(Instance *instance)
   return 0;
 }
 
-int Guardian_thread::stop_instances()
+/*
-{
+  Start Guardian shutdown. Attempt to start instances if requested.
-  Instance *instance;
-  Instance_map::Iterator iterator(instance_map);
 
-  while ((instance= iterator.next()))
+  SYNOPSYS
+    stop_instances()
+    stop_instances_arg          whether we should stop instances at shutdown
+
+  DESCRIPTION
+
+    Loops through the guarded_instances list and prepares them for shutdown.
+    If stop_instances was requested, we need to issue a stop command and change
+    the state accordingly. Otherwise we could simply delete an entry.
+    NOTE: Guardian should be locked by the calling function
+
+  RETURN
+    0 - ok
+    1 - error occured
+*/
+
+int Guardian_thread::stop_instances(bool stop_instances_arg)
 {
-    if ((instance->options.nonguarded == NULL))
+  LIST *node;
+  node= guarded_instances;
+  while (node != NULL)
   {
-      if (stop_guard(instance))
+    if (!stop_instances_arg)
-        return 1;
+    {
-      /* let us try to stop the server */
+      /* just forget about an instance */
-      instance->stop();
+      guarded_instances= list_delete(guarded_instances, node);
+      /*
+        This should still work fine, as we have only removed the
+        node from the list. The pointer to the next one is still valid
+      */
+      node= node->next;
+    }
+    else
+    {
+      GUARD_NODE *current_node= (GUARD_NODE *) node->data;
+      /*
+        If instance is running or was running (and now probably hanging),
+        request stop.
+      */
+      if (current_node->instance->is_running() ||
+          (current_node->state == STARTED))
+      {
+        current_node->state= STOPPING;
+        current_node->last_checked= time(NULL);
+      }
+      else
+        /* otherwise remove it from the list */
+        guarded_instances= list_delete(guarded_instances, node);
+      /* But try to kill it anyway. Just in case */
+      current_node->instance->kill_instance(SIGTERM);
+      node= node->next;
     }
   }
-
   return 0;
 }

server-tools/instance-manager/guardian.h

@@ -19,6 +19,7 @@
 #include <my_global.h>
 #include <my_sys.h>
 #include <my_list.h>
+#include "thread_registry.h"
 
 #ifdef __GNUC__
 #pragma interface
@@ -26,9 +27,8 @@
 
 class Instance;
 class Instance_map;
-
+class Thread_registry;
-#include "thread_registry.h"
+struct GUARD_NODE;
-#include "instance.h"
 
 C_MODE_START
 
@@ -36,19 +36,11 @@ pthread_handler_decl(guardian, arg);
 
 C_MODE_END
 
-typedef struct st_guard_node
-{
-  Instance *instance;
-  uint restart_counter;
-  time_t crash_moment;
-} GUARD_NODE;
-
-
 struct Guardian_thread_args
 {
   Thread_registry &thread_registry;
   Instance_map *instance_map;
-  uint monitoring_interval;
+  int monitoring_interval;
 
   Guardian_thread_args(Thread_registry &thread_registry_arg,
                        Instance_map *instance_map_arg,
@@ -72,36 +64,41 @@ public:
                   Instance_map *instance_map_arg,
                   uint monitoring_interval_arg);
   ~Guardian_thread();
+  /* Main funtion of the thread */
   void run();
-  int start();
+  /* Initialize list of guarded instances */
-  void shutdown();
+  int init();
-  void request_stop_instances();
+  /* Request guardian shutdown. Stop instances if needed */
+  void request_shutdown(bool stop_instances);
+  /* Start instance protection */
   int guard(Instance *instance);
+  /* Stop instance protection */
   int stop_guard(Instance *instance);
-  bool is_stopped;
+  /* Returns true if guardian thread is stopped */
+  int is_stopped();
 
 public:
   pthread_cond_t COND_guardian;
 
 private:
-  int stop_instances();
+  /* Prepares Guardian shutdown. Stops instances is needed */
-  int add_instance_to_list(Instance *instance, LIST **list);
+  int stop_instances(bool stop_instances_arg);
-  void move_to_list(LIST **from, LIST **to);
+  /* check instance state and act accordingly */
+  void process_instance(Instance *instance, GUARD_NODE *current_node,
+                        LIST **guarded_instances, LIST *elem);
+  int stopped;
 
 private:
+  /* states of an instance */
+  enum { NOT_STARTED= 1, STARTING, STARTED, JUST_CRASHED, CRASHED,
+         CRASHED_AND_ABANDONED, STOPPING };
   pthread_mutex_t LOCK_guardian;
   Thread_info thread_info;
   LIST *guarded_instances;
-  LIST *starting_instances;
   MEM_ROOT alloc;
   enum { MEM_ROOT_BLOCK_SIZE= 512 };
   /* this variable is set to TRUE when we want to stop Guardian thread */
-  bool shutdown_guardian;
+  bool shutdown_requested;
-  /*
-    This var is usually set together with shutdown_guardian. this way we
-    request guardian to shut down all instances before termination
-  */
-  bool request_stop;
 };
 
 #endif /* INCLUDES_MYSQL_INSTANCE_MANAGER_GUARDIAN_H */

server-tools/instance-manager/instance.cc

@@ -27,6 +27,19 @@
 #include <m_string.h>
 #include <sys/wait.h>
 
+
+C_MODE_START
+
+pthread_handler_decl(proxy, arg)
+{
+  Instance *instance= (Instance *) arg;
+  instance->fork_and_monitor();
+  return 0;
+}
+
+C_MODE_END
+
+
 /*
   The method starts an instance.
 
@@ -44,6 +57,12 @@ int Instance::start()
 {
   pid_t pid;
 
+  /* clear crash flag */
+  pthread_mutex_lock(&LOCK_instance);
+  crashed= 0;
+  pthread_mutex_unlock(&LOCK_instance);
+
+
   if (!is_running())
   {
     if ((pid= options.get_pid()) != 0)          /* check the pidfile */
@@ -52,17 +71,26 @@ int Instance::start()
                   since IM lacks permmissions or hasn't found the pidifle",
                   options.instance_name);
 
-    log_info("starting instance %s", options.instance_name);
+    /*
-    switch (pid= fork()) {
+      No need to monitor this thread in the Thread_registry, as all
-    case 0:
+      instances are to be stopped during shutdown.
-      execv(options.mysqld_path, options.argv);
+    */
-      /* exec never returns */
+    pthread_t proxy_thd_id;
-      exit(1);
+    pthread_attr_t proxy_thd_attr;
-    case -1:
+    int rc;
+
+    pthread_attr_init(&proxy_thd_attr);
+    pthread_attr_setdetachstate(&proxy_thd_attr, PTHREAD_CREATE_DETACHED);
+    rc= pthread_create(&proxy_thd_id, &proxy_thd_attr, proxy,
+                       this);
+    pthread_attr_destroy(&proxy_thd_attr);
+    if (rc)
+    {
+      log_error("Instance::start(): pthread_create(proxy) failed");
       return ER_CANNOT_START_INSTANCE;
-    default:
-      return 0;
     }
+
+    return 0;
   }
 
   /* the instance is started already */
@@ -70,9 +98,62 @@ int Instance::start()
 }
 
 
+void Instance::fork_and_monitor()
+{
+  pid_t pid;
+  log_info("starting instance %s", options.instance_name);
+  switch (pid= fork()) {
+  case 0:
+    execv(options.mysqld_path, options.argv);
+    /* exec never returns */
+    exit(1);
+  case -1:
+    log_info("cannot fork() to start instance %s", options.instance_name);
+    return;
+  default:
+    wait(NULL);
+    /* set instance state to crashed */
+    pthread_mutex_lock(&LOCK_instance);
+    crashed= 1;
+    pthread_mutex_unlock(&LOCK_instance);
+
+    /*
+      Wake connection threads waiting for an instance to stop. This
+      is needed if a user issued command to stop an instance via
+      mysql connection. This is not the case if Guardian stop the thread.
+    */
+    pthread_cond_signal(&COND_instance_restarted);
+    /* wake guardian */
+    pthread_cond_signal(&instance_map->guardian->COND_guardian);
+    /* thread exits */
+    return;
+  }
+  /* we should never end up here */
+  DBUG_ASSERT(0);
+}
+
+
+Instance::Instance(): crashed(0)
+{
+  pthread_mutex_init(&LOCK_instance, 0);
+  pthread_cond_init(&COND_instance_restarted, 0);
+}
+
+
 Instance::~Instance()
 {
   pthread_mutex_destroy(&LOCK_instance);
+  pthread_cond_destroy(&COND_instance_restarted);
+}
+
+
+int Instance::is_crashed()
+{
+  int val;
+  pthread_mutex_lock(&LOCK_instance);
+  val= crashed;
+  pthread_mutex_unlock(&LOCK_instance);
+  return val;
 }
 
 
@@ -95,20 +176,19 @@ bool Instance::is_running()
   pthread_mutex_lock(&LOCK_instance);
 
   mysql_init(&mysql);
-  /* try to connect to a server with the fake username/password pair */
+  /* try to connect to a server with a fake username/password pair */
   if (mysql_real_connect(&mysql, LOCAL_HOST, username,
                          password,
                          NullS, port,
                          socket, 0))
   {
     /*
-      Very strange. We have successfully connected to the server using
+      We have successfully connected to the server using fake
-      bullshit as username/password. Write a warning to the logfile.
+      username/password. Write a warning to the logfile.
     */
     log_info("The Instance Manager was able to log into you server \
              with faked compiled-in password while checking server status. \
              Looks like something is wrong.");
-    mysql_close(&mysql);
     pthread_mutex_unlock(&LOCK_instance);
     return_val= TRUE;                           /* server is alive */
   }
@@ -151,53 +231,30 @@ int Instance::stop()
   if (options.shutdown_delay != NULL)
     waitchild= atoi(options.shutdown_delay);
 
-  if ((pid= options.get_pid()) != 0)            /* get pid from pidfile */
+  kill_instance(SIGTERM);
-  {
-    /*
-      If we cannot kill mysqld, then it has propably crashed.
-      Let us try to remove staled pidfile and return succes as mysqld
-      is probably stopped
-    */
-    if (kill(pid, SIGTERM))
-    {
-      if (options.unlink_pidfile())
-        log_error("cannot remove pidfile for instance %i, this might be \
-                  since IM lacks permmissions or hasn't found the pidifle",
-                  options.instance_name);
-
-      return 0;
-    }
-
   /* sleep on condition to wait for SIGCHLD */
 
   timeout.tv_sec= time(NULL) + waitchild;
   timeout.tv_nsec= 0;
-    if (pthread_mutex_lock(&instance_map->pid_cond.LOCK_pid))
+  if (pthread_mutex_lock(&LOCK_instance))
-      goto err; /* perhaps this should be procecced differently */
+    goto err;
 
   while (options.get_pid() != 0)              /* while server isn't stopped */
   {
     int status;
 
-      status= pthread_cond_timedwait(&instance_map->pid_cond.COND_pid,
+    status= pthread_cond_timedwait(&COND_instance_restarted,
-                                     &instance_map->pid_cond.LOCK_pid,
+                                   &LOCK_instance,
                                    &timeout);
     if (status == ETIMEDOUT)
       break;
   }
 
-    pthread_mutex_unlock(&instance_map->pid_cond.LOCK_pid);
+  pthread_mutex_unlock(&LOCK_instance);
 
-    if (!kill(pid, SIGKILL))
+  kill_instance(SIGKILL);
-    {
-      log_error("The instance %s has been stopped forsibly. Normally \
-                it should not happed. Probably the instance has been \
-                hanging. You should also check your IM setup",
-                options.instance_name);
-    }
 
   return 0;
-  }
 
   return ER_INSTANCE_IS_NOT_STARTED;
 err:
@@ -205,6 +262,29 @@ err:
 }
 
 
+void Instance::kill_instance(int signum)
+{
+  pid_t pid;
+  /* if there are no pid, everything seems to be fine */
+  if ((pid= options.get_pid()) != 0)            /* get pid from pidfile */
+  {
+    /*
+      If we cannot kill mysqld, then it has propably crashed.
+      Let us try to remove staled pidfile and return successfully
+      as mysqld is probably stopped.
+    */
+    if (!kill(pid, signum))
+      options.unlink_pidfile();
+    else
+      if (signum == SIGKILL)      /* really killed instance with SIGKILL */
+        log_error("The instance %s is being stopped forsibly. Normally \
+                  it should not happed. Probably the instance has been \
+                  hanging. You should also check your IM setup",
+                  options.instance_name);
+  }
+  return;
+}
+
 /*
   We execute this function to initialize instance parameters.
   Return value: 0 - ok. 1 - unable to init DYNAMIC_ARRAY.
@@ -212,8 +292,6 @@ err:
 
 int Instance::init(const char *name_arg)
 {
-  pthread_mutex_init(&LOCK_instance, 0);
-
   return options.init(name_arg);
 }
 

server-tools/instance-manager/instance.h

@@ -30,14 +30,19 @@ class Instance_map;
 class Instance
 {
 public:
+  Instance();
+
   ~Instance();
   int init(const char *name);
   int complete_initialization(Instance_map *instance_map_arg);
 
-  /* check if the instance is running and set up mysql connection if yes */
   bool is_running();
   int start();
   int stop();
+  /* send a signal to the instance */
+  void kill_instance(int signo);
+  int is_crashed();
+  void fork_and_monitor();
 
 public:
   enum { DEFAULT_SHUTDOWN_DELAY= 35 };
@@ -49,7 +54,9 @@ private:
     double start of the instance. This happens when the instance is starting
     and we issue the start command once more.
   */
+  int crashed;
   pthread_mutex_t LOCK_instance;
+  pthread_cond_t COND_instance_restarted;
   Instance_map *instance_map;
 };
 

server-tools/instance-manager/instance_map.cc

@@ -120,9 +120,6 @@ Instance_map::Instance_map(const char *default_mysqld_path_arg)
 
 int Instance_map::init()
 {
-  pthread_mutex_init(&pid_cond.LOCK_pid, 0);
-  pthread_cond_init(&pid_cond.COND_pid, 0);
-
   if (hash_init(&hash, default_charset_info, START_HASH_SIZE, 0, 0,
                 get_instance_key, delete_instance, 0))
     return 1;
@@ -135,8 +132,6 @@ Instance_map::~Instance_map()
   hash_free(&hash);
   pthread_mutex_unlock(&LOCK_instance_map);
   pthread_mutex_destroy(&LOCK_instance_map);
-  pthread_mutex_destroy(&pid_cond.LOCK_pid);
-  pthread_cond_destroy(&pid_cond.COND_pid);
 }
 
 
@@ -198,21 +193,6 @@ void Instance_map::complete_initialization()
 }
 
 
-Instance *
-Instance_map::find(uint instance_number)
-{
-  Instance *instance;
-  char name[80];
-
-  snprintf(name, sizeof(name) - 1, "mysqld%i", instance_number);
-  name[sizeof(name) - 1]= 0; /* safety */
-  pthread_mutex_lock(&LOCK_instance_map);
-  instance= (Instance *) hash_search(&hash, (byte *) name, strlen(name));
-  pthread_mutex_unlock(&LOCK_instance_map);
-  return instance;
-}
-
-
 /* load options from config files and create appropriate instance structures */
 
 int Instance_map::load()

server-tools/instance-manager/instance_map.h

@@ -27,12 +27,6 @@
 #include "protocol.h"
 #include "guardian.h"
 
-typedef struct st_instance_cond
-{
-  pthread_mutex_t LOCK_pid;
-  pthread_cond_t COND_pid;
-} CHILD_COND;
-
 class Instance;
 extern int load_all_groups(char ***groups, const char *filename);
 extern void free_groups(char **groups);
@@ -83,8 +77,6 @@ public:
 public:
   const char *mysqld_path;
   Guardian_thread *guardian;
-  /* structure used for syncronization reasons in the stop command */
-  CHILD_COND pid_cond;
 
 private:
   enum { START_HASH_SIZE = 16 };

server-tools/instance-manager/instance_options.cc

@@ -27,25 +27,50 @@
 #include <m_string.h>
 
 
-/* option_name should be prefixed with "--" */
+/*
-int Instance_options::get_default_option(char *result, const char *option_name,
+  Get compiled-in value of default_option
-                                         size_t result_len)
+
+  SYNOPSYS
+    get_default_option()
+    result            buffer to put found value
+    result_len        buffer size
+    oprion_name       the name of the option, prefixed with "--"
+
+  DESCRIPTION
+
+   Get compile-in value of requested option from server
+
+  RETURN
+    0 - ok
+    1 - error occured
+*/
+
+int Instance_options::get_default_option(char *result, size_t result_len,
+                                         const char *option_name)
 {
   int position= 0;
+  int rc= 1;
   char verbose_option[]= " --no-defaults --verbose --help";
-  Buffer cmd;
 
+  Buffer cmd(strlen(mysqld_path)+sizeof(verbose_option)+1);
+  if (cmd.get_size()) /* malloc succeeded */
+  {
     cmd.append(position, mysqld_path, strlen(mysqld_path));
     position+=  strlen(mysqld_path);
     cmd.append(position, verbose_option, sizeof(verbose_option) - 1);
     position+= sizeof(verbose_option) - 1;
     cmd.append(position, "\0", 1);
-  /* get the value from "mysqld --help --verbose" */
-  if (parse_output_and_get_value(cmd.buffer, option_name + 2,
-                                 result, result_len))
-    return 1;
 
-  return 0;
+    if (cmd.is_error())
+      goto err;
+    /* get the value from "mysqld --help --verbose" */
+    rc= parse_output_and_get_value(cmd.buffer, option_name + 2,
+                                   result, result_len);
+  }
+
+  return rc;
+err:
+  return 1;
 }
 
 
@@ -56,50 +81,32 @@ void Instance_options::get_pid_filename(char *result)
 
   if (mysqld_datadir == NULL)
   {
-    get_default_option(datadir, "--datadir", MAX_PATH_LEN);
+    get_default_option(datadir, sizeof(datadir), "--datadir");
   }
   else
     strxnmov(datadir, MAX_PATH_LEN - 1, strchr(mysqld_datadir, '=') + 1,
              "/", NullS);
 
-  /* well, we should never get it */
+  DBUG_ASSERT(mysqld_pid_file);
-  if (mysqld_pid_file != NULL)
   pid_file= strchr(pid_file, '=') + 1;
-  else
-    DBUG_ASSERT(0);
 
   /* get the full path to the pidfile */
   my_load_path(result, pid_file, datadir);
-
 }
 
 
 int Instance_options::unlink_pidfile()
 {
-  char pid_file_path[MAX_PATH_LEN];
+  return unlink(pid_file_with_path);
-
-  /*
-    This works as we know that pid_file_path is of
-    MAX_PATH_LEN == FN_REFLEN length
-  */
-  get_pid_filename((char *)&pid_file_path);
-
-  return unlink(pid_file_path);
 }
 
 
 pid_t Instance_options::get_pid()
 {
-  char pid_file_path[MAX_PATH_LEN];
+  FILE *pid_file_stream;
-
-  /*
-    This works as we know that pid_file_path is of
-    MAX_PATH_LEN == FN_REFLEN length
-  */
-  get_pid_filename((char *)&pid_file_path);
 
   /* get the pid */
-  if (FILE *pid_file_stream= my_fopen(pid_file_path,
+  if (pid_file_stream= my_fopen(pid_file_with_path,
                                 O_RDONLY | O_BINARY, MYF(0)))
   {
     pid_t pid;
@@ -140,6 +147,8 @@ int Instance_options::complete_initialization(const char *default_path)
     add_option(pidfilename);
   }
 
+  get_pid_filename(pid_file_with_path);
+
   /* we need to reserve space for the final zero + possible default options */
   if (!(argv= (char**) alloc_root(&alloc, (options_array.elements + 1
                        + MAX_NUMBER_OF_DEFAULT_OPTIONS) * sizeof(char*))))
@@ -244,6 +253,8 @@ int Instance_options::add_to_argv(const char* option)
   return 0;
 }
 
+
+/* function for debug purposes */
 void Instance_options::print_argv()
 {
   int i;

server-tools/instance-manager/instance_options.h

@@ -39,7 +39,7 @@ public:
   Instance_options() :
     mysqld_socket(0), mysqld_datadir(0), mysqld_bind_address(0),
     mysqld_pid_file(0), mysqld_port(0), mysqld_path(0), nonguarded(0),
-    filled_default_options(0)
+    shutdown_delay(0), filled_default_options(0)
   {}
   ~Instance_options();
   /* fills in argv */
@@ -60,6 +60,7 @@ public:
   enum { MAX_PATH_LEN= 512 };
   enum { MAX_NUMBER_OF_DEFAULT_OPTIONS= 2 };
   enum { MEM_ROOT_BLOCK_SIZE= 512 };
+  char pid_file_with_path[MAX_PATH_LEN];
   char **argv;
   /* We need the some options, so we store them as a separate pointers */
   const char *mysqld_socket;
@@ -72,11 +73,12 @@ public:
   const char *mysqld_path;
   const char *nonguarded;
   const char *shutdown_delay;
+  /* this value is computed and cashed here */
   DYNAMIC_ARRAY options_array;
 private:
   int add_to_argv(const char *option);
-  int get_default_option(char *result, const char *option_name,
+  int get_default_option(char *result, size_t result_len,
-                           size_t result_len);
+                         const char *option_name);
 private:
   uint filled_default_options;
   MEM_ROOT alloc;

server-tools/instance-manager/listener.cc

@@ -85,6 +85,8 @@ void Listener_thread::run()
 
   thread_registry.register_thread(&thread_info);
 
+  my_thread_init();
+
   /* I. prepare 'listen' sockets */
 
   int ip_socket= socket(AF_INET, SOCK_STREAM, 0);
@@ -263,11 +265,13 @@ void Listener_thread::run()
   unlink(unix_socket_address.sun_path);
 
   thread_registry.unregister_thread(&thread_info);
+  my_thread_end();
   return;
 
 err:
   thread_registry.unregister_thread(&thread_info);
   thread_registry.request_shutdown();
+  my_thread_end();
   return;
 }
 

server-tools/instance-manager/manager.cc

@@ -90,7 +90,6 @@ void manager(const Options &options)
   sigemptyset(&mask);
   sigaddset(&mask, SIGINT);
   sigaddset(&mask, SIGTERM);
-  sigaddset(&mask, SIGCHLD);
   sigaddset(&mask, SIGPIPE);
   sigaddset(&mask, SIGHUP);
   /*
@@ -128,7 +127,7 @@ void manager(const Options &options)
     int rc;
 
     /*
-       NOTE: Guardian should be shutdowned first. Only then all other threads
+       NOTE: Guardian should be shutdown first. Only then all other threads
        need to be stopped. This should be done, as guardian is responsible for
        shutting down the instances, and this is a long operation.
     */
@@ -160,12 +159,8 @@ void manager(const Options &options)
      more then 10 alarms at the same time.
   */
   init_thr_alarm(10);
-  /*
+  /* init list of guarded instances */
-    Now we can init the list of guarded instances. We have to do it after
+  guardian_thread.init();
-    alarm structures initialization as we have to use net_* functions while
-    making the list. And they in their turn need alarms for timeout suppport.
-  */
-  guardian_thread.start();
   /*
     After the list of guarded instances have been initialized,
     Guardian should start them.
@@ -182,18 +177,12 @@ void manager(const Options &options)
       case THR_SERVER_ALARM:
         process_alarm(signo);
       break;
-      case SIGCHLD:
-        wait(NULL);
-        /* wake threads waiting for an instance to shutdown */
-        pthread_cond_broadcast(&instance_map.pid_cond.COND_pid);
-        /* wake guardian */
-        pthread_cond_signal(&guardian_thread.COND_guardian);
-      break;
       default:
-        if (!guardian_thread.is_stopped)
       {
-          guardian_thread.request_stop_instances();
+        if (!guardian_thread.is_stopped())
-          guardian_thread.shutdown();
+        {
+          bool stop_instances= true;
+          guardian_thread.request_shutdown(stop_instances);
           pthread_cond_signal(&guardian_thread.COND_guardian);
         }
         else
@@ -201,6 +190,7 @@ void manager(const Options &options)
           thread_registry.deliver_shutdown();
           shutdown_complete= TRUE;
         }
+      }
       break;
     }
   }

server-tools/instance-manager/options.cc

@@ -23,6 +23,8 @@
 #include <my_global.h>
 #include <my_sys.h>
 #include <my_getopt.h>
+#include <m_string.h>
+#include <mysql_com.h>
 
 #include "priv.h"
 
@@ -77,6 +79,9 @@ static struct my_option my_long_options[] =
     (gptr *) &Options::socket_file_name, (gptr *) &Options::socket_file_name,
     0, GET_STR, REQUIRED_ARG, 0, 0, 0, 0, 0, 0 },
 
+  { "passwd", 'P', "Prepare entry for passwd file and exit.", 0, 0, 0,
+    GET_NO_ARG, NO_ARG, 0, 0, 0, 0, 0, 0 },
+
   { "bind-address", OPT_BIND_ADDRESS, "Bind address to use for connection.",
     (gptr *) &Options::bind_address, (gptr *) &Options::bind_address,
     0, GET_STR, REQUIRED_ARG, 0, 0, 0, 0, 0, 0 },
@@ -142,6 +147,34 @@ static void usage()
   my_print_variables(my_long_options);
 }
 
+
+static void passwd()
+{
+  char user[1024], pw[1024], *p;
+  char crypted_pw[SCRAMBLED_PASSWORD_CHAR_LENGTH + 1];
+
+  fprintf(stderr, "Creating record for new user.\n");
+  fprintf(stderr, "Enter user name: ");
+  if (!fgets(user, sizeof(user), stdin))
+  {
+    fprintf(stderr, "Unable to read user.\n");
+    return;
+  }
+  if ((p= strchr(user, '\n'))) *p= 0;
+
+  fprintf(stderr, "Enter password: ");
+  if (! fgets(pw, sizeof(pw), stdin))
+  {
+    fprintf(stderr, "Unable to read password.\n");
+    return;
+  }
+  if ((p= strchr(pw, '\n'))) *p= 0;
+
+  make_scrambled_password(crypted_pw, pw);
+  printf("%s:%s\n", user, crypted_pw);
+}
+
+
 C_MODE_START
 
 static my_bool
@@ -153,7 +186,9 @@ get_one_option(int optid,
   case 'V':
     version();
     exit(0);
-  case 'I':
+  case 'P':
+    passwd();
+    exit(0);
   case '?':
     usage();
     exit(0);

server-tools/instance-manager/parse.cc

@@ -49,29 +49,6 @@ static struct tokens_st tokens[]= {
 };
 
 
-/*
-  tries to find next word in the text
-  if found, returns the beginning and puts word length to word_len argument.
-  if not found returns pointer to first non-space or to '\0', word_len == 0
-*/
-
-inline void get_word(const char **text, uint *word_len)
-{
-  const char *word_end;
-
-  /* skip space */
-  while (my_isspace(default_charset_info, **text))
-    ++(*text);
-
-  word_end= *text;
-
-  while (my_isalnum(default_charset_info, *word_end))
-    ++word_end;
-
-  *word_len= word_end - *text;
-}
-
-
 /*
   Returns token no if word corresponds to some token, otherwise returns
   TOK_NOT_FOUND

server-tools/instance-manager/parse.h

@@ -20,4 +20,34 @@
 
 Command *parse_command(Command_factory *factory, const char  *text);
 
+/* define kinds of the word seek method */
+enum { ALPHANUM= 1, NONSPACE };
+
+/*
+  tries to find next word in the text
+  if found, returns the beginning and puts word length to word_len argument.
+  if not found returns pointer to first non-space or to '\0', word_len == 0
+*/
+
+inline void get_word(const char **text, uint *word_len,
+                     int seek_method= ALPHANUM)
+{
+  const char *word_end;
+
+  /* skip space */
+  while (my_isspace(default_charset_info, **text))
+    ++(*text);
+
+  word_end= *text;
+
+  if (seek_method == ALPHANUM)
+    while (my_isalnum(default_charset_info, *word_end))
+      ++word_end;
+  else
+    while (!my_isspace(default_charset_info, *word_end))
+      ++word_end;
+
+  *word_len= word_end - *text;
+}
+
 #endif /* INCLUDES_MYSQL_INSTANCE_MANAGER_PARSE_H */

server-tools/instance-manager/parse_output.cc

@@ -14,43 +14,38 @@
    along with this program; if not, write to the Free Software
    Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA */
 
+#include "parse.h"
+
 #include <stdio.h>
 #include <my_global.h>
 #include <my_sys.h>
 #include <string.h>
 
-/* buf should be of appropriate size. Otherwise the word will be truncated */
-static int get_word(FILE *file, char *buf, size_t size)
-{
-  int currchar;
 
-  currchar= getc(file);
+/*
+  Parse output of the given command
 
-  /* skip space */
+  SYNOPSYS
-  while (my_isspace(default_charset_info, (char) currchar) &&
+    parse_output_and_get_value()
-         currchar != EOF && size > 1)
-  {
-    currchar= getc(file);
-  }
 
-  while (!my_isspace(default_charset_info, (char) currchar) &&
+    command      the command to execue with popen.
-         currchar != EOF && size > 1)
+    word         the word to look for (usually an option name)
-  {
+    result       the buffer to store the next word (option value)
-    *buf++= (char) currchar;
+    result_len   self-explanatory
-    currchar= getc(file);
-    size--;
-  }
 
-  *buf= '\0';
+  DESCRIPTION
-  return 0;
-}
 
+    Parse output of the "command". Find the "word" and return the next one
+*/
 
 int parse_output_and_get_value(const char *command, const char *word,
                                char *result, size_t result_len)
 {
   FILE *output;
-  int wordlen;
+  uint wordlen;
+  /* should be enought to store the string from the output */
+  enum { MAX_LINE_LEN= 512 };
+  char linebuf[MAX_LINE_LEN];
 
   wordlen= strlen(word);
 
@@ -62,19 +57,32 @@ int parse_output_and_get_value(const char *command, const char *word,
   */
   setvbuf(output, NULL, _IOFBF, 0);
 
-  get_word(output, result, result_len);
+  while (fgets(linebuf, sizeof(linebuf) - 1, output))
-  while (strncmp(word, result, wordlen) && *result != '\0')
   {
-    get_word(output, result, result_len);
+    uint lineword_len= 0;
-  }
+    char *linep= linebuf;
 
+    linebuf[sizeof(linebuf) - 1]= '\0';        /* safety */
+
+    /*
+      Get the word, which might contain non-alphanumeric characters. (Usually
+      these are '/', '-' and '.' in the path expressions and filenames)
+    */
+    get_word((const char **) &linep, &lineword_len, NONSPACE);
+    if (!strncmp(word, linep, wordlen) && *result != '\0')
+    {
       /*
         If we have found the word, return the next one. This is usually
         an option value.
       */
-  if (*result != '\0')
+      get_word((const char **) &linep, &lineword_len, NONSPACE);
-    get_word(output, result, result_len);
+      DBUG_ASSERT(result_len > lineword_len);
+      strncpy(result, linep, lineword_len);
+      goto pclose;
+    }
+  }
 
+pclose:
   if (pclose(output))
     return 1;
 

server-tools/instance-manager/protocol.cc

@@ -154,7 +154,8 @@ int send_fields(struct st_net *net, LIST *fields)
     store_to_string(&send_buff, (char *) "", &position); /* table name alias */
     store_to_string(&send_buff, field->name, &position); /* column name */
     store_to_string(&send_buff, field->name, &position); /* column name alias */
-    if (send_buff.reserve(position, 12))
+    send_buff.reserve(position, 12);
+    if (send_buff.is_error())
       goto err;
     send_buff.buffer[position++]= 12;
     int2store(send_buff.buffer + position, 1); /* charsetnr */

server-tools/instance-manager/user_map.cc

@@ -69,7 +69,7 @@ int User::init(const char *line)
 
   return 0;
 err:
-  log_error("error parsing user and password at line %d", line);
+  log_error("error parsing user and password at line %s", line);
   return 1;
 }