BUG#27779: Slave cannot read old rows log events.

Problem: Replication fails when master is mysql-5.1-wl2325-5.0-drop6 and
slave is mysql-5.1-new-rpl. The reason is that, in
mysql-5.1-wl2325-5.0-drop6, the event type id's were different than in
mysql-5.1-new-rpl.
Fix (in mysql-5.1-new-rpl):
 (1) detect that the server that generated the events uses the old
format, by checking the server version of the format_description_log_event
This patch recognizes mysql-5.1-wl2325-5.0-drop6p13-alpha,
mysql-5.1-wl2325-5.0-drop6, mysql-5.1-wl2325-5.0, mysql-5.1-wl2325-no-dd.
 (2) if the generating server is old, map old event types to new event
types using a permutation array.

I've also added a test case which reads binlogs for four different
versions.


mysql-test/suite/binlog/std_data/binlog_old_version_5_1-telco.000001:
  BitKeeper file /home/sven/bk/b27779-old_row_events/5.1-new-rpl/mysql-test/suite/binlog/std_data/binlog_old_version_5_1-telco.000001
mysql-test/suite/binlog/std_data/binlog_old_version_5_1-wl2325_row.000001:
  BitKeeper file /home/sven/bk/b27779-old_row_events/5.1-new-rpl/mysql-test/suite/binlog/std_data/binlog_old_version_5_1-wl2325_row.000001
mysql-test/suite/binlog/std_data/binlog_old_version_5_1-wl2325_stm.000001:
  BitKeeper file /home/sven/bk/b27779-old_row_events/5.1-new-rpl/mysql-test/suite/binlog/std_data/binlog_old_version_5_1-wl2325_stm.000001
mysql-test/suite/binlog/std_data/binlog_old_version_5_1_17.000001:
  BitKeeper file /home/sven/bk/b27779-old_row_events/5.1-new-rpl/mysql-test/suite/binlog/std_data/binlog_old_version_5_1_17.000001
mysql-test/suite/binlog/std_data/binlog_old_version_5_1_23.000001:
  BitKeeper file /home/sven/bk/b27779-old_row_events/5.1-new-rpl/mysql-test/suite/binlog/std_data/binlog_old_version_5_1_23.000001
sql/log_event.cc:
  Added code to read events generated by
  mysql-5.1-wl2325-5.0-drop6p13-alpha, mysql-5.1-wl2325-5.0-drop6,
  mysql-5.1-wl2325-5.0, mysql-5.1-wl2325-no-dd.
  More precisely, the event type id's had different numbers in
  those versions. To fix, we add a permutation array which maps old_id to
  new_id when the format_description_log_event indicates that the
  originating server is of the old type. We also need to permute the
  post_header_len array accordingly.
sql/log_event.h:
  sql/log_event.h@1.169, 2008-01-09 11:34:37+01:00, sven@riska.(none) +5 -1
  Added declaration needed in log_event.cc. Also, the destructor of
  Format_description_log_event is sometimes called when post_header_len is
  null, so we must pass the MY_ALLOW_ZERO_PTR flag to my_free.
mysql-test/suite/binlog/r/binlog_old_versions.result:
  Result file for new test.
mysql-test/suite/binlog/t/binlog_old_versions.test:
  New test case that loads binlogs from several old versions.

mysql-test/suite/binlog/r/binlog_old_versions.result

@@ -0,0 +1,61 @@
+DROP TABLE IF EXISTS t1, t2, t3;
+==== Read modern binlog (version 5.1.23) ====
+SELECT * FROM t1 ORDER BY a;
+a	b
+0	last_insert_id
+1	one
+3	last stm in trx: next event should be xid
+4	four
+674568	random
+SELECT * FROM t2 ORDER BY a;
+a	b
+3	first stm in trx
+SELECT COUNT(*) FROM t3;
+COUNT(*)
+17920
+DROP TABLE t1, t2, t3;
+==== Read binlog from version 5.1.17 ====
+SELECT * FROM t1 ORDER BY a;
+a	b
+0	last_insert_id
+1	one
+3	last stm in trx: next event should be xid
+4	four
+764247	random
+SELECT * FROM t2 ORDER BY a;
+a	b
+3	first stm in trx
+SELECT COUNT(*) FROM t3;
+COUNT(*)
+17920
+DROP TABLE t1, t2, t3;
+==== Read binlog from alcatel tree (mysql-5.1-wl2325-5.0-drop6) ====
+SELECT * FROM t1 ORDER BY a;
+a	b
+0	last_insert_id
+1	one
+3	last stm in trx: next event should be xid
+4	four
+781729	random
+SELECT * FROM t2 ORDER BY a;
+a	b
+3	first stm in trx
+SELECT COUNT(*) FROM t3;
+COUNT(*)
+17920
+DROP TABLE t1, t2, t3;
+==== Read binlog from ndb tree (mysql-5.1-telco-6.1) ====
+SELECT * FROM t1 ORDER BY a;
+a	b
+0	last_insert_id
+1	one
+3	last stm in trx: next event should be xid
+4	four
+703356	random
+SELECT * FROM t2 ORDER BY a;
+a	b
+3	first stm in trx
+SELECT COUNT(*) FROM t3;
+COUNT(*)
+17920
+DROP TABLE t1, t2, t3;

mysql-test/suite/binlog/t/binlog_old_versions.test

@@ -0,0 +1,147 @@
+# Test that old binlog formats can be read.
+
+# Some previous versions of MySQL use their own binlog format,
+# especially in row-based replication.  This test uses saved binlogs
+# from those old versions to test that we can replicate from old
+# versions to the present version.
+
+# Replicating from old versions to new versions is necessary in an
+# online upgrade scenario, where the .
+
+# The previous versions we currently test are:
+#  - version 5.1.17 and earlier trees
+#  - mysql-5.1-wl2325-xxx trees (AKA alcatel trees)
+#  - mysql-5.1-telco-6.1 trees (AKA ndb trees)
+# For completeness, we also test mysql-5.1-new_rpl, which is supposed
+# to be the "correct" version.
+
+# All binlogs were generated with the same commands (listed at the end
+# of this test for reference).  The binlogs contain the following
+# events: Table_map, Write_rows, Update_rows, Delete_rows Query, Xid,
+# User_var, Int_var, Rand, Begin_load, Append_file, Execute_load.
+
+# Related bugs: BUG#27779, BUG#31581, BUG#31582, BUG#31583, BUG#32407
+
+
+--disable_warnings
+DROP TABLE IF EXISTS t1, t2, t3;
+
+
+--echo ==== Read modern binlog (version 5.1.23) ====
+
+# Read binlog.
+--exec $MYSQL_BINLOG suite/binlog/std_data/binlog_old_version_5_1_23.000001 | $MYSQL
+# Show result.
+SELECT * FROM t1 ORDER BY a;
+SELECT * FROM t2 ORDER BY a;
+SELECT COUNT(*) FROM t3;
+# Reset.
+DROP TABLE t1, t2, t3;
+
+
+--echo ==== Read binlog from version 5.1.17 ====
+
+# Read binlog.
+--exec $MYSQL_BINLOG suite/binlog/std_data/binlog_old_version_5_1_17.000001 | $MYSQL
+# Show result.
+SELECT * FROM t1 ORDER BY a;
+SELECT * FROM t2 ORDER BY a;
+SELECT COUNT(*) FROM t3;
+# Reset.
+DROP TABLE t1, t2, t3;
+
+
+--echo ==== Read binlog from alcatel tree (mysql-5.1-wl2325-5.0-drop6) ====
+
+# In this version, it was not possible to switch between row-based and
+# statement-based binlogging without restarting the server.  So, we
+# have two binlogs; one for row based and one for statement based
+# replication.
+
+# Read rbr binlog.
+--exec $MYSQL_BINLOG suite/binlog/std_data/binlog_old_version_5_1-wl2325_row.000001 | $MYSQL
+# Read stm binlog.
+--exec $MYSQL_BINLOG suite/binlog/std_data/binlog_old_version_5_1-wl2325_stm.000001 | $MYSQL
+# Show result.
+SELECT * FROM t1 ORDER BY a;
+SELECT * FROM t2 ORDER BY a;
+SELECT COUNT(*) FROM t3;
+# Reset.
+DROP TABLE t1, t2, t3;
+
+
+--echo ==== Read binlog from ndb tree (mysql-5.1-telco-6.1) ====
+
+# Read binlog.
+--exec $MYSQL_BINLOG suite/binlog/std_data/binlog_old_version_5_1-telco.000001 | $MYSQL
+# Show resulting tablea.
+SELECT * FROM t1 ORDER BY a;
+SELECT * FROM t2 ORDER BY a;
+SELECT COUNT(*) FROM t3;
+# Reset.
+DROP TABLE t1, t2, t3;
+
+
+#### The following commands were used to generate the binlogs ####
+#
+#source include/master-slave.inc;
+#
+## ==== initialize ====
+#USE test;
+#CREATE TABLE t1 (a int, b char(50)) ENGINE = MyISAM;
+#CREATE TABLE t2 (a int, b char(50)) ENGINE = InnoDB;
+#CREATE TABLE t3 (a char(20));
+#
+#
+## ==== row based tests ====
+#SET BINLOG_FORMAT='row';
+#
+## ---- get write, update, and delete rows events ----
+#INSERT INTO t1 VALUES (0, 'one'), (1, 'two');
+#UPDATE t1 SET a=a+1;
+#DELETE FROM t1 WHERE a=2;
+#
+#
+## ==== statement based tests ====
+#SET BINLOG_FORMAT = 'statement';
+#
+## ---- get xid events ----
+#BEGIN;
+#INSERT INTO t2 VALUES (3, 'first stm in trx');
+#INSERT INTO t1 VALUES (3, 'last stm in trx: next event should be xid');
+#COMMIT;
+#
+## ---- get user var events ----
+#SET @x = 4;
+#INSERT INTO t1 VALUES (@x, 'four');
+#
+## ---- get rand event ----
+#INSERT INTO t1 VALUES (RAND() * 1000000, 'random');
+#
+## ---- get intvar event ----
+#INSERT INTO t1 VALUES (LAST_INSERT_ID(), 'last_insert_id');
+#
+## ---- get begin, append and execute load events ----
+## double the file until we have more than 2^17 bytes, so that the
+## event has to be split and we can use Append_file_log_event.
+#
+#SET SQL_LOG_BIN=0;
+#CREATE TABLE temp (a char(20));
+#LOAD DATA INFILE '../std_data_ln/words.dat' INTO TABLE temp;
+#INSERT INTO temp SELECT * FROM temp;
+#INSERT INTO temp SELECT * FROM temp;
+#INSERT INTO temp SELECT * FROM temp;
+#INSERT INTO temp SELECT * FROM temp;
+#INSERT INTO temp SELECT * FROM temp;
+#INSERT INTO temp SELECT * FROM temp;
+#INSERT INTO temp SELECT * FROM temp;
+#INSERT INTO temp SELECT * FROM temp;
+#SELECT a FROM temp INTO OUTFILE 'big_file.dat';
+#DROP TABLE temp;
+#SET SQL_LOG_BIN=1;
+#
+#LOAD DATA INFILE 'big_file.dat' INTO TABLE t3;
+#
+#SELECT * FROM t1 ORDER BY a;
+#SELECT * FROM t2 ORDER BY a;
+#SELECT COUNT(*) FROM t3;

sql/log_event.cc

@@ -1071,6 +1071,29 @@ Log_event* Log_event::read_log_event(const char* buf, uint event_len,
   }
   else
   {
+    /*
+      In some previuos versions (see comment in
+      Format_description_log_event::Format_description_log_event(char*,...)),
+      event types were assigned different id numbers than in the
+      present version. In order to replicate from such versions to the
+      present version, we must map those event type id's to our event
+      type id's.  The mapping is done with the event_type_permutation
+      array, which was set up when the Format_description_log_event
+      was read.
+    */
+    if (description_event->event_type_permutation)
+    {
+      IF_DBUG({
+          int new_event_type=
+            description_event->event_type_permutation[event_type];
+          DBUG_PRINT("info",
+                     ("converting event type %d to %d (%s)",
+                      event_type, new_event_type,
+                      get_type_str((Log_event_type)new_event_type)));
+        });
+      event_type= description_event->event_type_permutation[event_type];
+    }
+
     switch(event_type) {
     case QUERY_EVENT:
       ev  = new Query_log_event(buf, event_len, description_event, QUERY_EVENT);
@@ -2771,7 +2794,7 @@ int Start_log_event_v3::do_apply_event(Relay_log_info const *rli)
 
 Format_description_log_event::
 Format_description_log_event(uint8 binlog_ver, const char* server_ver)
-  :Start_log_event_v3()
+  :Start_log_event_v3(), event_type_permutation(0)
 {
   binlog_version= binlog_ver;
   switch (binlog_ver) {
@@ -2896,7 +2919,7 @@ Format_description_log_event(const char* buf,
                              const
                              Format_description_log_event*
                              description_event)
-  :Start_log_event_v3(buf, description_event)
+  :Start_log_event_v3(buf, description_event), event_type_permutation(0)
 {
   DBUG_ENTER("Format_description_log_event::Format_description_log_event(char*,...)");
   buf+= LOG_EVENT_MINIMAL_HEADER_LEN;
@@ -2911,6 +2934,65 @@ Format_description_log_event(const char* buf,
                                       number_of_event_types*
                                       sizeof(*post_header_len), MYF(0));
   calc_server_version_split();
+
+  /*
+    In some previous versions, the events were given other event type
+    id numbers than in the present version. When replicating from such
+    a version, we therefore set up an array that maps those id numbers
+    to the id numbers of the present server.
+
+    If post_header_len is null, it means malloc failed, and is_valid
+    will fail, so there is no need to do anything.
+
+    The trees which have wrong event id's are:
+    mysql-5.1-wl2325-5.0-drop6p13-alpha, mysql-5.1-wl2325-5.0-drop6,
+    mysql-5.1-wl2325-5.0, mysql-5.1-wl2325-no-dd (`grep -C2
+    BEGIN_LOAD_QUERY_EVENT /home/bk/ * /sql/log_event.h`). The
+    corresponding version (`grep mysql, configure.in` in those trees)
+    strings are 5.2.2-a_drop6p13-alpha, 5.2.2-a_drop6p13c,
+    5.1.5-a_drop5p20, 5.1.2-a_drop5p5.
+  */
+  if (post_header_len &&
+      (strncmp(server_version, "5.1.2-a_drop5", 13) == 0 ||
+       strncmp(server_version, "5.1.5-a_drop5", 13) == 0 ||
+       strncmp(server_version, "5.2.2-a_drop6", 13) == 0))
+  {
+    if (number_of_event_types != 22)
+    {
+      DBUG_PRINT("info", (" number_of_event_types=%d",
+                          number_of_event_types));
+      /* this makes is_valid() return false. */
+      my_free(post_header_len, MYF(MY_ALLOW_ZERO_PTR));
+      post_header_len= NULL;
+      DBUG_VOID_RETURN;
+    }
+    static const uint8 perm[23]=
+      {
+        UNKNOWN_EVENT, START_EVENT_V3, QUERY_EVENT, STOP_EVENT, ROTATE_EVENT,
+        INTVAR_EVENT, LOAD_EVENT, SLAVE_EVENT, CREATE_FILE_EVENT,
+        APPEND_BLOCK_EVENT, EXEC_LOAD_EVENT, DELETE_FILE_EVENT,
+        NEW_LOAD_EVENT,
+        RAND_EVENT, USER_VAR_EVENT,
+        FORMAT_DESCRIPTION_EVENT,
+        TABLE_MAP_EVENT,
+        PRE_GA_WRITE_ROWS_EVENT,
+        PRE_GA_UPDATE_ROWS_EVENT,
+        PRE_GA_DELETE_ROWS_EVENT,
+        XID_EVENT,
+        BEGIN_LOAD_QUERY_EVENT,
+        EXECUTE_LOAD_QUERY_EVENT,
+      };
+    event_type_permutation= perm;
+    /*
+      Since we use (permuted) event id's to index the post_header_len
+      array, we need to permute the post_header_len array too.
+    */
+    uint8 post_header_len_temp[23];
+    for (int i= 1; i < 23; i++)
+      post_header_len_temp[perm[i] - 1]= post_header_len[i - 1];
+    for (int i= 0; i < 22; i++)
+      post_header_len[i] = post_header_len_temp[i];
+  }
   DBUG_VOID_RETURN;
 }
 

sql/log_event.h

@@ -2106,12 +2106,16 @@ public:
   /* The list of post-headers' lengthes */
   uint8 *post_header_len;
   uchar server_version_split[3];
+  const uint8 *event_type_permutation;
 
   Format_description_log_event(uint8 binlog_ver, const char* server_ver=0);
   Format_description_log_event(const char* buf, uint event_len,
                                const Format_description_log_event
                                *description_event);
-  ~Format_description_log_event() { my_free((uchar*)post_header_len, MYF(0)); }
+  ~Format_description_log_event()
+  {
+    my_free((uchar*)post_header_len, MYF(MY_ALLOW_ZERO_PTR));
+  }
   Log_event_type get_type_code() { return FORMAT_DESCRIPTION_EVENT;}
 #ifndef MYSQL_CLIENT
   bool write(IO_CACHE* file);