Better optimization for InnoDB and BDB tables for ORDER BY

Build-tools/Do-compile:
  Merge with 4.0
Docs/manual.texi:
  Changelog
mysql-test/r/innodb.result:
  New tests cases for ORDER BY
mysql-test/r/myisam.result:
  New tests cases for ORDER BY
mysql-test/t/innodb.test:
  New tests cases for ORDER BY
mysql-test/t/myisam.test:
  New tests cases for ORDER BY

Build-tools/Do-compile

@@ -36,6 +36,10 @@ if (defined($gcc_version) && ! $opt_config_env)
   {
     $opt_config_env= 'CC=gcc CFLAGS="-O2 -fno-omit-frame-pointer" CXX=gcc CXXFLAGS="-O2 -fno-omit-frame-pointer"';
   }
+  elsif ($tmp =~ /version 3\.0\./)
+  {
+    $opt_config_env= 'CC=gcc CFLAGS="-O3 -fno-omit-frame-pointer" CXXFLAGS="-O3 -fno-omit-frame-pointer -felide-constructors -fno-exceptions -fno-rtti"';
+  }
   else
   {
     $opt_config_env= 'CC=gcc CFLAGS="-O3 -fno-omit-frame-pointer" CXX=gcc CXXFLAGS="-O3 -fno-omit-frame-pointer -felide-constructors -fno-exceptions -fno-rtti"';
@@ -65,10 +69,11 @@ $sendmail=find("/usr/lib/sendmail","/usr/sbin/sendmail");
 $sur= $opt_sur ? "/my/local/bin/sur" : "";
 delete $ENV{'MYSQL_PWD'};	# Reset possibly password
 delete $ENV{'MY_BASEDIR_VERSION'};
-$ENV{'MYSQL_TCP_PORT'}= $mysql_tcp_port= 3334 + $opt_build_thread;
+$ENV{'MYSQL_TCP_PORT'}= $mysql_tcp_port= 3334 + $opt_build_thread*2;
 $ENV{'MYSQL_UNIX_PORT'}=$mysql_unix_port="$opt_tmp/mysql$opt_suffix.build";
 $ENV{"PERL5LIB"}="$pwd/$host/perl5:$pwd/$host/perl5/site_perl";
 $slave_port=$mysql_tcp_port+16;
+$manager_port=$mysql_tcp_port+1;
 
 if (-x "$host/bin/mysqladmin")
 {
@@ -78,6 +83,7 @@ if (-x "$host/bin/mysqladmin")
   log_system("$host/bin/mysqladmin --no-defaults -u root -P 9306 -h $host -s shutdown");
   log_system("$host/bin/mysqladmin --no-defaults -u root -P 9307 -h $host -s shutdown");
 }
+kill_all("mysqlmanager");
 
 if ($opt_stage == 0)
 {
@@ -106,7 +112,21 @@ safe_cd("$host");
 if ($opt_stage == 0 && ! $opt_use_old_distribution)
 {
   safe_system("gunzip < $opt_distribution | $tar xf -");
+
+  # Fix file times; This is needed because the time for files may be
+  # in the future
+  system("touch timestamp; find . -newer timestamp -print | xargs touch; rm -f timestamp");
+  sleep(2);
+  # Ensure that files we don't want to rebuild are newer than other files
+  foreach $name ("configure",
+		 "Docs/include.texi",
+		 "Docs/*.html", "Docs/manual.txt", "Docs/mysql.info",
+		 "sql/sql_yacc.h", "sql/sql_yacc.cc")
+  {
+    system("touch $name");
+  }
 }
+
 safe_cd($ver);
 if ($opt_stage <= 1)
 {
@@ -136,7 +156,7 @@ if ($opt_stage <= 1)
   {
     $opt_config_options.= " --with-innodb"
   }
-  check_system("$opt_config_env ./configure --prefix=/usr/local/mysql \"--with-comment=Official MySQL$version_suffix binary\" --with-extra-charsets=complex \"--with-server-suffix=$version_suffix\" $opt_config_options","Thank you for choosing MySQL");
+  check_system("$opt_config_env ./configure --prefix=/usr/local/mysql \"--with-comment=Official MySQL$version_suffix binary\" --with-extra-charsets=complex \"--with-server-suffix=$version_suffix\" --enable-thread-safe-client $opt_config_options","Thank you for choosing MySQL");
   if (-d "$pwd/$host/include-mysql")
   {
     safe_system("cp -r $pwd/$host/include-mysql/* $pwd/$host/$ver/include");
@@ -192,7 +212,7 @@ if ($opt_stage <= 4 && !$opt_no_test)
 $tar_file =~ /(mysql-[^\/]*)\.tar/;
 $ver=$1;
 $test_dir="$pwd/$host/test/$ver";
-$ENV{"LD_LIBRARY_PATH"}= "$testdir/lib:" . $ENV{"LD_LIBRARY_PATH"};
+$ENV{"LD_LIBRARY_PATH"}= "$test_dir/lib:" . $ENV{"LD_LIBRARY_PATH"};
 
 if ($opt_stage <= 5 && !$opt_no_test && !$opt_no_mysqltest)
 {
@@ -220,7 +240,7 @@ if (!$opt_no_test)
   {
     $extra.=" --innodb_data_file_path=ibdata1:100M";
   }
-  safe_system("./bin/mysqld --no-defaults --basedir . --datadir ./data --skip-l\ocking $extra >> $log 2>&1 &");
+  safe_system("./bin/mysqld --no-defaults --basedir . --datadir ./data --skip-locking $extra >> $log 2>&1 &");
   sleep(2);
 }
 
@@ -300,7 +320,7 @@ exit 0;
 sub usage
 {
 print <<EOF;
-$0  version 1.2
+$0  version 1.4
 
 $0 takes the following options:
 
@@ -314,7 +334,7 @@ Compile with support for Innodb tables
 Compile with support for Berkeley DB tables
 
 --user 'user_name'
-Mail 'user_name'\@analytikerna.se if something went wrong.
+Mail 'user_name'\@mysql.com if something went wrong.
 If user is empty then no mail is sent.
 
 --distribution 'distribution_file'
@@ -513,3 +533,44 @@ sub rm_all
     }
   }
 }
+
+sub kill_all
+{
+  my ($pattern) = @_;
+  my ($USER,$BSD,$LINUX,$pscmd, $user, $pid);
+  $user=$ENV{'USER'};
+  $BSD = -f '/vmunix' || $ENV{"OS"} eq "SunOS4";
+  $LINUX = $^O eq 'linux';
+  $pscmd = $BSD ? "/bin/ps -auxww" : $LINUX ? "/bin/ps axuw" : "/bin/ps -ef";
+
+  open(PS, "$pscmd|") || die "can't run $pscmd: $!";
+
+  # Catch any errors with eval.  A bad pattern, for instance.
+ process:
+  while ($cand = <PS>)
+  {
+    chop($cand);
+    ($pid_user, $pid) = split(' ', $cand);
+    next if $pid == $$;
+    next process if (! ($cand =~ $pattern) || $pid_user ne $user);
+    print LOG "Killing $_\n";
+    &killpid($pid);
+  }
+}
+
+sub killpid
+{
+  local($pid) = @_;
+  kill 15, $pid;
+  for (1..5)
+  {
+    sleep 2;
+    return if kill(0, $pid) == 0;
+  }
+  kill 9, $pid;
+  for (1..5) {
+    sleep 2;
+    return if kill(0, $pid) == 0;
+  }
+  print LOG "$pid will not die!\n";
+}

Docs/manual.texi

@@ -46838,6 +46838,9 @@ not yet 100% confident in this code.
 @appendixsubsec Changes in release 3.23.46
 @itemize @bullet
 @item
+InnoDB and BDB tables will now use index when doing an @code{ORDER BY}
+on the whole table.
+@item
 Fixed bug where one got an empty set instead of a DEADLOCK error when using
 BDB tables.
 @item

mysql-test/r/innodb.result

@@ -483,3 +483,19 @@ id	name	value	uid
 1	one	one value	101
 3	three	three value	103
 6	two	other value	102
+table	type	possible_keys	key	key_len	ref	rows	Extra
+t1	index	NULL	PRIMARY	4	NULL	4	
+table	type	possible_keys	key	key_len	ref	rows	Extra
+t1	index	NULL	b	4	NULL	4	
+table	type	possible_keys	key	key_len	ref	rows	Extra
+t1	ALL	NULL	NULL	NULL	NULL	4	Using filesort
+table	type	possible_keys	key	key_len	ref	rows	Extra
+t1	index	NULL	PRIMARY	4	NULL	4	Using index
+table	type	possible_keys	key	key_len	ref	rows	Extra
+t1	index	NULL	b	4	NULL	4	Using index
+table	type	possible_keys	key	key_len	ref	rows	Extra
+t1	index	NULL	b	4	NULL	4	Using index
+table	type	possible_keys	key	key_len	ref	rows	Extra
+t1	index	NULL	b	4	NULL	4	Using index
+table	type	possible_keys	key	key_len	ref	rows	Extra
+t1	ALL	NULL	NULL	NULL	NULL	4	

mysql-test/r/myisam.result

@@ -20,3 +20,19 @@ test.t1	optimize	status	Table is already up to date
 Table	Non_unique	Key_name	Seq_in_index	Column_name	Collation	Cardinality	Sub_part	Packed	Comment
 t1	0	PRIMARY	1	a	A	5	NULL	NULL	
 t1	1	b	1	b	A	1	NULL	NULL	
+table	type	possible_keys	key	key_len	ref	rows	Extra
+t1	ALL	NULL	NULL	NULL	NULL	4	Using filesort
+table	type	possible_keys	key	key_len	ref	rows	Extra
+t1	ALL	NULL	NULL	NULL	NULL	4	Using filesort
+table	type	possible_keys	key	key_len	ref	rows	Extra
+t1	ALL	NULL	NULL	NULL	NULL	4	Using filesort
+table	type	possible_keys	key	key_len	ref	rows	Extra
+t1	index	NULL	PRIMARY	4	NULL	4	Using index
+table	type	possible_keys	key	key_len	ref	rows	Extra
+t1	index	NULL	b	4	NULL	4	Using index
+table	type	possible_keys	key	key_len	ref	rows	Extra
+t1	ALL	NULL	NULL	NULL	NULL	4	Using filesort
+table	type	possible_keys	key	key_len	ref	rows	Extra
+t1	ALL	NULL	NULL	NULL	NULL	4	
+table	type	possible_keys	key	key_len	ref	rows	Extra
+t1	ALL	NULL	NULL	NULL	NULL	4	

mysql-test/t/innodb.test

@@ -513,3 +513,19 @@ set insert_id=6;
 replace into t1 (value,name,uid) values ('other value','two',102);
 select * from t1;
 drop table t1;
+
+#
+# Test of how ORDER BY works when doing it on the whole table
+#
+
+create table t1 (a int not null, b int not null, c int not null, primary key (a),key(b)) type=innodb;
+insert into t1 values (3,3,3),(1,1,1),(2,2,2),(4,4,4);
+explain select * from t1 order by a;
+explain select * from t1 order by b;
+explain select * from t1 order by c;
+explain select a from t1 order by a;
+explain select b from t1 order by b;
+explain select a,b from t1 order by b;
+explain select a,b from t1;
+explain select a,b,c from t1;
+drop table t1;

mysql-test/t/myisam.test

@@ -48,3 +48,19 @@ show index from t1;
 optimize table t1;
 show index from t1;
 drop table t1;
+
+#
+# Test of how ORDER BY works when doing it on the whole table
+#
+
+create table t1 (a int not null, b int not null, c int not null, primary key (a),key(b)) type=myisam;
+insert into t1 values (3,3,3),(1,1,1),(2,2,2),(4,4,4);
+explain select * from t1 order by a;
+explain select * from t1 order by b;
+explain select * from t1 order by c;
+explain select a from t1 order by a;
+explain select b from t1 order by b;
+explain select a,b from t1 order by b;
+explain select a,b from t1;
+explain select a,b,c from t1;
+drop table t1;

sql/ha_berkeley.h

@@ -107,6 +107,7 @@ class ha_berkeley: public handler
   uint extra_rec_buf_length()	 { return BDB_HIDDEN_PRIMARY_KEY_LENGTH; }
   ha_rows estimate_number_of_rows();
   bool fast_key_read()	   { return 1;}
+  key_map keys_to_use_for_scanning() { return ~(key_map) 0; }
   bool has_transactions()  { return 1;}
 
   int open(const char *name, int mode, uint test_if_locked);

sql/ha_innobase.h

@@ -101,6 +101,7 @@ class ha_innobase: public handler
 				bytes */
   	uint max_key_length()    const { return 7000; }
   	bool fast_key_read()	 { return 1;}
+	key_map keys_to_use_for_scanning() { return ~(key_map) 0; }
   	bool has_transactions()  { return 1;}
 
   	int open(const char *name, int mode, uint test_if_locked);

sql/handler.h

@@ -223,6 +223,7 @@ public:
     { return ulonglong2double(data_file_length) / IO_SIZE + 1; }
   virtual double read_time(ha_rows rows) { return rows; }
   virtual bool fast_key_read() { return 0;}
+  virtual key_map keys_to_use_for_scanning() { return 0; }
   virtual bool has_transactions(){ return 0;}
   virtual uint extra_rec_buf_length() { return 0; }
   virtual ha_rows estimate_number_of_rows() { return records+EXTRA_RECORDS; }

sql/sql_select.cc

@@ -5199,7 +5199,7 @@ test_if_skip_sort_order(JOIN_TAB *tab,ORDER *order,ha_rows select_limit)
       retrieving all rows through an index.
     */
     if (select_limit >= table->file->records)
-      keys&= table->used_keys;
+      keys&= (table->used_keys | table->file->keys_to_use_for_scanning());
 
     for (nr=0; keys ; keys>>=1, nr++)
     {
@@ -5213,6 +5213,11 @@ test_if_skip_sort_order(JOIN_TAB *tab,ORDER *order,ha_rows select_limit)
 				    join_init_read_last_with_key);
 	  table->file->index_init(nr);
 	  tab->type=JT_NEXT;	// Read with index_first(), index_next()
+	  if (table->used_keys & ((key_map) 1 << nr))
+	  {
+	    table->key_read=1;
+	    table->file->extra(HA_EXTRA_KEYREAD);
+	  }
 	  DBUG_RETURN(1);
 	}
       }