diff --git a/client/mysqltest.cc b/client/mysqltest.cc
index 3812a1bb8d1..81586c4ffc0 100644
--- a/client/mysqltest.cc
+++ b/client/mysqltest.cc
@@ -7711,6 +7711,7 @@ int main(int argc, char **argv)
     if (!ok_to_do)
     {
       if (command->type == Q_SOURCE ||
+          command->type == Q_ERROR ||
           command->type == Q_WRITE_FILE ||
           command->type == Q_APPEND_FILE ||
 	  command->type == Q_PERL)
diff --git a/mysql-test/include/check-warnings.test b/mysql-test/include/check-warnings.test
index 5295dd51a85..41b0a98e43b 100644
--- a/mysql-test/include/check-warnings.test
+++ b/mysql-test/include/check-warnings.test
@@ -57,5 +57,5 @@ if (`select @result = 0`){
   skip OK;
 }
 --enable_query_log
-echo ^ Found warnings!!;
+echo ^ Found warnings in $log_error;
 exit;
diff --git a/mysql-test/include/mtr_warnings.sql b/mysql-test/include/mtr_warnings.sql
index 52bf4cd51ef..d4c11b056e3 100644
--- a/mysql-test/include/mtr_warnings.sql
+++ b/mysql-test/include/mtr_warnings.sql
@@ -220,7 +220,7 @@ BEGIN
     WHERE suspicious=1;
 
   IF @num_warnings > 0 THEN
-    SELECT file_name, line
+    SELECT line
         FROM error_log WHERE suspicious=1;
     --SELECT * FROM test_suppressions;
     -- Return 2 -> check failed
diff --git a/mysql-test/include/not_windows_embedded.inc b/mysql-test/include/not_windows_embedded.inc
new file mode 100644
index 00000000000..46f5e0ccfce
--- /dev/null
+++ b/mysql-test/include/not_windows_embedded.inc
@@ -0,0 +1,11 @@
+let $is_win = `select convert(@@version_compile_os using latin1) IN ("Win32","Win64","Windows")`;
+let $is_embedded = `select version() like '%embedded%'`;
+#echo is_win: $is_win;
+#echo is_embedded: $is_embedded;
+if ($is_win)
+{
+  if ($is_embedded)
+  {
+    skip Not supported with embedded on windows;
+  }
+}
diff --git a/mysql-test/lib/My/SafeProcess/safe_kill_win.cc b/mysql-test/lib/My/SafeProcess/safe_kill_win.cc
index c6256fd92e1..963a02c8099 100755
--- a/mysql-test/lib/My/SafeProcess/safe_kill_win.cc
+++ b/mysql-test/lib/My/SafeProcess/safe_kill_win.cc
@@ -30,7 +30,7 @@ int main(int argc, const char** argv )
   DWORD pid= -1;
   HANDLE shutdown_event;
   char safe_process_name[32]= {0};
-  int retry_open_event= 100;
+  int retry_open_event= 2;
   /* Ignore any signals */
   signal(SIGINT,   SIG_IGN);
   signal(SIGBREAK, SIG_IGN);
@@ -51,15 +51,31 @@ int main(int argc, const char** argv )
   {
      /*
       Check if the process is alive, otherwise there is really
-      no idea to retry the open of the event
+      no sense to retry the open of the event
      */
     HANDLE process;
-    if ((process= OpenProcess(SYNCHRONIZE, FALSE, pid)) == NULL)
+    DWORD exit_code;
+    process= OpenProcess(SYNCHRONIZE| PROCESS_QUERY_INFORMATION, FALSE, pid);
+    if (!process)
     {
-      fprintf(stderr, "Could not open event or process %d, error: %d\n",
-            pid, GetLastError());
-      exit(3);
+      /* Already died */
+      exit(1);
     }
+
+    if (!GetExitCodeProcess(process,&exit_code))
+    {
+       fprintf(stderr,  "GetExitCodeProcess failed, pid= %d, err= %d\n",
+         pid, GetLastError());
+       exit(1);
+    }
+
+    if (exit_code != STILL_ACTIVE)
+    {
+       /* Already died */
+       CloseHandle(process);
+       exit(2);
+    }
+
     CloseHandle(process);
 
     if (retry_open_event--)
diff --git a/mysql-test/mysql-test-run.pl b/mysql-test/mysql-test-run.pl
index f76e467ab40..e14c4bff258 100755
--- a/mysql-test/mysql-test-run.pl
+++ b/mysql-test/mysql-test-run.pl
@@ -1337,6 +1337,9 @@ sub command_line_setup {
     push(@valgrind_args, @default_valgrind_args)
       unless @valgrind_args;
 
+    # Make valgrind run in quiet mode so it only print errors
+    push(@valgrind_args, "--quiet" );
+
     mtr_report("Running valgrind with options \"",
 	       join(" ", @valgrind_args), "\"");
   }
@@ -1794,7 +1797,7 @@ sub environment_setup {
   # --------------------------------------------------------------------------
   # Add the path where mysqld will find ha_example.so
   # --------------------------------------------------------------------------
-  if ($mysql_version_id >= 50100 && !(IS_WINDOWS && $opt_embedded_server)) {
+  if ($mysql_version_id >= 50100) {
     my $plugin_filename;
     if (IS_WINDOWS)
     {
@@ -3252,6 +3255,12 @@ sub run_testcase ($) {
 
   mtr_verbose("Running test:", $tinfo->{name});
 
+  # Allow only alpanumerics pluss _ - + . in combination names
+  my $combination= $tinfo->{combination};
+  if ($combination && $combination !~ /^\w[\w-\.\+]+$/)
+  {
+    mtr_error("Combination '$combination' contains illegal characters");
+  }
   # -------------------------------------------------------
   # Init variables that can change between each test case
   # -------------------------------------------------------
@@ -3660,7 +3669,7 @@ sub extract_warning_lines ($$) {
      # of patterns. For more info see BUG#42408
      qr/^Warning:|mysqld: Warning|\[Warning\]/,
      qr/^Error:|\[ERROR\]/,
-     qr/^==.* at 0x/,
+     qr/^==\d*==/, # valgrind errors
      qr/InnoDB: Warning|InnoDB: Error/,
      qr/^safe_mutex:|allocated at line/,
      qr/missing DBUG_RETURN/,
@@ -4285,7 +4294,8 @@ sub mysqld_start ($$) {
 				 $opt_start_timeout,
 				 $mysqld->{'proc'}))
   {
-    mtr_error("Failed to start mysqld $mysqld->name()");
+    my $mname= $mysqld->name();
+    mtr_error("Failed to start mysqld $mname with command $exe");
   }
 
   # Remember options used when starting
diff --git a/mysql-test/r/mysqltest.result b/mysql-test/r/mysqltest.result
index f68413264e4..2e3a9489593 100644
--- a/mysql-test/r/mysqltest.result
+++ b/mysql-test/r/mysqltest.result
@@ -317,6 +317,7 @@ here is the sourced script
 outer=2 ifval=0
 outer=1 ifval=1
 here is the sourced script
+ERROR 42S02: Table 'test.nowhere' doesn't exist
 
 In loop
 here is the sourced script
diff --git a/mysql-test/t/mysqltest.test b/mysql-test/t/mysqltest.test
index 9859e73cfae..bcf33aa8c27 100644
--- a/mysql-test/t/mysqltest.test
+++ b/mysql-test/t/mysqltest.test
@@ -854,6 +854,7 @@ while ($outer)
 }
 
 # Test source in an if in a while which is false on 1st iteration
+# Also test --error in same context
 let $outer= 2; # Number of outer loops
 let $ifval= 0; # false 1st time
 while ($outer)
@@ -862,6 +863,8 @@ while ($outer)
 
   if ($ifval) {
     --source $MYSQLTEST_VARDIR/tmp/sourced.inc
+    --error ER_NO_SUCH_TABLE
+    SELECT * from nowhere;
   }
   dec $outer;
   inc $ifval;
diff --git a/mysql-test/t/plugin.test b/mysql-test/t/plugin.test
index 7fc62b445c9..788a7b336ef 100644
--- a/mysql-test/t/plugin.test
+++ b/mysql-test/t/plugin.test
@@ -1,3 +1,4 @@
+--source include/not_windows_embedded.inc
 --source include/have_example_plugin.inc
 
 CREATE TABLE t1(a int) ENGINE=EXAMPLE;
diff --git a/mysql-test/t/plugin_load.test b/mysql-test/t/plugin_load.test
index 8555247dd71..97b2afbe219 100644
--- a/mysql-test/t/plugin_load.test
+++ b/mysql-test/t/plugin_load.test
@@ -1,3 +1,4 @@
+--source include/not_windows_embedded.inc
 --source include/have_example_plugin.inc
 
 SELECT @@global.example_enum_var = 'e2';