MDEV-4425 REGEXP enhancements

Adding pcre_stack_guard to avoid crashes in pcre_compile()
on a long recursive patterns with parenthesizes:

SELECT a RLIKE '((((...((((x)))...))))';

pcre/pcre.h.in

@@ -486,6 +486,7 @@ PCRE_EXP_DECL void  (*pcre_free)(void *);
 PCRE_EXP_DECL void *(*pcre_stack_malloc)(size_t);
 PCRE_EXP_DECL void  (*pcre_stack_free)(void *);
 PCRE_EXP_DECL int   (*pcre_callout)(pcre_callout_block *);
+PCRE_EXP_DECL int   (*pcre_stack_guard)(void);
 
 PCRE_EXP_DECL void *(*pcre16_malloc)(size_t);
 PCRE_EXP_DECL void  (*pcre16_free)(void *);
@@ -504,6 +505,7 @@ PCRE_EXP_DECL void  pcre_free(void *);
 PCRE_EXP_DECL void *pcre_stack_malloc(size_t);
 PCRE_EXP_DECL void  pcre_stack_free(void *);
 PCRE_EXP_DECL int   pcre_callout(pcre_callout_block *);
+PCRE_EXP_DECL int   pcre_stack_guard(void);
 
 PCRE_EXP_DECL void *pcre16_malloc(size_t);
 PCRE_EXP_DECL void  pcre16_free(void *);

pcre/pcre_compile.c

@@ -7107,6 +7107,12 @@ unsigned int orig_bracount;
 unsigned int max_bracount;
 branch_chain bc;
 
+if (pcre_stack_guard && pcre_stack_guard())
+{
+  *errorcodeptr= ERR23;
+  return FALSE;
+}
+ 
 bc.outer = bcptr;
 bc.current_branch = code;
 

pcre/pcre_globals.c

@@ -72,6 +72,7 @@ PCRE_EXP_DATA_DEFN void  (*PUBL(free))(void *) = LocalPcreFree;
 PCRE_EXP_DATA_DEFN void *(*PUBL(stack_malloc))(size_t) = LocalPcreMalloc;
 PCRE_EXP_DATA_DEFN void  (*PUBL(stack_free))(void *) = LocalPcreFree;
 PCRE_EXP_DATA_DEFN int   (*PUBL(callout))(PUBL(callout_block) *) = NULL;
+PCRE_EXP_DATA_DEFN int   (*PUBL(stack_guard))(void) = NULL;
 
 #elif !defined VPCOMPAT
 PCRE_EXP_DATA_DEFN void *(*PUBL(malloc))(size_t) = malloc;
@@ -79,6 +80,7 @@ PCRE_EXP_DATA_DEFN void  (*PUBL(free))(void *) = free;
 PCRE_EXP_DATA_DEFN void *(*PUBL(stack_malloc))(size_t) = malloc;
 PCRE_EXP_DATA_DEFN void  (*PUBL(stack_free))(void *) = free;
 PCRE_EXP_DATA_DEFN int   (*PUBL(callout))(PUBL(callout_block) *) = NULL;
+PCRE_EXP_DATA_DEFN int   (*PUBL(stack_guard))(void) = NULL;
 #endif
 
 /* End of pcre_globals.c */

sql/mysqld.cc

@@ -3340,22 +3340,30 @@ sizeof(load_default_groups)/sizeof(load_default_groups[0]);
 /**
   This function is used to check for stack overrun for pathological
   cases of  regular expressions and 'like' expressions.
-  The call to current_thd is  quite expensive, so we try to avoid it
+*/
-  for the normal cases.
+extern "C" int
+check_enough_stack_size_slow()
+{
+  uchar stack_top;
+  THD *my_thd= current_thd;
+  if (my_thd != NULL)
+    return check_stack_overrun(my_thd, STACK_MIN_SIZE * 2, &stack_top);
+  return 0;
+}
+
+
+/*
+  The call to current_thd in check_enough_stack_size_slow is quite expensive,
+  so we try to avoid it for the normal cases.
   The size of  each stack frame for the wildcmp() routines is ~128 bytes,
   so checking  *every* recursive call is not necessary.
  */
 extern "C" int
 check_enough_stack_size(int recurse_level)
 {
-  uchar stack_top;
   if (recurse_level % 16 != 0)
     return 0;
-
+  return check_enough_stack_size_slow();
-  THD *my_thd= current_thd;
-  if (my_thd != NULL)
-    return check_stack_overrun(my_thd, STACK_MIN_SIZE * 2, &stack_top);
-  return 0;
 }
 #endif
 
@@ -3922,6 +3930,7 @@ static int init_common_variables()
   init_pcre();
 #ifndef EMBEDDED_LIBRARY
   my_string_stack_guard= check_enough_stack_size;
+  pcre_stack_guard= check_enough_stack_size_slow;
 #endif
   /*
     Process a comma-separated character set list and choose