MDEV-41622 INSERT crashes on deeply nested GeometryCollection

- get_data_size() and init_from_wkb() return their error values on
  stack overrun, not a valid-looking length
- add stack checks to the remaining recursive GeometryCollection
  methods: wkt, json, mbr, area, length, dimension, store_shapes
- test non-GEOMETRY column and SELECT paths

Assisted-by: Claude:claude-opus-5-5

mysql-test/main/lotofstack.result

@@ -116,3 +116,20 @@ ERROR HY000: Thread stack overrun:  'used bytes' used of a 'available' byte stac
 SELECT ExtractValue('<a/>', REPEAT('/n', 100000));
 ERROR HY000: Thread stack overrun:  'used bytes' used of a 'available' byte stack, and 'X' bytes needed. Consider increasing the thread_stack system variable.
 # End of 10.6 tests
+#
+# MDEV-41622 INSERT crashes on deeply nested GeometryCollection
+#
+set @g=concat(0x00000000, 0x0107000000, repeat(0x010000000107000000, 400000), 0x00000000);
+create table t1 (g geometry);
+create table t2 (g polygon);
+insert t1 values (@g);
+insert t2 values (@g);
+select st_astext(@g);
+select st_asgeojson(@g);
+select st_envelope(@g);
+select st_area(@g);
+select st_length(@g);
+select st_dimension(@g);
+select st_boundary(@g);
+drop table t1, t2;
+# End of 10.11 tests

mysql-test/main/lotofstack.test

@@ -12,7 +12,7 @@ if (`select count(*) from information_schema.system_variables
      where variable_name='have_sanitizer'
      and global_value REGEXP '(A|UB)SAN'`)
 {
---skip Can't be run in UBSAN/ASAN (stack size predictability)
+--skip Cannot be run in UBSAN/ASAN (stack size predictability)
 }
 
 #
@@ -172,3 +172,33 @@ SELECT ExtractValue('<a/>', REPEAT('(', 100000));
 SELECT ExtractValue('<a/>', REPEAT('/n', 100000));
 
 --echo # End of 10.6 tests
+
+--echo #
+--echo # MDEV-41622 INSERT crashes on deeply nested GeometryCollection
+--echo #
+set @g=concat(0x00000000, 0x0107000000, repeat(0x010000000107000000, 400000), 0x00000000);
+create table t1 (g geometry);
+create table t2 (g polygon);
+--disable_result_log
+--error ER_STACK_OVERRUN_NEED_MORE
+insert t1 values (@g);
+--error ER_STACK_OVERRUN_NEED_MORE
+insert t2 values (@g);
+--error ER_STACK_OVERRUN_NEED_MORE
+select st_astext(@g);
+--error ER_STACK_OVERRUN_NEED_MORE
+select st_asgeojson(@g);
+--error ER_STACK_OVERRUN_NEED_MORE
+select st_envelope(@g);
+--error ER_STACK_OVERRUN_NEED_MORE
+select st_area(@g);
+--error ER_STACK_OVERRUN_NEED_MORE
+select st_length(@g);
+--error ER_STACK_OVERRUN_NEED_MORE
+select st_dimension(@g);
+--error ER_STACK_OVERRUN_NEED_MORE
+select st_boundary(@g);
+--enable_result_log
+drop table t1, t2;
+
+--echo # End of 10.11 tests

sql/spatial.cc

@@ -1,6 +1,6 @@
 /*
    Copyright (c) 2002, 2013, Oracle and/or its affiliates.
-   Copyright (c) 2011, 2021, MariaDB Corporation.
+   Copyright (c) 2011, 2026, MariaDB plc.
 
    This program is free software; you can redistribute it and/or modify
    it under the terms of the GNU General Public License as published by
@@ -3359,6 +3359,9 @@ uint32 Gis_geometry_collection::get_data_size() const
   n_objects= uint4korr(data);
   data+= 4;
 
+  if (check_stack_overrun(current_thd, STACK_MIN_SIZE, (uchar*)&buffer))
+    return GET_SIZE_ERROR;
+
   while (n_objects--)
   {
     uint32 wkb_type,object_size;
@@ -3491,7 +3494,7 @@ uint Gis_geometry_collection::init_from_wkb(const char *wkb, uint len,
   n_geom= wkb_get_uint(wkb, bo);
 
   if (check_stack_overrun(current_thd, STACK_MIN_SIZE, (uchar*)&wkb_orig))
-    return 1;
+    return 0;
 
   if (res->reserve(4, 512))
     return 0;
@@ -3578,6 +3581,9 @@ bool Gis_geometry_collection::get_data_as_wkt(String *txt,
   n_objects= uint4korr(data);
   data+= 4;
 
+  if (check_stack_overrun(current_thd, STACK_MIN_SIZE, (uchar*)&buffer))
+    return 1;
+
   if (n_objects == 0)
   {
     txt->append(STRING_WITH_LEN(" EMPTY"), 512);
@@ -3622,6 +3628,9 @@ bool Gis_geometry_collection::get_data_as_json(String *txt, uint max_dec_digits,
   n_objects= uint4korr(data);
   data+= 4;
 
+  if (check_stack_overrun(current_thd, STACK_MIN_SIZE, (uchar*)&buffer))
+    return 1;
+
   bool any_objects= false;
   txt->qs_append('[');
   while (n_objects--)
@@ -3666,6 +3675,9 @@ bool Gis_geometry_collection::get_mbr(MBR *mbr, const char **end) const
   if (n_objects == 0)
     goto exit;
 
+  if (check_stack_overrun(current_thd, STACK_MIN_SIZE, (uchar*)&buffer))
+    return 1;
+
   while (n_objects--)
   {
     uint32 wkb_type;
@@ -3704,6 +3716,9 @@ int Gis_geometry_collection::area(double *ar,  const char **end) const
   if (n_objects == 0)
     goto exit;
 
+  if (check_stack_overrun(current_thd, STACK_MIN_SIZE, (uchar*)&buffer))
+    return 1;
+
   while (n_objects--)
   {
     uint32 wkb_type;
@@ -3744,6 +3759,9 @@ int Gis_geometry_collection::geom_length(double *len, const char **end) const
   if (n_objects == 0)
     goto exit;
 
+  if (check_stack_overrun(current_thd, STACK_MIN_SIZE, (uchar*)&buffer))
+    return 1;
+
   while (n_objects--)
   {
     uint32 wkb_type;
@@ -3841,6 +3859,9 @@ bool Gis_geometry_collection::dimension(uint32 *res_dim, const char **end) const
   n_objects= uint4korr(data);
   data+= 4;
 
+  if (check_stack_overrun(current_thd, STACK_MIN_SIZE, (uchar*)&buffer))
+    return 1;
+
   *res_dim= 0;
   while (n_objects--)
   {
@@ -3887,6 +3908,9 @@ int Gis_geometry_collection::store_shapes(Gcalc_shape_transporter *trn) const
     return 0;
   }
 
+  if (check_stack_overrun(current_thd, STACK_MIN_SIZE, (uchar*)&buffer))
+   return 1;
+
   if (trn->start_collection(n_objects))
     return 1;