From 50d95bf60ab5a9dc59df31af5aec45394c4cc727 Mon Sep 17 00:00:00 2001
From: Sergey Zefirov <72864488+mariadb-SergeyZefirov@users.noreply.github.com>
Date: Mon, 22 Aug 2022 21:34:38 +0300
Subject: [PATCH] MCOL-5196 REPLACE function may trigger invalid capacity
 assertion (#2522)

When length of string to replace minus length of string to replace to is
bigger than input string and processing mode allows for binary (memcmp
or std::string::find()) comparison, REPLACE may trigger invalid capacity
assertion and query processing will stop.

The fix is to properly count the number of occurences of the string to
replace, basically.
---
 mysql-test/columnstore/basic/r/mcol-5196.result |  9 +++++++++
 mysql-test/columnstore/basic/t/mcol-5196.test   | 15 +++++++++++++++
 utils/funcexp/func_replace.cpp                  | 17 ++++++++++-------
 3 files changed, 34 insertions(+), 7 deletions(-)
 create mode 100644 mysql-test/columnstore/basic/r/mcol-5196.result
 create mode 100644 mysql-test/columnstore/basic/t/mcol-5196.test

diff --git a/mysql-test/columnstore/basic/r/mcol-5196.result b/mysql-test/columnstore/basic/r/mcol-5196.result
new file mode 100644
index 000000000..8070fde6c
--- /dev/null
+++ b/mysql-test/columnstore/basic/r/mcol-5196.result
@@ -0,0 +1,9 @@
+DROP DATABASE IF EXISTS mcol5196;
+CREATE DATABASE mcol5196;
+USE mcol5196;
+CREATE TABLE t1 (a VARCHAR(7) CHARACTER SET latin1 COLLATE latin1_bin) ENGINE=COLUMNSTORE;
+INSERT INTO t1(a) VALUES ('aa');
+SELECT REPLACE(a, 'pqrs', 'a') FROM t1;
+REPLACE(a, 'pqrs', 'a')
+aa
+DROP DATABASE mcol5196;
diff --git a/mysql-test/columnstore/basic/t/mcol-5196.test b/mysql-test/columnstore/basic/t/mcol-5196.test
new file mode 100644
index 000000000..b5e56bdb8
--- /dev/null
+++ b/mysql-test/columnstore/basic/t/mcol-5196.test
@@ -0,0 +1,15 @@
+--disable_warnings
+DROP DATABASE IF EXISTS mcol5196;
+--enable_warnings
+
+CREATE DATABASE mcol5196;
+
+USE mcol5196;
+
+CREATE TABLE t1 (a VARCHAR(7) CHARACTER SET latin1 COLLATE latin1_bin) ENGINE=COLUMNSTORE;
+
+INSERT INTO t1(a) VALUES ('aa');
+
+SELECT REPLACE(a, 'pqrs', 'a') FROM t1;
+
+DROP DATABASE mcol5196;
diff --git a/utils/funcexp/func_replace.cpp b/utils/funcexp/func_replace.cpp
index da18fb3f1..afcb17f42 100644
--- a/utils/funcexp/func_replace.cpp
+++ b/utils/funcexp/func_replace.cpp
@@ -71,19 +71,22 @@ std::string Func_replace::getStrVal(rowgroup::Row& row, FunctionParm& fp, bool&
   if (binaryCmp)
   {
     // Count the number of fromstr in strend so we can reserve buffer space.
-    int count = 0;
-    do
+    size_t count = 0;
+    while ( string::npos != (pos = str.find(fromstr, pos)))
     {
       ++count;
-      pos = str.find(fromstr, pos + fromLen);
-    } while (pos != string::npos);
+      pos += fromLen;
+    }
 
-    newstr.reserve(strLen + (count * ((int)toLen - (int)fromLen)) + 1);
+    if (count == 0)
+    {
+      return str;
+    }
+
+    newstr.reserve(strLen + (count * (toLen - fromLen)) + 1);
 
     uint32_t i = 0;
     pos = str.find(fromstr);
-    if (pos == string::npos)
-      return str;
     // Move the stuff into newstr
     do
     {