factor: fix suffix validation on large integers

* src/factor.c (strtouuint): Return LONGINT_OVERFLOW only if all of the
characters are digits; otherwise, return LONGINT_INVALID.
* tests/factor/factor.pl: Add two test cases.
* NEWS: Mention the bug fix.
Discovered by Anthropic's OSS Scanner, as bug ANT-2026-IZ8PFQ7Q.

NEWS

@@ -13,6 +13,11 @@ GNU coreutils NEWS                                    -*- outline -*-
   the \u0085 (next line) character as a separator on non-GLIBC platforms.
   [bug introduced in coreutils-9.11]
 
+  'factor' no longer treats inputs requiring the use of arbitrary-precision
+  integers followed by a non-numeric suffix as zero if the suffix includes
+  non-space characters, or the number itself when followed by spaces.
+  [bugs introduced in coreutils-9.8]
+
   'fold' no longer prematurely wraps lines with a wide character towards
   the start of a line, followed by multiple backspaces.  Also backspaces
   after a tab character are accounted for correctly.

src/factor.c

@@ -1635,7 +1635,9 @@ mp_factor (mpz_t t)
 }
 
 /* Convert to *U the value represented by S, and return LONGINT_OK.
-   However, on error simply return a value other than LONGINT_OK.  */
+   Return LONGINT_OVERFLOW if S is composed of all digits but is too
+   large to fit in *U, otherwise LONGINT_INVALID if S is not a valid
+   positive integer.  */
 static strtol_error
 strtouuint (uuint *u, char const *s)
 {
@@ -1647,7 +1649,7 @@ strtouuint (uuint *u, char const *s)
   for (; LIKELY (0 <= *s - '0' && *s - '0' <= 9); s++)
     {
       if (UNLIKELY (ckd_mul (&hi, hi, 10)))
-        return LONGINT_OVERFLOW;
+        goto overflow;
 
       int lo_carry = (lo >> (W_TYPE_SIZE - 3)) + (lo >> (W_TYPE_SIZE - 1));
       lo_carry += 10 * lo < 2 * lo;
@@ -1655,7 +1657,7 @@ strtouuint (uuint *u, char const *s)
       lo = 10 * lo;
       lo_carry += ckd_add (&lo, lo, *s - '0');
       if (UNLIKELY (ckd_add (&hi, hi, lo_carry)))
-        return LONGINT_OVERFLOW;
+        goto overflow;
     }
 
   if (UNLIKELY (*s))
@@ -1663,6 +1665,11 @@ strtouuint (uuint *u, char const *s)
 
   *u = make_uuint (hi, lo);
   return LONGINT_OK;
+
+ overflow:
+  /* Since s[0] caused the overflow, we know it is a valid integer.  */
+  s++;
+  return s[strspn (s, "0123456789")] ? LONGINT_INVALID : LONGINT_OVERFLOW;
 }
 
 /* FACTOR_PIPE_BUF is chosen to give good performance,
@@ -1892,6 +1899,7 @@ print_factors (char const *input)
      way.  */
   uuint u;
   strtol_error err = strtouuint (&u, str);
+  mpz_t t;
 
   switch (err)
     {
@@ -1905,7 +1913,9 @@ print_factors (char const *input)
       FALLTHROUGH;
     case LONGINT_OVERFLOW:
       /* Try GMP.  */
-      break;
+      if (0 <= mpz_init_set_str (t, str, 10))
+        break;
+      FALLTHROUGH;
 
     case LONGINT_INVALID:
     case LONGINT_INVALID_SUFFIX_CHAR:
@@ -1916,8 +1926,6 @@ print_factors (char const *input)
     }
 
   devmsg ("[using arbitrary-precision arithmetic] ");
-  mpz_t t;
-  mpz_init_set_str (t, str, 10);
   if (MP_FACTOR_USING_POLLARD_RHO_N_MAX < mp_size (t))
     xalloc_die ();
 

tests/factor/factor.pl

@@ -84,6 +84,19 @@ my @Tests =
       {OUT => '5 4073 18899 522591721 63874247821'}],
      ['bug-gmp-2_sup_128', '340282366920938463463374607431768211456',
       {OUT => '2 'x127 . '2'}],
+
+     # v9.8 to v9.12 inclusive would treat these as 0.
+     ['bug-gmp-2_sup_128-suffix-char',
+      '340282366920938463463374607431768211456a',
+      {EXIT => 1},
+      {ERR => ("$prog: '340282366920938463463374607431768211456a'"
+               . " is not a valid positive integer\n")}],
+     ['bug-gmp-2_sup_128-suffix-space',
+      '340282366920938463463374607431768211456 ',
+      {EXIT => 1},
+      {ERR => ("$prog: '340282366920938463463374607431768211456 '"
+               . " is not a valid positive integer\n")}],
+
      ['bug-gmp-2_sup_256',
       '115792089237316195423570985008687907853'
       . '269984665640564039457584007913129639936',