Files
coreutils/tests/wc/wc.pl
T
Pádraig Brady 3aae63af9a wc: optimize ASCII and UTF8 character counting
Generate ASCII and UTF-8 data:
  $ yes $(yes eeeaae | head -n10K | paste -s -d,) | head -n10K > ll.in
  $ yes $(yes éééááé | head -n9 | paste -s -d,) | head -n1M > mb.in

ASCII is 14x faster:
  $ time src/wc-before -m < ll.in
  real	0m1.801s
  $ time src/wc-after -m < ll.in
  real	0m0.123s

UTF-8 is 7x faster:
  $ time src/wc-before -m < mb.in
  real	0m1.399s
  $ time src/wc-after -m < mb.in
  real	0m0.189s

* src/wc.c (wc): Use the u8_count() routine like cut(1),
to optimize counting valid UTF-8 characters.
* tests/wc/wc.pl: Add a test case.
* NEWS: Mention the improvement.
2026-10-03 15:47:15 +01:00

128 lines
4.4 KiB
Perl
Executable File

#!/usr/bin/perl
# Basic tests for "wc".
# Copyright (C) 1997-2026 Free Software Foundation, Inc.
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <https://www.gnu.org/licenses/>.
use strict;
my $prog = 'wc';
# Turn off localization of executable's output.
@ENV{qw(LANGUAGE LANG LC_ALL)} = ('C') x 3;
my $mb_locale = $ENV{LOCALE_FR_UTF8};
my $single_byte_locale = 'C';
{
my $codeset = qx(LC_ALL=C locale charmap 2>/dev/null);
chomp $codeset;
if ($codeset eq 'UTF-8')
{
my $fr_locale = $ENV{LOCALE_FR};
$single_byte_locale
= defined $fr_locale && $fr_locale ne 'none' ? $fr_locale : undef;
}
}
my @Tests =
(
['a0', '-c', {IN_PIPE=>''}, {OUT=>"0\n"}],
['a1', '-l', {IN_PIPE=>''}, {OUT=>"0\n"}],
['a2', '-w', {IN_PIPE=>''}, {OUT=>"0\n"}],
['a3', '-c', {IN_PIPE=>'x'}, {OUT=>"1\n"}],
['a4', '-w', {IN_PIPE=>'x'}, {OUT=>"1\n"}],
['a5', '-w', {IN_PIPE=>"x y\n"}, {OUT=>"2\n"}],
['a6', '-w', {IN_PIPE=>"x y\nz"}, {OUT=>"3\n"}],
# Remember, -l counts *newline* bytes, not logical lines.
['a7', '-l', {IN_PIPE=>"x y"}, {OUT=>"0\n"}],
['a8', '-l', {IN_PIPE=>"x y\n"}, {OUT=>"1\n"}],
['a9', '-l', {IN_PIPE=>"x\ny\n"}, {OUT=>"2\n"}],
['b0', '', {IN_PIPE=>""}, {OUT=>" 0 0 0\n"}],
['b1', '', {IN_PIPE=>"a b\nc\n"}, {OUT=>" 2 3 6\n"}],
['c0', '-L', {IN_PIPE=>"1\n12\n"}, {OUT=>"2\n"}],
['c1', '-L', {IN_PIPE=>"1\n123\n1\n"}, {OUT=>"3\n"}],
['c2', '-L', {IN_PIPE=>"\n123456"}, {OUT=>"6\n"}],
['d1', '-w', {IN_PIPE=>"\1\n"}, {OUT=>"1\n"}],
);
# Test the behavior of -m with a multi-byte locale.
my @mbTests =
(
# LATIN SMALL LETTER A
['mb-m-1byte-1', '-m', {IN_PIPE=>"\x61"}, {OUT=>"1\n"},
{ENV=>"LC_ALL=$mb_locale"}],
# NO-BREAK SPACE
['mb-m-2byte-1', '-m', {IN_PIPE=>"\xC2\xA0"}, {OUT=>"1\n"},
{ENV=>"LC_ALL=$mb_locale"}],
# SAMARITAN LETTER ALAF
['mb-m-3byte-1', '-m', {IN_PIPE=>"\xE0\xA0\x80"}, {OUT=>"1\n"},
{ENV=>"LC_ALL=$mb_locale"}],
# LINEAR B SYLLABLE B008 A
['mb-m-4byte-1', '-m', {IN_PIPE=>"\xF0\x90\x80\x80"}, {OUT=>"1\n"},
{ENV=>"LC_ALL=$mb_locale"}],
);
defined $mb_locale && $mb_locale ne 'none' and push @Tests, @mbTests;
# Make sure -m counts bytes in single-byte locales.
if (defined $single_byte_locale)
{
my @new;
my $in_len;
for my $t (@mbTests)
{
my @new_t = map { ref $_ eq 'HASH' ? { %$_ } : $_ } @$t;
(my $test_name = shift @new_t) =~ s/^mb/c/;
for my $e (@new_t)
{
next unless ref $e eq 'HASH';
$e->{ENV} = "LC_ALL=$single_byte_locale" if exists $e->{ENV};
$in_len = length $e->{IN_PIPE} if exists $e->{IN_PIPE};
}
for my $e (@new_t)
{
next unless ref $e eq 'HASH';
$e->{OUT} = "$in_len\n" if exists $e->{OUT};
}
unshift @new_t, $test_name;
push @new, \@new_t;
}
push @Tests, @new;
};
# A split EM SPACE must not leave continuation bytes to start another word.
# Separated here so we read from file and thus exercise IO_BUFSIZE.
if (defined $mb_locale && $mb_locale ne 'none')
{
my $bufsize = getlimits ()->{IO_BUFSIZE};
push @Tests,
['mb-split-space', '-w', '<',
{IN=>('a' x ($bufsize - 1)) . "\xe2\x80\x83"},
{OUT=>"1\n"}, {ENV=>"LC_ALL=$mb_locale"}];
# Ensure we handle a split character, NUL, and encoding errors.
push @Tests,
['mb-count-buffer', '-m', '<',
{IN=>('a' x ($bufsize - 1)) . "\xc3\xa9\0\xffZ\xe2\x82"},
{OUT=>($bufsize + 2) . "\n"}, {ENV=>"LC_ALL=$mb_locale"}];
}
my $save_temps = $ENV{DEBUG};
my $verbose = $ENV{VERBOSE};
my $fail = run_tests ($prog, $prog, \@Tests, $save_temps, $verbose);
exit $fail;