mirror of
https://git.savannah.gnu.org/git/coreutils.git
synced 2026-11-17 18:02:09 +03:00
cut: refactor -f to byte search and character processing
Not sure about this at all. Only worthwhile if can also remove cut_fields_line_delim. Refactored src/cut.c so the old UTF-8 byte-search path is now the general byte-search field engine for all safe byte-search cases: ordinary single-byte delimiters and valid UTF-8 delimiters. The old cut_fields path did not go away completely; it is now cut_fields_line_delim and is used only when the field delimiter equals the record delimiter, because -d $'\n' and -z -d '' have different semantics from normal line-based field splitting. As part of that, I also folded the duplicated “start selected field” logic into a shared helper, and renamed the byte-search helpers to match their broader use. The current dispatcher in src/cut.c is now: whitespace parser, then line- delimiter field mode, then byte-search field mode, then the decoded multibyte parser.
This commit is contained in:
1 parent
16b1ff40ae
commit
b3ef6231bd
1 file changed
+50
-34
@@ -287,10 +287,9 @@ is_range_start_index (uintmax_t k)
|
||||
}
|
||||
|
||||
static inline bool
|
||||
single_byte_field_delim_ok (void)
|
||||
field_delim_is_line_delim (void)
|
||||
{
|
||||
return delim_length == 1
|
||||
&& (MB_CUR_MAX <= 1 || to_uchar (delim_bytes[0]) < 0x30);
|
||||
return delim_length == 1 && delim_bytes[0] == line_delim;
|
||||
}
|
||||
|
||||
/* Return true if the current charset is UTF-8. */
|
||||
@@ -313,6 +312,15 @@ utf8_field_delim_ok (void)
|
||||
return ! delim_mcel.err && is_utf8_charset ();
|
||||
}
|
||||
|
||||
static inline bool
|
||||
bytesearch_field_delim_ok (void)
|
||||
{
|
||||
return ! field_delim_is_line_delim ()
|
||||
&& (delim_length == 1
|
||||
? MB_CUR_MAX <= 1 || to_uchar (delim_bytes[0]) < 0x30
|
||||
: utf8_field_delim_ok ());
|
||||
}
|
||||
|
||||
static inline bool
|
||||
field_delim_eq (mcel_t g)
|
||||
{
|
||||
@@ -497,13 +505,13 @@ scan_mb_field (mbbuf_t *mbbuf, struct mbfield_parser *parser,
|
||||
}
|
||||
}
|
||||
|
||||
/* Return a pointer to the next field delimiter in the UTF-8 record BUF,
|
||||
searching LEN bytes. Return NULL if none is found. DELIM_BYTES must
|
||||
/* Return a pointer to the next field delimiter in BUF, searching LEN bytes.
|
||||
Return NULL if none is found. DELIM_BYTES must be a single byte or
|
||||
represent a valid UTF-8 character. BUF can contain invalid/NUL bytes,
|
||||
but must be NUL terminated. */
|
||||
and must have room for a trailing NUL byte at BUF[LEN]. */
|
||||
ATTRIBUTE_PURE
|
||||
static char *
|
||||
find_utf8_field_delim (char *buf, size_t len)
|
||||
find_bytesearch_field_delim (char *buf, size_t len)
|
||||
{
|
||||
#if ! __GLIBC__ /* Only S390 has optimized memmem on glibc-2.42 */
|
||||
return memmem (buf, len, delim_bytes, delim_length);
|
||||
@@ -547,11 +555,11 @@ find_utf8_field_delim (char *buf, size_t len)
|
||||
}
|
||||
|
||||
static inline char *
|
||||
find_utf8_field_terminator (char *buf, idx_t len, bool *is_delim)
|
||||
find_bytesearch_field_terminator (char *buf, idx_t len, bool *is_delim)
|
||||
{
|
||||
char *line_end = memchr ((void *) buf, line_delim, len);
|
||||
idx_t line_len = line_end ? line_end - buf : len;
|
||||
char *field_end = find_utf8_field_delim (buf, line_len);
|
||||
char *field_end = find_bytesearch_field_delim (buf, line_len);
|
||||
|
||||
if (field_end)
|
||||
{
|
||||
@@ -564,8 +572,8 @@ find_utf8_field_terminator (char *buf, idx_t len, bool *is_delim)
|
||||
}
|
||||
|
||||
static inline bool
|
||||
begin_utf8_field (uintmax_t field_idx, bool buffer_first_field,
|
||||
bool *found_any_selected_field)
|
||||
begin_field_output (uintmax_t field_idx, bool buffer_first_field,
|
||||
bool *found_any_selected_field)
|
||||
{
|
||||
bool write_field = print_kth (field_idx);
|
||||
|
||||
@@ -579,6 +587,19 @@ begin_utf8_field (uintmax_t field_idx, bool buffer_first_field,
|
||||
return write_field;
|
||||
}
|
||||
|
||||
static inline idx_t
|
||||
bytesearch_safe_prefix (mbbuf_t *mbbuf, idx_t overlap)
|
||||
{
|
||||
idx_t available = mbbuf_fill (mbbuf, overlap + 1);
|
||||
if (available == 0)
|
||||
return 0;
|
||||
|
||||
if (feof (mbbuf->fp))
|
||||
return available;
|
||||
|
||||
return overlap < available ? available - overlap : 0;
|
||||
}
|
||||
|
||||
static inline void
|
||||
reset_field_line (uintmax_t *field_idx, bool *found_any_selected_field,
|
||||
bool *have_pending_line, struct mbfield_parser *parser)
|
||||
@@ -780,14 +801,8 @@ cut_fields_mb_any (FILE *stream, bool whitespace_mode)
|
||||
}
|
||||
|
||||
enum field_terminator terminator;
|
||||
bool write_field = print_kth (field_idx);
|
||||
|
||||
if (write_field)
|
||||
{
|
||||
if (found_any_selected_field)
|
||||
write_bytes (output_delimiter_string, output_delimiter_length);
|
||||
found_any_selected_field = true;
|
||||
}
|
||||
bool write_field = begin_field_output (field_idx, buffer_first_field,
|
||||
&found_any_selected_field);
|
||||
|
||||
terminator = scan_mb_field (&mbbuf, &parser, &have_pending_line,
|
||||
write_field, NULL);
|
||||
@@ -820,10 +835,10 @@ cut_fields_mb (FILE *stream)
|
||||
}
|
||||
|
||||
/* Read from STREAM, printing to standard output any selected fields,
|
||||
using UTF-8-aware byte searches for the field delimiter. */
|
||||
using byte searches for a single-byte or valid UTF-8 field delimiter. */
|
||||
|
||||
static void
|
||||
cut_fields_mb_utf8 (FILE *stream)
|
||||
cut_fields_bytesearch (FILE *stream)
|
||||
{
|
||||
/* Leave 1 byte unused so space to NUL terminate (for strstr). */
|
||||
static char line_in[IO_BUFSIZE+1];
|
||||
@@ -839,12 +854,12 @@ cut_fields_mb_utf8 (FILE *stream)
|
||||
current_rp = frp;
|
||||
buffer_first_field = suppress_non_delimited ^ !print_kth (1);
|
||||
mbbuf_init (&mbbuf, line_in, sizeof line_in - 1, stream);
|
||||
write_field = begin_utf8_field (field_idx, buffer_first_field,
|
||||
&found_any_selected_field);
|
||||
write_field = begin_field_output (field_idx, buffer_first_field,
|
||||
&found_any_selected_field);
|
||||
|
||||
while (true)
|
||||
{
|
||||
idx_t safe = mbbuf_utf8_safe_prefix (&mbbuf, overlap);
|
||||
idx_t safe = bytesearch_safe_prefix (&mbbuf, overlap);
|
||||
idx_t processed = 0;
|
||||
|
||||
if (safe == 0)
|
||||
@@ -860,8 +875,8 @@ cut_fields_mb_utf8 (FILE *stream)
|
||||
{
|
||||
bool is_delim = false;
|
||||
char *terminator
|
||||
= find_utf8_field_terminator (chunk + processed, safe - processed,
|
||||
&is_delim);
|
||||
= find_bytesearch_field_terminator (chunk + processed,
|
||||
safe - processed, &is_delim);
|
||||
idx_t field_len = terminator ? terminator - (chunk + processed)
|
||||
: safe - processed;
|
||||
|
||||
@@ -892,8 +907,8 @@ cut_fields_mb_utf8 (FILE *stream)
|
||||
|
||||
processed += delim_length;
|
||||
next_item (&field_idx);
|
||||
write_field = begin_utf8_field (field_idx, buffer_first_field,
|
||||
&found_any_selected_field);
|
||||
write_field = begin_field_output (field_idx, buffer_first_field,
|
||||
&found_any_selected_field);
|
||||
}
|
||||
else
|
||||
{
|
||||
@@ -916,8 +931,8 @@ cut_fields_mb_utf8 (FILE *stream)
|
||||
current_rp = frp;
|
||||
found_any_selected_field = false;
|
||||
have_pending_line = false;
|
||||
write_field = begin_utf8_field (field_idx, buffer_first_field,
|
||||
&found_any_selected_field);
|
||||
write_field = begin_field_output (field_idx, buffer_first_field,
|
||||
&found_any_selected_field);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -952,10 +967,11 @@ cut_fields_ws (FILE *stream)
|
||||
cut_fields_mb_any (stream, true);
|
||||
}
|
||||
|
||||
/* Read from stream STREAM, printing to standard output any selected fields. */
|
||||
/* Read from stream STREAM, printing to standard output any selected fields,
|
||||
using the line delimiter as the field delimiter. */
|
||||
|
||||
static void
|
||||
cut_fields (FILE *stream)
|
||||
cut_fields_line_delim (FILE *stream)
|
||||
{
|
||||
int c; /* Each character from the file. */
|
||||
uintmax_t field_idx = 1;
|
||||
@@ -1329,8 +1345,8 @@ main (int argc, char **argv)
|
||||
|
||||
case CUT_MODE_FIELDS:
|
||||
cut_stream = whitespace_delimited ? cut_fields_ws
|
||||
: single_byte_field_delim_ok () ? cut_fields
|
||||
: utf8_field_delim_ok () ? cut_fields_mb_utf8
|
||||
: field_delim_is_line_delim () ? cut_fields_line_delim
|
||||
: bytesearch_field_delim_ok () ? cut_fields_bytesearch
|
||||
: cut_fields_mb;
|
||||
break;
|
||||
}
|
||||
|
||||
Reference in new issue
Block a user