mirror of
https://git.savannah.gnu.org/git/coreutils.git
synced 2026-11-17 18:02:09 +03:00
cut: add utf8 helper to mbbuf
* gl/lib/mbbuf.h: To safely search a bounded buffer, without needed to have unbounded memory with getndelim2. TODO: rename mbbuf_utf8_safe_prefix to mbbuf_fill_utf8
This commit is contained in:
1 parent
801686242e
commit
ba7c1fbadb
1 file changed
+72
-9
+72
-9
@@ -49,6 +49,12 @@ typedef struct
|
||||
idx_t offset; /* Current position in BUFFER. */
|
||||
} mbbuf_t;
|
||||
|
||||
MBBUF_INLINE idx_t
|
||||
mbbuf_avail (mbbuf_t const *mbbuf)
|
||||
{
|
||||
return mbbuf->length - mbbuf->offset;
|
||||
}
|
||||
|
||||
/* Initialize MBBUF with an allocated BUFFER of SIZE bytes and a file stream
|
||||
FP open for reading. SIZE must be greater than or equal to MCEL_LEN_MAX.
|
||||
*/
|
||||
@@ -64,15 +70,17 @@ mbbuf_init (mbbuf_t *mbbuf, char *buffer, idx_t size, FILE *fp)
|
||||
mbbuf->offset = 0;
|
||||
}
|
||||
|
||||
/* Get the next character in the buffer, filling it from FP if necessary.
|
||||
If an invalid multi-byte character is seen, we assume the program wants to
|
||||
fall back to the read byte. */
|
||||
MBBUF_INLINE mcel_t
|
||||
mbbuf_get_char (mbbuf_t *mbbuf)
|
||||
/* Fill the input buffer with at least MIN_AVAILABLE bytes if possible.
|
||||
Return the number of bytes available from the current offset. */
|
||||
MBBUF_INLINE idx_t
|
||||
mbbuf_fill (mbbuf_t *mbbuf, idx_t min_available)
|
||||
{
|
||||
idx_t available = mbbuf->length - mbbuf->offset;
|
||||
/* Check if we need to fill the input buffer. */
|
||||
if (available < MCEL_LEN_MAX && ! feof (mbbuf->fp))
|
||||
idx_t available = mbbuf_avail (mbbuf);
|
||||
|
||||
if (mbbuf->size < min_available)
|
||||
min_available = mbbuf->size;
|
||||
|
||||
if (available < min_available && ! feof (mbbuf->fp))
|
||||
{
|
||||
idx_t start;
|
||||
if (!(0 < available))
|
||||
@@ -85,8 +93,63 @@ mbbuf_get_char (mbbuf_t *mbbuf)
|
||||
mbbuf->length = fread (mbbuf->buffer + start, 1, mbbuf->size - start,
|
||||
mbbuf->fp) + start;
|
||||
mbbuf->offset = 0;
|
||||
available = mbbuf->length - mbbuf->offset;
|
||||
available = mbbuf_avail (mbbuf);
|
||||
}
|
||||
|
||||
return available;
|
||||
}
|
||||
|
||||
/* Consume N bytes from the current buffer. */
|
||||
MBBUF_INLINE void
|
||||
mbbuf_advance (mbbuf_t *mbbuf, idx_t n)
|
||||
{
|
||||
if (mbbuf_avail (mbbuf) < n)
|
||||
unreachable ();
|
||||
mbbuf->offset += n;
|
||||
}
|
||||
|
||||
/* Return the largest prefix of the current contents that is safe to process
|
||||
with byte searches, while leaving at least OVERLAP bytes unprocessed unless
|
||||
EOF has been seen. The returned prefix never ends in the middle of a UTF-8
|
||||
sequence, but it may include invalid bytes. */
|
||||
MBBUF_INLINE idx_t
|
||||
mbbuf_utf8_safe_prefix (mbbuf_t *mbbuf, idx_t overlap)
|
||||
{
|
||||
idx_t available = mbbuf_fill (mbbuf, overlap + 4);
|
||||
if (available == 0)
|
||||
return 0;
|
||||
|
||||
if (feof (mbbuf->fp))
|
||||
return available;
|
||||
|
||||
if (available <= overlap)
|
||||
return 0;
|
||||
|
||||
idx_t end = available - overlap;
|
||||
char const *buf = mbbuf->buffer + mbbuf->offset;
|
||||
idx_t start = end - 1;
|
||||
|
||||
while (0 < start
|
||||
&& ((unsigned char) buf[start] & 0xC0) == 0x80)
|
||||
start--;
|
||||
|
||||
unsigned char lead = buf[start];
|
||||
idx_t len = (lead < 0x80 ? 1
|
||||
: (lead & 0xE0) == 0xC0 ? 2
|
||||
: (lead & 0xF0) == 0xE0 ? 3
|
||||
: (lead & 0xF8) == 0xF0 ? 4
|
||||
: 1);
|
||||
|
||||
return start + len <= end ? end : start;
|
||||
}
|
||||
|
||||
/* Get the next character in the buffer, filling it from FP if necessary.
|
||||
If an invalid multi-byte character is seen, we assume the program wants to
|
||||
fall back to the read byte. */
|
||||
MBBUF_INLINE mcel_t
|
||||
mbbuf_get_char (mbbuf_t *mbbuf)
|
||||
{
|
||||
idx_t available = mbbuf_fill (mbbuf, MCEL_LEN_MAX);
|
||||
if (available <= 0)
|
||||
return (mcel_t) { .ch = MBBUF_EOF };
|
||||
mcel_t g = mcel_scan (mbbuf->buffer + mbbuf->offset,
|
||||
|
||||
Reference in new issue
Block a user