mirror of
https://git.savannah.gnu.org/git/coreutils.git
synced 2026-11-17 18:02:09 +03:00
dd: avoid buffer allocations unless needed
* src/dd.c: Add new static global variable ibuf. (alloc_ibuf, alloc_obuf): New functions factored from dd_copy(). (dd_copy): Call the new functions to allocate memory for ibuf and obuf when necessary. (skip): Likewise. * tests/dd/no-allocate.sh: New test. * tests/local.mk: Reference the test.
This commit is contained in:
1 parent
4114c93af3
commit
8901e010fa
3 files changed
+136
-53
No files matched your search
@@ -236,6 +236,9 @@ static uintmax_t r_truncate = 0;
|
||||
static char newline_character = '\n';
|
||||
static char space_character = ' ';
|
||||
|
||||
/* Input buffer. */
|
||||
static char *ibuf;
|
||||
|
||||
/* Output buffer. */
|
||||
static char *obuf;
|
||||
|
||||
@@ -646,6 +649,65 @@ Options are:\n\
|
||||
exit (status);
|
||||
}
|
||||
|
||||
static char *
|
||||
human_size (size_t n)
|
||||
{
|
||||
static char hbuf[LONGEST_HUMAN_READABLE + 1];
|
||||
int human_opts =
|
||||
(human_autoscale | human_round_to_nearest | human_base_1024
|
||||
| human_space_before_unit | human_SI | human_B);
|
||||
return human_readable (n, hbuf, human_opts, 1, 1);
|
||||
}
|
||||
|
||||
/* Ensure input buffer IBUF is allocated. */
|
||||
|
||||
static void
|
||||
alloc_ibuf (void)
|
||||
{
|
||||
if (ibuf)
|
||||
return;
|
||||
|
||||
char *real_buf = malloc (input_blocksize + INPUT_BLOCK_SLOP);
|
||||
if (!real_buf)
|
||||
error (EXIT_FAILURE, 0,
|
||||
_("memory exhausted by input buffer of size %zu bytes (%s)"),
|
||||
input_blocksize, human_size (input_blocksize));
|
||||
|
||||
real_buf += SWAB_ALIGN_OFFSET; /* allow space for swab */
|
||||
|
||||
ibuf = ptr_align (real_buf, page_size);
|
||||
}
|
||||
|
||||
/* Ensure output buffer OBUF is allocated/initialized. */
|
||||
|
||||
static void
|
||||
alloc_obuf (void)
|
||||
{
|
||||
if (obuf)
|
||||
return;
|
||||
|
||||
if (conversions_mask & C_TWOBUFS)
|
||||
{
|
||||
/* Page-align the output buffer, too. */
|
||||
char *real_obuf = malloc (output_blocksize + OUTPUT_BLOCK_SLOP);
|
||||
if (!real_obuf)
|
||||
error (EXIT_FAILURE, 0,
|
||||
_("memory exhausted by output buffer of size %zu bytes (%s)"),
|
||||
output_blocksize, human_size (output_blocksize));
|
||||
obuf = ptr_align (real_obuf, page_size);
|
||||
}
|
||||
else
|
||||
{
|
||||
alloc_ibuf ();
|
||||
obuf = ibuf;
|
||||
}
|
||||
|
||||
/* Write a sentinel to the slop after the buffer,
|
||||
to allow efficient checking for NUL blocks. */
|
||||
assert (sizeof (uintptr_t) <= OUTPUT_BLOCK_SLOP);
|
||||
memset (obuf + output_blocksize, 1, sizeof (uintptr_t));
|
||||
}
|
||||
|
||||
static void
|
||||
translate_charset (char const *new_trans)
|
||||
{
|
||||
@@ -1526,7 +1588,7 @@ skip_via_lseek (char const *filename, int fdesc, off_t offset, int whence)
|
||||
|
||||
/* Throw away RECORDS blocks of BLOCKSIZE bytes plus BYTES bytes on
|
||||
file descriptor FDESC, which is open with read permission for FILE.
|
||||
Store up to BLOCKSIZE bytes of the data at a time in BUF, if
|
||||
Store up to BLOCKSIZE bytes of the data at a time in IBUF or OBUF, if
|
||||
necessary. RECORDS or BYTES must be nonzero. If FDESC is
|
||||
STDIN_FILENO, advance the input offset. Return the number of
|
||||
records remaining, i.e., that were not skipped because EOF was
|
||||
@@ -1535,7 +1597,7 @@ skip_via_lseek (char const *filename, int fdesc, off_t offset, int whence)
|
||||
|
||||
static uintmax_t
|
||||
skip (int fdesc, char const *file, uintmax_t records, size_t blocksize,
|
||||
size_t *bytes, char *buf)
|
||||
size_t *bytes)
|
||||
{
|
||||
uintmax_t offset = records * blocksize + *bytes;
|
||||
|
||||
@@ -1607,6 +1669,18 @@ skip (int fdesc, char const *file, uintmax_t records, size_t blocksize,
|
||||
}
|
||||
/* else file_size && offset > OFF_T_MAX or file ! seekable */
|
||||
|
||||
char *buf;
|
||||
if (fdesc == STDIN_FILENO)
|
||||
{
|
||||
alloc_ibuf ();
|
||||
buf = ibuf;
|
||||
}
|
||||
else
|
||||
{
|
||||
alloc_obuf ();
|
||||
buf = obuf;
|
||||
}
|
||||
|
||||
do
|
||||
{
|
||||
ssize_t nread = iread_fnc (fdesc, buf, records ? blocksize : *bytes);
|
||||
@@ -1823,26 +1897,12 @@ set_fd_flags (int fd, int add_flags, char const *name)
|
||||
}
|
||||
}
|
||||
|
||||
static char *
|
||||
human_size (size_t n)
|
||||
{
|
||||
static char hbuf[LONGEST_HUMAN_READABLE + 1];
|
||||
int human_opts =
|
||||
(human_autoscale | human_round_to_nearest | human_base_1024
|
||||
| human_space_before_unit | human_SI | human_B);
|
||||
return human_readable (n, hbuf, human_opts, 1, 1);
|
||||
}
|
||||
|
||||
/* The main loop. */
|
||||
|
||||
static int
|
||||
dd_copy (void)
|
||||
{
|
||||
char *ibuf, *bufstart; /* Input buffer. */
|
||||
/* These are declared static so that even though we don't free the
|
||||
buffers, valgrind will recognize that there is no "real" leak. */
|
||||
static char *real_buf; /* real buffer address before alignment */
|
||||
static char *real_obuf;
|
||||
char *bufstart; /* Input buffer. */
|
||||
ssize_t nread; /* Bytes read in the current block. */
|
||||
|
||||
/* If nonzero, then the previously read block was partial and
|
||||
@@ -1869,45 +1929,12 @@ dd_copy (void)
|
||||
It is necessary when accessing raw (i.e. character special) disk
|
||||
devices on Unixware or other SVR4-derived system. */
|
||||
|
||||
real_buf = malloc (input_blocksize + INPUT_BLOCK_SLOP);
|
||||
if (!real_buf)
|
||||
error (EXIT_FAILURE, 0,
|
||||
_("memory exhausted by input buffer of size %zu bytes (%s)"),
|
||||
input_blocksize, human_size (input_blocksize));
|
||||
|
||||
ibuf = real_buf;
|
||||
ibuf += SWAB_ALIGN_OFFSET; /* allow space for swab */
|
||||
|
||||
ibuf = ptr_align (ibuf, page_size);
|
||||
|
||||
if (conversions_mask & C_TWOBUFS)
|
||||
{
|
||||
/* Page-align the output buffer, too. */
|
||||
real_obuf = malloc (output_blocksize + OUTPUT_BLOCK_SLOP);
|
||||
if (!real_obuf)
|
||||
error (EXIT_FAILURE, 0,
|
||||
_("memory exhausted by output buffer of size %zu bytes (%s)"),
|
||||
output_blocksize, human_size (output_blocksize));
|
||||
obuf = ptr_align (real_obuf, page_size);
|
||||
}
|
||||
else
|
||||
{
|
||||
real_obuf = NULL;
|
||||
obuf = ibuf;
|
||||
}
|
||||
|
||||
/* Write a sentinel to the slop after the buffer,
|
||||
to allow efficient checking for NUL blocks. */
|
||||
assert (sizeof (uintptr_t) <= OUTPUT_BLOCK_SLOP);
|
||||
memset (obuf + output_blocksize, 1, sizeof (uintptr_t));
|
||||
|
||||
if (skip_records != 0 || skip_bytes != 0)
|
||||
{
|
||||
uintmax_t us_bytes = input_offset + (skip_records * input_blocksize)
|
||||
+ skip_bytes;
|
||||
uintmax_t us_blocks = skip (STDIN_FILENO, input_file,
|
||||
skip_records, input_blocksize, &skip_bytes,
|
||||
ibuf);
|
||||
skip_records, input_blocksize, &skip_bytes);
|
||||
us_bytes -= input_offset;
|
||||
|
||||
/* POSIX doesn't say what to do when dd detects it has been
|
||||
@@ -1927,8 +1954,7 @@ dd_copy (void)
|
||||
{
|
||||
size_t bytes = seek_bytes;
|
||||
uintmax_t write_records = skip (STDOUT_FILENO, output_file,
|
||||
seek_records, output_blocksize, &bytes,
|
||||
obuf);
|
||||
seek_records, output_blocksize, &bytes);
|
||||
|
||||
if (write_records != 0 || bytes != 0)
|
||||
{
|
||||
@@ -1955,6 +1981,9 @@ dd_copy (void)
|
||||
if (max_records == 0 && max_bytes == 0)
|
||||
return exit_status;
|
||||
|
||||
alloc_ibuf ();
|
||||
alloc_obuf ();
|
||||
|
||||
while (1)
|
||||
{
|
||||
if (r_partial + r_full >= max_records + !!max_bytes)
|
||||
|
||||
Executable
+53
@@ -0,0 +1,53 @@
|
||||
#!/bin/sh
|
||||
# make sure that dd doesn't allocate memory unnecessarily
|
||||
|
||||
# Copyright (C) 2013 Free Software Foundation, Inc.
|
||||
|
||||
# This program is free software: you can redistribute it and/or modify
|
||||
# it under the terms of the GNU General Public License as published by
|
||||
# the Free Software Foundation, either version 3 of the License, or
|
||||
# (at your option) any later version.
|
||||
|
||||
# This program is distributed in the hope that it will be useful,
|
||||
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
# GNU General Public License for more details.
|
||||
|
||||
# You should have received a copy of the GNU General Public License
|
||||
# along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
. "${srcdir=.}/tests/init.sh"; path_prepend_ ./src
|
||||
print_ver_ dd
|
||||
require_ulimit_
|
||||
|
||||
# count and skip is zero, we don't need to allocate memory
|
||||
(ulimit -v 20000; dd bs=30M count=0) || fail=1
|
||||
(ulimit -v 20000; dd ibs=30M count=0) || fail=1
|
||||
(ulimit -v 20000; dd obs=30M count=0) || fail=1
|
||||
|
||||
|
||||
# Use a fifo for which seek fails, but read does not
|
||||
if mkfifo tape; then
|
||||
# for non seekable output we need to allocate buffer when needed
|
||||
echo 1 > tape&
|
||||
(ulimit -v 20000; dd bs=30M skip=1 count=0 if=tape) && fail=1
|
||||
|
||||
echo 1 > tape&
|
||||
(ulimit -v 20000; dd ibs=30M skip=1 count=0 if=tape) && fail=1
|
||||
|
||||
echo 1 > tape&
|
||||
(ulimit -v 20000; dd obs=30M skip=1 count=0 if=tape) || fail=1
|
||||
|
||||
|
||||
# for non seekable output we need to allocate buffer when needed
|
||||
echo 1 > tape&
|
||||
(ulimit -v 20000; dd bs=30M seek=1 count=0 of=tape) && fail=1
|
||||
|
||||
echo 1 > tape&
|
||||
(ulimit -v 20000; dd obs=30M seek=1 count=0 of=tape) && fail=1
|
||||
|
||||
echo 1 > tape&
|
||||
(ulimit -v 20000; dd ibs=30M seek=1 count=0 of=tape) || fail=1
|
||||
fi
|
||||
|
||||
Exit $fail
|
||||
@@ -465,6 +465,7 @@ all_tests = \
|
||||
tests/df/skip-rootfs.sh \
|
||||
tests/dd/direct.sh \
|
||||
tests/dd/misc.sh \
|
||||
tests/dd/no-allocate.sh \
|
||||
tests/dd/nocache.sh \
|
||||
tests/dd/not-rewound.sh \
|
||||
tests/dd/reblock.sh \
|
||||
|
||||
Reference in new issue
Block a user