mirror of
https://git.savannah.gnu.org/git/coreutils.git
synced 2026-11-17 18:02:09 +03:00
dd: add support for the conv=sparse option
Notes: Small seeks are not coalesced to larger ones, like is done in cache_round() for example. conv= is used rather then oflag= for FreeBSD compatibility. * src/dd.c (final_op_was_seek): A new global boolean to flag whether the final "write" was converted to a seek. (usage): Describe the new conf=sparse option. (iwrite): Convert a write of a NUL block to a seek if requested. (do_copy): Initialize the output buffer to have a sentinel, to allow for efficient testing for NUL output blocks. If the last block in the file was converted to a seek, then convert back to a write so the size is updated. * NEWS: Mention the new feature. * tests/dd/sparse: A new test for the feature. * tests/Makefile.am: Reference the new test.
This commit is contained in:
1 parent
697d00c321
commit
4e776faa84
5 files changed
+158
-3
No files matched your search
@@ -7,6 +7,9 @@ GNU coreutils NEWS -*- outline -*-
|
||||
dd now accepts the count_bytes, skip_bytes iflags and the seek_bytes
|
||||
oflag, to more easily allow processing portions of a file.
|
||||
|
||||
dd now accepts the conv=sparse flag to attempt to create sparse
|
||||
output, by seeking rather than writing to the output file.
|
||||
|
||||
split now accepts an optional "from" argument to --numeric-suffixes,
|
||||
which changes the start number from the default of 0.
|
||||
|
||||
|
||||
@@ -8140,6 +8140,17 @@ Change lowercase letters to uppercase.
|
||||
|
||||
The @samp{lcase} and @samp{ucase} conversions are mutually exclusive.
|
||||
|
||||
@item sparse
|
||||
@opindex sparse
|
||||
Try to seek rather than write @sc{nul} output blocks.
|
||||
On a file system that supports sparse files, this will create
|
||||
sparse output when extending the output file.
|
||||
Be careful when using this option in conjunction with
|
||||
@samp{conv=notrunc} or @samp{oflag=append}.
|
||||
With @samp{conv=notrunc}, existing data in the output
|
||||
corresponding to @sc{nul} blocks from the input, will be untouched.
|
||||
With @samp{oflag=append} the seeks performed will be ineffective.
|
||||
|
||||
@item swab
|
||||
@opindex swab @r{(byte-swapping)}
|
||||
@cindex byte-swapping
|
||||
|
||||
@@ -20,6 +20,7 @@
|
||||
|
||||
#define SWAB_ALIGN_OFFSET 2
|
||||
|
||||
#include <assert.h>
|
||||
#include <sys/types.h>
|
||||
#include <signal.h>
|
||||
#include <getopt.h>
|
||||
@@ -126,7 +127,9 @@ enum
|
||||
C_NOCREAT = 010000,
|
||||
C_EXCL = 020000,
|
||||
C_FDATASYNC = 040000,
|
||||
C_FSYNC = 0100000
|
||||
C_FSYNC = 0100000,
|
||||
|
||||
C_SPARSE = 0200000
|
||||
};
|
||||
|
||||
/* Status bit masks. */
|
||||
@@ -167,6 +170,9 @@ static uintmax_t seek_records = 0;
|
||||
output. */
|
||||
static uintmax_t seek_bytes = 0;
|
||||
|
||||
/* Whether the final output was done with a seek (rather than a write). */
|
||||
static bool final_op_was_seek;
|
||||
|
||||
/* Copy only this many records. The default is effectively infinity. */
|
||||
static uintmax_t max_records = (uintmax_t) -1;
|
||||
|
||||
@@ -271,6 +277,7 @@ static struct symbol_value const conversions[] =
|
||||
{"unblock", C_UNBLOCK | C_TWOBUFS}, /* Fixed to variable length records. */
|
||||
{"lcase", C_LCASE | C_TWOBUFS}, /* Translate upper to lower case. */
|
||||
{"ucase", C_UCASE | C_TWOBUFS}, /* Translate lower to upper case. */
|
||||
{"sparse", C_SPARSE}, /* Try to sparsely write output. */
|
||||
{"swab", C_SWAB | C_TWOBUFS}, /* Swap bytes of input. */
|
||||
{"noerror", C_NOERROR}, /* Ignore i/o errors. */
|
||||
{"nocreat", C_NOCREAT}, /* Do not create output file. */
|
||||
@@ -548,6 +555,7 @@ Each CONV symbol may be:\n\
|
||||
unblock replace trailing spaces in cbs-size records with newline\n\
|
||||
lcase change upper case to lower case\n\
|
||||
ucase change lower case to upper case\n\
|
||||
sparse try to seek rather than write the output for NUL input blocks\n\
|
||||
swab swap every pair of input bytes\n\
|
||||
sync pad every input block with NULs to ibs-size; when used\n\
|
||||
with block or unblock, pad with spaces rather than NULs\n\
|
||||
@@ -989,6 +997,27 @@ iread_fullblock (int fd, char *buf, size_t size)
|
||||
return nread;
|
||||
}
|
||||
|
||||
/* Return whether the buffer consists entirely of NULs.
|
||||
Note the word after the buffer must be non NUL. */
|
||||
|
||||
static bool _GL_ATTRIBUTE_PURE
|
||||
is_nul (const char *buf, size_t bufsize)
|
||||
{
|
||||
typedef uintptr_t word;
|
||||
|
||||
/* Find first nonzero *word*, or the word with the sentinel. */
|
||||
word *wp = (word *) buf;
|
||||
while (*wp++ == 0)
|
||||
continue;
|
||||
|
||||
/* Find the first nonzero *byte*, or the sentinel. */
|
||||
char *cp = (char *) (wp - 1);
|
||||
while (*cp++ == 0)
|
||||
continue;
|
||||
|
||||
return cp > buf + bufsize;
|
||||
}
|
||||
|
||||
/* Write to FD the buffer BUF of size SIZE, processing any signals
|
||||
that arrive. Return the number of bytes written, setting errno if
|
||||
this is less than SIZE. Keep trying if there are partial
|
||||
@@ -1020,9 +1049,28 @@ iwrite (int fd, char const *buf, size_t size)
|
||||
|
||||
while (total_written < size)
|
||||
{
|
||||
ssize_t nwritten;
|
||||
ssize_t nwritten = 0;
|
||||
process_signals ();
|
||||
nwritten = write (fd, buf + total_written, size - total_written);
|
||||
|
||||
/* Perform a seek for a NUL block if sparse output is enabled. */
|
||||
final_op_was_seek = false;
|
||||
if ((conversions_mask & C_SPARSE) && is_nul (buf, size))
|
||||
{
|
||||
if (lseek (fd, size, SEEK_CUR) < 0)
|
||||
{
|
||||
conversions_mask &= ~C_SPARSE;
|
||||
/* Don't warn about the advisory sparse request. */
|
||||
}
|
||||
else
|
||||
{
|
||||
final_op_was_seek = true;
|
||||
nwritten = size;
|
||||
}
|
||||
}
|
||||
|
||||
if (!nwritten)
|
||||
nwritten = write (fd, buf + total_written, size - total_written);
|
||||
|
||||
if (nwritten < 0)
|
||||
{
|
||||
if (errno != EINTR)
|
||||
@@ -1861,6 +1909,11 @@ dd_copy (void)
|
||||
obuf = ibuf;
|
||||
}
|
||||
|
||||
/* Write a sentinel to the slop after the buffer,
|
||||
to allow efficient checking for NUL blocks. */
|
||||
assert (sizeof (uintptr_t) <= OUTPUT_BLOCK_SLOP);
|
||||
memset (obuf + output_blocksize, 1, sizeof (uintptr_t));
|
||||
|
||||
if (skip_records != 0 || skip_bytes != 0)
|
||||
{
|
||||
uintmax_t us_bytes = input_offset + (skip_records * input_blocksize)
|
||||
@@ -2072,6 +2125,33 @@ dd_copy (void)
|
||||
}
|
||||
}
|
||||
|
||||
/* If the last write was converted to a seek, then for a regular file,
|
||||
ftruncate to extend the size. */
|
||||
if (final_op_was_seek)
|
||||
{
|
||||
struct stat stdout_stat;
|
||||
if (fstat (STDOUT_FILENO, &stdout_stat) != 0)
|
||||
{
|
||||
error (0, errno, _("cannot fstat %s"), quote (output_file));
|
||||
return EXIT_FAILURE;
|
||||
}
|
||||
if (S_ISREG (stdout_stat.st_mode))
|
||||
{
|
||||
off_t output_offset = lseek (STDOUT_FILENO, 0, SEEK_CUR);
|
||||
if (output_offset > stdout_stat.st_size)
|
||||
{
|
||||
if (ftruncate (STDOUT_FILENO, output_offset) != 0)
|
||||
{
|
||||
error (0, errno,
|
||||
_("failed to truncate to %"PRIuMAX" bytes"
|
||||
" in output file %s"),
|
||||
output_offset, quote (output_file));
|
||||
return EXIT_FAILURE;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if ((conversions_mask & C_FDATASYNC) && fdatasync (STDOUT_FILENO) != 0)
|
||||
{
|
||||
if (errno != ENOSYS && errno != EINVAL)
|
||||
|
||||
@@ -373,6 +373,7 @@ TESTS = \
|
||||
dd/skip-seek2 \
|
||||
dd/bytes \
|
||||
dd/skip-seek-past-file \
|
||||
dd/sparse \
|
||||
dd/stderr \
|
||||
dd/unblock \
|
||||
dd/unblock-sync \
|
||||
|
||||
Executable
+60
@@ -0,0 +1,60 @@
|
||||
#!/bin/sh
|
||||
|
||||
# Copyright (C) 2012 Free Software Foundation, Inc.
|
||||
|
||||
# This program is free software: you can redistribute it and/or modify
|
||||
# it under the terms of the GNU General Public License as published by
|
||||
# the Free Software Foundation, either version 3 of the License, or
|
||||
# (at your option) any later version.
|
||||
|
||||
# This program is distributed in the hope that it will be useful,
|
||||
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
# GNU General Public License for more details.
|
||||
|
||||
# You should have received a copy of the GNU General Public License
|
||||
# along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
. "${srcdir=.}/init.sh"; path_prepend_ ../src
|
||||
print_ver_ dd
|
||||
require_sparse_support_
|
||||
|
||||
# Ensure basic sparse generation works
|
||||
truncate -s1M sparse
|
||||
dd bs=32K if=sparse of=sparse.dd conv=sparse
|
||||
test $(stat -c %s sparse) = $(stat -c %s sparse.dd) || fail=1
|
||||
|
||||
# Demonstrate that conv=sparse with oflag=append,
|
||||
# will do ineffective seeks in the output
|
||||
printf 'a\000\000b' > file.in
|
||||
printf 'ab' > exp
|
||||
dd if=file.in bs=1 conv=sparse oflag=append > out
|
||||
compare exp out || fail=1
|
||||
|
||||
# Demonstrate conv=sparse with conv=notrunc,
|
||||
# where data in file.out is not overwritten with NULs
|
||||
printf '____' > out
|
||||
printf 'a__b' > exp
|
||||
dd if=file.in bs=1 conv=sparse,notrunc of=out
|
||||
compare exp out || fail=1
|
||||
|
||||
# Ensure we fall back to write if seek fails
|
||||
dd if=file.in bs=1 conv=sparse | cat > file.out
|
||||
cmp file.in file.out || fail=1
|
||||
|
||||
# Setup for block size tests
|
||||
dd if=/dev/urandom of=file.in bs=1M count=1
|
||||
truncate -s+1M file.in
|
||||
dd if=/dev/urandom of=file.in bs=1M count=1 conv=notrunc oflag=append
|
||||
|
||||
# Ensure NUL blocks smaller than the block size are not made sparse
|
||||
dd if=file.in of=file.out bs=2M conv=sparse
|
||||
test $(stat -c %s file.in) = $(stat -c %s file.out) || fail=1
|
||||
test $(stat -c %b file.in) = $(stat -c %b file.out) && fail=1
|
||||
|
||||
# Ensure NUL blocks >= block size are made sparse
|
||||
dd if=file.in of=file.out bs=1M conv=sparse
|
||||
test $(stat -c %s file.in) = $(stat -c %s file.out) || fail=1
|
||||
test $(stat -c %b file.in) = $(stat -c %b file.out) || fail=1
|
||||
|
||||
Exit $fail
|
||||
Reference in new issue
Block a user