copy: process empty extents more efficiently

* src/copy.c (extent_copy): Treat an allocated but empty extent
much like a hole.  I.E. don't read data we know is going to be NUL.
Also we convert the empty extent to a hole only when SPARSE_ALWAYS
so that the source and dest have the same allocation.  This will
be improved soon, when we use fallocate() to do the allocation.
* tests/cp/fiemap-empty: A new test for efficiency and correctness
of copying empty extents.
* tests/Makefile.am: Reference the new test.
* NEWS: Mention the change in behavior.
This commit is contained in:
Pádraig Brady committed 2011-04-01 15:04:18 +01:00
1 parent b56b53bd70
commit bfdb6b585b
4 files changed
+162 -16

No files matched your search

+4
View File
@@ -40,6 +40,10 @@ GNU coreutils NEWS -*- outline -*-
The sync in only needed on Linux kernels before 2.6.38.
[The sync was introduced in coreutils-8.10]
cp now copies empty extents efficiently, when doing a FIEMAP copy.
It no longer reads the zero bytes from the input, and also can efficiently
create a hole in the output file when --sparse=always is specified.
df now aligns columns consistently, and no longer wraps entries
with longer device identifiers, over two lines.
+66 -14
View File
@@ -39,6 +39,7 @@
#include "extent-scan.h"
#include "error.h"
#include "fcntl--.h"
#include "fiemap.h"
#include "file-set.h"
#include "filemode.h"
#include "filenamecat.h"
@@ -330,11 +331,28 @@ extent_copy (int src_fd, int dest_fd, char *buf, size_t buf_size,
}
unsigned int i;
for (i = 0; i < scan.ei_count; i++)
bool empty_extent = false;
for (i = 0; i < scan.ei_count || empty_extent; i++)
{
off_t ext_start = scan.ext_info[i].ext_logical;
uint64_t ext_len = scan.ext_info[i].ext_length;
uint64_t hole_size = ext_start - last_ext_start - last_ext_len;
off_t ext_start;
uint64_t ext_len;
uint64_t hole_size;
if (i < scan.ei_count)
{
ext_start = scan.ext_info[i].ext_logical;
ext_len = scan.ext_info[i].ext_length;
}
else /* empty extent at EOF. */
{
i--;
ext_start = last_ext_start + scan.ext_info[i].ext_length;
ext_len = 0;
}
hole_size = ext_start - last_ext_start - last_ext_len;
wrote_hole_at_eof = false;
if (hole_size)
{
@@ -346,38 +364,72 @@ extent_copy (int src_fd, int dest_fd, char *buf, size_t buf_size,
return false;
}
if (sparse_mode != SPARSE_NEVER)
if ((empty_extent && sparse_mode == SPARSE_ALWAYS)
|| (!empty_extent && sparse_mode != SPARSE_NEVER))
{
if (lseek (dest_fd, ext_start, SEEK_SET) < 0)
{
error (0, errno, _("cannot lseek %s"), quote (dst_name));
goto fail;
}
wrote_hole_at_eof = true;
}
else
{
/* When not inducing holes and when there is a hole between
the end of the previous extent and the beginning of the
current one, write zeros to the destination file. */
if (! write_zeros (dest_fd, hole_size))
off_t nzeros = hole_size;
if (empty_extent)
nzeros = MIN (src_total_size - dest_pos, hole_size);
if (! write_zeros (dest_fd, nzeros))
{
error (0, errno, _("%s: write failed"), quote (dst_name));
goto fail;
}
dest_pos = MIN (src_total_size, ext_start);
}
}
last_ext_start = ext_start;
last_ext_len = ext_len;
off_t n_read;
if ( ! sparse_copy (src_fd, dest_fd, buf, buf_size,
sparse_mode == SPARSE_ALWAYS, src_name, dst_name,
ext_len, &n_read,
&wrote_hole_at_eof))
return false;
/* Treat an unwritten but allocated extent much like a hole.
I.E. don't read, but don't convert to a hole in the destination,
unless SPARSE_ALWAYS. */
if (scan.ext_info[i].ext_flags & FIEMAP_EXTENT_UNWRITTEN)
{
empty_extent = true;
last_ext_len = 0;
if (ext_len == 0) /* The last extent is empty and processed. */
empty_extent = false;
}
else
{
off_t n_read;
empty_extent = false;
last_ext_len = ext_len;
dest_pos = ext_start + n_read;
if ( ! sparse_copy (src_fd, dest_fd, buf, buf_size,
sparse_mode == SPARSE_ALWAYS,
src_name, dst_name, ext_len, &n_read,
&wrote_hole_at_eof))
return false;
dest_pos = ext_start + n_read;
}
/* If the file ends with unwritten extents not accounted for in the
size, then skip processing them, and the associated redundant
read() calls which will always return 0. We will need to
remove this when we add fallocate() so that we can maintain
extents beyond the apparent size. */
if (dest_pos == src_total_size)
{
scan.hit_final_extent = true;
break;
}
}
/* Release the space allocated to scan->ext_info. */
+3 -2
View File
@@ -321,8 +321,9 @@ TESTS = \
cp/dir-vs-file \
cp/existing-perm-race \
cp/fail-perm \
cp/fiemap-perf \
cp/fiemap-2 \
cp/fiemap-empty \
cp/fiemap-perf \
cp/fiemap-2 \
cp/file-perm-race \
cp/into-self \
cp/link \
+89
View File
@@ -0,0 +1,89 @@
#!/bin/sh
# Test cp reads unwritten extents efficiently
# Copyright (C) 2011 Free Software Foundation, Inc.
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
. "${srcdir=.}/init.sh"; path_prepend_ ../src
print_ver_ cp
touch fiemap_chk
fiemap_capable_ fiemap_chk ||
skip_test_ 'this file system lacks FIEMAP support'
rm fiemap_chk
# TODO: rather than requiring `fallocate`, possible add
# this functionality to truncate --alloc
fallocate --help >/dev/null || skip_test_ 'The fallocate utility is required'
fallocate -l 1 -n falloc.test ||
skip_test_ 'this file system lacks FALLOCATE support'
rm falloc.test
fallocate -l 600000000 space.test ||
skip_test_ 'this test needs at least 600MB free space'
# Disable this test on old BTRFS (e.g. Fedora 14)
# which reports ordinary extents for unwritten ones.
filefrag space.test || skip_test_ 'the `filefrag` utility is missing'
filefrag -v space.test | grep -F 'unwritten' > /dev/null ||
skip_test_ 'this file system does not report empty extents as "unwritten"'
rm space.test
# Ensure we read a large empty file quickly
fallocate -l 300000000 empty.big || framework_failure
timeout 3 cp --sparse=always empty.big cp.test || fail=1
test $(stat -c %s empty.big) = $(stat -c %s cp.test) || fail=1
rm empty.big cp.test
# Ensure we handle extents beyond file size correctly.
# Note until we support fallocate, we will not maintain
# the file allocation. Ammend this test when fallocate is supported
fallocate -l 10000000 -n unwritten.withdata || framework_failure
dd count=10 if=/dev/urandom conv=notrunc iflag=fullblock of=unwritten.withdata
cp unwritten.withdata cp.test || fail=1
test $(stat -c %s unwritten.withdata) = $(stat -c %s cp.test) || fail=1
cmp unwritten.withdata cp.test || fail=1
rm unwritten.withdata cp.test
# The following to generate unaccounted extents followed by a hole, is not
# supported by ext4 at least. The ftruncate discards all extents not
# accounted for in the size.
# fallocate -l 10000000 -n unacc.withholes
# dd count=10 if=/dev/urandom conv=notrunc iflag=fullblock of=unacc.withholes
# truncate -s20000000 unacc.withholes
# Ensure we handle a hole after empty extents correctly.
# Since all extents are accounted for in the size,
# we can maintain the allocation independently from
# fallocate() support.
fallocate -l 10000000 empty.withholes
truncate -s 20000000 empty.withholes
sectors_per_block=$(expr $(stat -c %o .) / 512)
cp empty.withholes cp.test || fail=1
test $(stat -c %s empty.withholes) = $(stat -c %s cp.test) || fail=1
# These are usually equal but can vary by an IO block due to alignment
alloc_diff=$(expr $(stat -c %b empty.withholes) - $(stat -c %b cp.test))
alloc_diff=$(echo $alloc_diff | tr -d -- -) # abs()
test $alloc_diff -le $sectors_per_block || fail=1
# Again with SPARSE_ALWAYS
cp --sparse=always empty.withholes cp.test || fail=1
test $(stat -c %s empty.withholes) = $(stat -c %s cp.test) || fail=1
# cp.test should take 0 space, but allowing for some systems
# that store default extended attributes in data blocks
test $(stat -c %b cp.test) -le $sectors_per_block || fail=1
rm empty.withholes cp.test
Exit $fail