mirror of
https://git.savannah.gnu.org/git/coreutils.git
synced 2026-11-19 03:33:43 +03:00
copy: process empty extents more efficiently
* src/copy.c (extent_copy): Treat an allocated but empty extent much like a hole. I.E. don't read data we know is going to be NUL. Also we convert the empty extent to a hole only when SPARSE_ALWAYS so that the source and dest have the same allocation. This will be improved soon, when we use fallocate() to do the allocation. * tests/cp/fiemap-empty: A new test for efficiency and correctness of copying empty extents. * tests/Makefile.am: Reference the new test. * NEWS: Mention the change in behavior.
This commit is contained in:
1 parent
b56b53bd70
commit
bfdb6b585b
4 files changed
+162
-16
No files matched your search
@@ -40,6 +40,10 @@ GNU coreutils NEWS -*- outline -*-
|
||||
The sync in only needed on Linux kernels before 2.6.38.
|
||||
[The sync was introduced in coreutils-8.10]
|
||||
|
||||
cp now copies empty extents efficiently, when doing a FIEMAP copy.
|
||||
It no longer reads the zero bytes from the input, and also can efficiently
|
||||
create a hole in the output file when --sparse=always is specified.
|
||||
|
||||
df now aligns columns consistently, and no longer wraps entries
|
||||
with longer device identifiers, over two lines.
|
||||
|
||||
|
||||
+66
-14
@@ -39,6 +39,7 @@
|
||||
#include "extent-scan.h"
|
||||
#include "error.h"
|
||||
#include "fcntl--.h"
|
||||
#include "fiemap.h"
|
||||
#include "file-set.h"
|
||||
#include "filemode.h"
|
||||
#include "filenamecat.h"
|
||||
@@ -330,11 +331,28 @@ extent_copy (int src_fd, int dest_fd, char *buf, size_t buf_size,
|
||||
}
|
||||
|
||||
unsigned int i;
|
||||
for (i = 0; i < scan.ei_count; i++)
|
||||
bool empty_extent = false;
|
||||
for (i = 0; i < scan.ei_count || empty_extent; i++)
|
||||
{
|
||||
off_t ext_start = scan.ext_info[i].ext_logical;
|
||||
uint64_t ext_len = scan.ext_info[i].ext_length;
|
||||
uint64_t hole_size = ext_start - last_ext_start - last_ext_len;
|
||||
off_t ext_start;
|
||||
uint64_t ext_len;
|
||||
uint64_t hole_size;
|
||||
|
||||
if (i < scan.ei_count)
|
||||
{
|
||||
ext_start = scan.ext_info[i].ext_logical;
|
||||
ext_len = scan.ext_info[i].ext_length;
|
||||
}
|
||||
else /* empty extent at EOF. */
|
||||
{
|
||||
i--;
|
||||
ext_start = last_ext_start + scan.ext_info[i].ext_length;
|
||||
ext_len = 0;
|
||||
}
|
||||
|
||||
hole_size = ext_start - last_ext_start - last_ext_len;
|
||||
|
||||
wrote_hole_at_eof = false;
|
||||
|
||||
if (hole_size)
|
||||
{
|
||||
@@ -346,38 +364,72 @@ extent_copy (int src_fd, int dest_fd, char *buf, size_t buf_size,
|
||||
return false;
|
||||
}
|
||||
|
||||
if (sparse_mode != SPARSE_NEVER)
|
||||
if ((empty_extent && sparse_mode == SPARSE_ALWAYS)
|
||||
|| (!empty_extent && sparse_mode != SPARSE_NEVER))
|
||||
{
|
||||
if (lseek (dest_fd, ext_start, SEEK_SET) < 0)
|
||||
{
|
||||
error (0, errno, _("cannot lseek %s"), quote (dst_name));
|
||||
goto fail;
|
||||
}
|
||||
wrote_hole_at_eof = true;
|
||||
}
|
||||
else
|
||||
{
|
||||
/* When not inducing holes and when there is a hole between
|
||||
the end of the previous extent and the beginning of the
|
||||
current one, write zeros to the destination file. */
|
||||
if (! write_zeros (dest_fd, hole_size))
|
||||
off_t nzeros = hole_size;
|
||||
if (empty_extent)
|
||||
nzeros = MIN (src_total_size - dest_pos, hole_size);
|
||||
|
||||
if (! write_zeros (dest_fd, nzeros))
|
||||
{
|
||||
error (0, errno, _("%s: write failed"), quote (dst_name));
|
||||
goto fail;
|
||||
}
|
||||
|
||||
dest_pos = MIN (src_total_size, ext_start);
|
||||
}
|
||||
}
|
||||
|
||||
last_ext_start = ext_start;
|
||||
last_ext_len = ext_len;
|
||||
|
||||
off_t n_read;
|
||||
if ( ! sparse_copy (src_fd, dest_fd, buf, buf_size,
|
||||
sparse_mode == SPARSE_ALWAYS, src_name, dst_name,
|
||||
ext_len, &n_read,
|
||||
&wrote_hole_at_eof))
|
||||
return false;
|
||||
/* Treat an unwritten but allocated extent much like a hole.
|
||||
I.E. don't read, but don't convert to a hole in the destination,
|
||||
unless SPARSE_ALWAYS. */
|
||||
if (scan.ext_info[i].ext_flags & FIEMAP_EXTENT_UNWRITTEN)
|
||||
{
|
||||
empty_extent = true;
|
||||
last_ext_len = 0;
|
||||
if (ext_len == 0) /* The last extent is empty and processed. */
|
||||
empty_extent = false;
|
||||
}
|
||||
else
|
||||
{
|
||||
off_t n_read;
|
||||
empty_extent = false;
|
||||
last_ext_len = ext_len;
|
||||
|
||||
dest_pos = ext_start + n_read;
|
||||
if ( ! sparse_copy (src_fd, dest_fd, buf, buf_size,
|
||||
sparse_mode == SPARSE_ALWAYS,
|
||||
src_name, dst_name, ext_len, &n_read,
|
||||
&wrote_hole_at_eof))
|
||||
return false;
|
||||
|
||||
dest_pos = ext_start + n_read;
|
||||
}
|
||||
|
||||
/* If the file ends with unwritten extents not accounted for in the
|
||||
size, then skip processing them, and the associated redundant
|
||||
read() calls which will always return 0. We will need to
|
||||
remove this when we add fallocate() so that we can maintain
|
||||
extents beyond the apparent size. */
|
||||
if (dest_pos == src_total_size)
|
||||
{
|
||||
scan.hit_final_extent = true;
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
/* Release the space allocated to scan->ext_info. */
|
||||
|
||||
+3
-2
@@ -321,8 +321,9 @@ TESTS = \
|
||||
cp/dir-vs-file \
|
||||
cp/existing-perm-race \
|
||||
cp/fail-perm \
|
||||
cp/fiemap-perf \
|
||||
cp/fiemap-2 \
|
||||
cp/fiemap-empty \
|
||||
cp/fiemap-perf \
|
||||
cp/fiemap-2 \
|
||||
cp/file-perm-race \
|
||||
cp/into-self \
|
||||
cp/link \
|
||||
|
||||
Executable
+89
@@ -0,0 +1,89 @@
|
||||
#!/bin/sh
|
||||
# Test cp reads unwritten extents efficiently
|
||||
|
||||
# Copyright (C) 2011 Free Software Foundation, Inc.
|
||||
|
||||
# This program is free software: you can redistribute it and/or modify
|
||||
# it under the terms of the GNU General Public License as published by
|
||||
# the Free Software Foundation, either version 3 of the License, or
|
||||
# (at your option) any later version.
|
||||
|
||||
# This program is distributed in the hope that it will be useful,
|
||||
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
# GNU General Public License for more details.
|
||||
|
||||
# You should have received a copy of the GNU General Public License
|
||||
# along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
. "${srcdir=.}/init.sh"; path_prepend_ ../src
|
||||
print_ver_ cp
|
||||
|
||||
touch fiemap_chk
|
||||
fiemap_capable_ fiemap_chk ||
|
||||
skip_test_ 'this file system lacks FIEMAP support'
|
||||
rm fiemap_chk
|
||||
|
||||
# TODO: rather than requiring `fallocate`, possible add
|
||||
# this functionality to truncate --alloc
|
||||
fallocate --help >/dev/null || skip_test_ 'The fallocate utility is required'
|
||||
fallocate -l 1 -n falloc.test ||
|
||||
skip_test_ 'this file system lacks FALLOCATE support'
|
||||
rm falloc.test
|
||||
|
||||
fallocate -l 600000000 space.test ||
|
||||
skip_test_ 'this test needs at least 600MB free space'
|
||||
|
||||
# Disable this test on old BTRFS (e.g. Fedora 14)
|
||||
# which reports ordinary extents for unwritten ones.
|
||||
filefrag space.test || skip_test_ 'the `filefrag` utility is missing'
|
||||
filefrag -v space.test | grep -F 'unwritten' > /dev/null ||
|
||||
skip_test_ 'this file system does not report empty extents as "unwritten"'
|
||||
|
||||
rm space.test
|
||||
|
||||
# Ensure we read a large empty file quickly
|
||||
fallocate -l 300000000 empty.big || framework_failure
|
||||
timeout 3 cp --sparse=always empty.big cp.test || fail=1
|
||||
test $(stat -c %s empty.big) = $(stat -c %s cp.test) || fail=1
|
||||
rm empty.big cp.test
|
||||
|
||||
# Ensure we handle extents beyond file size correctly.
|
||||
# Note until we support fallocate, we will not maintain
|
||||
# the file allocation. Ammend this test when fallocate is supported
|
||||
fallocate -l 10000000 -n unwritten.withdata || framework_failure
|
||||
dd count=10 if=/dev/urandom conv=notrunc iflag=fullblock of=unwritten.withdata
|
||||
cp unwritten.withdata cp.test || fail=1
|
||||
test $(stat -c %s unwritten.withdata) = $(stat -c %s cp.test) || fail=1
|
||||
cmp unwritten.withdata cp.test || fail=1
|
||||
rm unwritten.withdata cp.test
|
||||
|
||||
# The following to generate unaccounted extents followed by a hole, is not
|
||||
# supported by ext4 at least. The ftruncate discards all extents not
|
||||
# accounted for in the size.
|
||||
# fallocate -l 10000000 -n unacc.withholes
|
||||
# dd count=10 if=/dev/urandom conv=notrunc iflag=fullblock of=unacc.withholes
|
||||
# truncate -s20000000 unacc.withholes
|
||||
|
||||
# Ensure we handle a hole after empty extents correctly.
|
||||
# Since all extents are accounted for in the size,
|
||||
# we can maintain the allocation independently from
|
||||
# fallocate() support.
|
||||
fallocate -l 10000000 empty.withholes
|
||||
truncate -s 20000000 empty.withholes
|
||||
sectors_per_block=$(expr $(stat -c %o .) / 512)
|
||||
cp empty.withholes cp.test || fail=1
|
||||
test $(stat -c %s empty.withholes) = $(stat -c %s cp.test) || fail=1
|
||||
# These are usually equal but can vary by an IO block due to alignment
|
||||
alloc_diff=$(expr $(stat -c %b empty.withholes) - $(stat -c %b cp.test))
|
||||
alloc_diff=$(echo $alloc_diff | tr -d -- -) # abs()
|
||||
test $alloc_diff -le $sectors_per_block || fail=1
|
||||
# Again with SPARSE_ALWAYS
|
||||
cp --sparse=always empty.withholes cp.test || fail=1
|
||||
test $(stat -c %s empty.withholes) = $(stat -c %s cp.test) || fail=1
|
||||
# cp.test should take 0 space, but allowing for some systems
|
||||
# that store default extended attributes in data blocks
|
||||
test $(stat -c %b cp.test) -le $sectors_per_block || fail=1
|
||||
rm empty.withholes cp.test
|
||||
|
||||
Exit $fail
|
||||
Reference in new issue
Block a user