Files
coreutils/tests/Makefile.am
T
Chen Guo 9face836f3 sort: parallelize internal sort
This patch is by Gene Auyeung, Chris Dickens, Chen Guo, and Mike
Nichols, based off of a patch by Paul Eggert, Glen Lenker, et. al.,
with a basic heap implementation based off of the GDSL heap,
originally by Nicolas Darnis.

The number of sorts done in parallel is limited to the number
of available processors by default, or can be further restricted
with the --parallel option.

On a dual-die, 8 core Intel Xeon, results show sorting with
8 threads is almost 4 times faster than using a single thread.
Timings when sorting a 96MB file:
THREADS     TIME (s)
1            5.10
2            2.87
4            1.75
8            1.31

Single threaded sorting has also been improved,
especially for cheaper comparison operations:
COMMAND             BEFORE (s)  AFTER (s)
sort                 8.822       8.716
sort -g             10.336      10.222
sort -n              3.077       2.961
LANG=C sort          2.169       2.066

* bootstrap.conf: Add heap, pthread.
* coreutils.texi (sort): Describe the new --parallel option.
* gl/lib/heap.c: New file. Very basic heap implementation.
* gl/lib/heap.h: New file.
* gl/modules/heap: New file.
* src/Makefile.am: Add LIB_PTHREAD.
* src/sort.c: Include heap.h, nproc.h, pthread.h.
(MAX_MERGE): New macro.
(SUBTHREAD_LINES_HEURISTIC, PARALLEL_OPTION): New constants.
(MERGE_END, MERGE_ROOT): New constants.
(struct merge_node): New struct.
(struct merge_node_queue): New struct.
(sortlines temp): Remove declaration.
(usage, long_options, main): New option, --parallel.
(specify_nthreads): New function.
(mergelines): New signature, to emphasize the fact that the HI area
must be part of the destination.  All callers changed.
(sequential_sort): New function, renamed from sortlines. Merge in
the functionality of sortlines_temp.
(compare_nodes): New function.
(lock_node, unlock_node): New functions.
(queue_destroy): New function.
(queue_init): New function.
(queue_insert): New function.
(queue_pop): New function.
(write_unique): New function.
(mergelines_node): New function.
(check_insert): New function.
(update_parent): New function.
(merge_loop): New function.
(sortlines): Rewrite to support and use parallelism, with a new
signature. All callers changed.
(struct thread_args): New struct.
(sortlines_thread): New function.
(sortlines_temp): Remove.
(sort): New argument NTHREADS. All uses changed. Output moved to
mergelines_node.
(main): disable threading if we are sorting at random.
* tests/Makefile.am (TESTS): Add misc/sort-benchmark-random.
* tests/misc/sort-benchmark-random: New file.

Signed-off-by: Pádraig Brady <P@draigBrady.com>
2010-07-13 01:44:46 +01:00

646 lines
14 KiB
Makefile

## Process this file with automake to produce Makefile.in -*-Makefile-*-.
# Sort in traditional ASCII order, regardless of the current locale;
# otherwise we may get into trouble with distinct strings that the
# current locale considers to be equal.
ASSORT = LC_ALL=C sort
EXTRA_DIST = \
Coreutils.pm \
CuTmpdir.pm \
check.mk \
envvar-check \
init.cfg \
init.sh \
lang-default \
other-fs-tmpdir \
require-perl \
sample-test \
test-lib.sh \
$(pr_data)
root_tests = \
chown/basic \
cp/cp-a-selinux \
cp/preserve-gid \
cp/special-bits \
cp/cp-mv-enotsup-xattr \
cp/capability \
dd/skip-seek-past-dev \
install/install-C-root \
ls/capability \
ls/nameless-uid \
misc/chcon \
misc/chroot-credentials \
misc/selinux \
misc/truncate-owned-by-other \
mkdir/writable-under-readonly \
mv/sticky-to-xpart \
rm/fail-2eperm \
rm/no-give-up \
rm/one-file-system \
rm/read-only \
tail-2/append-only \
touch/now-owned-by-other
.PHONY: check-root
check-root:
$(MAKE) check TESTS='$(root_tests)'
check-recursive: root-hint
# Advertise `check-root' target.
.PHONY: root-hint
root-hint:
@echo '***********************************************************'
@echo "NOTICE: Some tests may be run only as root."
@echo " See the 'Running tests as root' section in README."
@echo '***********************************************************'
EXTRA_DIST += $(TESTS)
# Do not choose a name that is a shell keyword like 'if', or a
# commonly-used utility like 'cat' or 'test', as the name of a test.
# Otherwise, VPATH builds will fail on hosts like Solaris, since they
# will expand 'if test ...' to 'if .../test ...', and the '.../test'
# will execute the test script rather than the standard utility.
# Notes on the ordering of these tests:
# Place early in the list tests of the tools that
# are most commonly used in test scripts themselves.
# E.g., nearly every test script uses rm and chmod.
# help-version comes early because it's a basic sanity test.
# Put seq early, since lots of other tests use it.
# Put tests that sleep early, but not all together, so in parallel builds
# they share time with tests that burn CPU, not with others that sleep.
# Put head-elide-tail early, because it's long-running.
TESTS = \
misc/help-version \
tail-2/inotify-race \
misc/invalid-opt \
rm/ext3-perf \
rm/cycle \
cp/link-heap \
tail-2/inotify-hash-abuse \
tail-2/inotify-hash-abuse2 \
tail-2/F-vs-rename \
tail-2/inotify-rotate \
chmod/no-x \
chgrp/basic \
rm/dangling-symlink \
misc/ls-time \
rm/deep-1 \
rm/deep-2 \
rm/dir-no-w \
rm/dir-nonrecur \
rm/dot-rel \
rm/isatty \
rm/empty-inacc \
rm/empty-name \
rm/f-1 \
rm/fail-eacces \
rm/fail-eperm \
tail-2/assert \
rm/hash \
rm/i-1 \
rm/i-never \
rm/i-no-r \
tail-2/infloop-1 \
rm/ignorable \
rm/inaccessible \
rm/interactive-always \
rm/interactive-once \
rm/ir-1 \
rm/one-file-system2 \
rm/r-1 \
rm/r-2 \
rm/r-3 \
rm/r-4 \
rm/readdir-bug \
rm/rm1 \
touch/empty-file \
rm/rm2 \
rm/rm3 \
rm/rm4 \
rm/rm5 \
rm/sunos-1 \
rm/unread2 \
rm/unread3 \
rm/unreadable \
rm/v-slash \
chgrp/default-no-deref \
chgrp/deref \
chgrp/no-x \
chgrp/posix-H \
chgrp/recurse \
misc/env \
misc/ptx \
misc/test \
misc/seq \
misc/seq-long-double \
misc/head \
misc/head-elide-tail \
tail-2/tail-n0f \
misc/ls-misc \
misc/date \
misc/date-next-dow \
misc/ptx-overrun \
misc/xstrtol \
tail-2/pid \
misc/od \
misc/mktemp \
misc/arch \
misc/pr \
misc/join \
pr/pr-tests \
misc/df-P \
misc/pwd-option \
misc/pwd-unreadable-parent \
misc/chcon-fail \
misc/cut \
misc/wc \
misc/wc-files0-from \
misc/wc-files0 \
misc/wc-parallel \
misc/cat-proc \
misc/cat-buf \
misc/base64 \
misc/basename \
misc/close-stdout \
misc/chroot-fail \
misc/comm \
misc/csplit \
misc/date-sec \
misc/dircolors \
misc/df \
misc/dirname \
misc/env-null \
misc/expand \
misc/expr \
misc/factor \
misc/false-status \
misc/fmt \
misc/fmt-long-line \
misc/fold \
misc/groups-dash \
misc/groups-version \
misc/head-c \
misc/head-pos \
misc/id-context \
misc/id-groups \
misc/md5sum \
misc/md5sum-newline \
misc/md5sum-parallel \
misc/mknod \
misc/nice \
misc/nice-fail \
misc/nl \
misc/nohup \
misc/nproc-avail \
misc/nproc-positive \
misc/od-N \
misc/od-multiple-t \
misc/od-x8 \
misc/paste \
misc/pathchk1 \
misc/printenv \
misc/printf \
misc/printf-cov \
misc/printf-hex \
misc/printf-surprise \
misc/pwd-long \
misc/readlink-fp-loop \
misc/runcon-no-reorder \
misc/sha1sum \
misc/sha1sum-vec \
misc/sha224sum \
misc/sha256sum \
misc/sha384sum \
misc/sha512sum \
misc/shred-exact \
misc/shred-passes \
misc/shred-remove \
misc/shuf \
misc/sort \
misc/sort-benchmark-random \
misc/sort-compress \
misc/sort-continue \
misc/sort-debug-keys \
misc/sort-debug-warn \
misc/sort-files0-from \
misc/sort-float \
misc/sort-merge \
misc/sort-merge-fdlimit \
misc/sort-month \
misc/sort-rand \
misc/sort-version \
misc/split-a \
misc/split-fail \
misc/split-l \
misc/stat-fmt \
misc/stat-hyphen \
misc/stat-printf \
misc/stat-slash \
misc/stdbuf \
misc/stty \
misc/stty-invalid \
misc/stty-row-col \
misc/su-fail \
misc/sum \
misc/sum-sysv \
misc/tac \
misc/tac-continue \
misc/tail \
misc/tee \
misc/tee-dash \
misc/test-diag \
misc/timeout \
misc/timeout-parameters \
misc/tr \
misc/truncate-dangling-symlink \
misc/truncate-dir-fail \
misc/truncate-fail-diag \
misc/truncate-fifo \
misc/truncate-no-create-missing \
misc/truncate-overflow \
misc/truncate-parameters \
misc/truncate-relative \
misc/tsort \
misc/tty-eof \
misc/unexpand \
misc/uniq \
misc/xattr \
tail-2/wait \
chmod/c-option \
chmod/equal-x \
chmod/equals \
chmod/inaccessible \
chmod/octal \
chmod/setgid \
chmod/silent \
chmod/thru-dangling \
chmod/umask-x \
chmod/usage \
chown/deref \
chown/preserve-root \
chown/separator \
cp/abuse \
cp/acl \
cp/backup-1 \
cp/backup-dir \
cp/backup-is-src \
cp/cp-HL \
cp/cp-deref \
cp/cp-i \
cp/cp-mv-backup \
cp/cp-parents \
cp/deref-slink \
cp/dir-rm-dest \
cp/dir-slash \
cp/dir-vs-file \
cp/existing-perm-race \
cp/fail-perm \
cp/file-perm-race \
cp/into-self \
cp/link \
cp/link-no-deref \
cp/link-preserve \
cp/no-deref-link1 \
cp/no-deref-link2 \
cp/no-deref-link3 \
cp/parent-perm \
cp/parent-perm-race \
cp/perm \
cp/preserve-2 \
cp/preserve-slink-time \
cp/proc-short-read \
cp/proc-zero-len \
cp/r-vs-symlink \
cp/reflink-auto \
cp/reflink-perm \
cp/same-file \
cp/slink-2-slink \
cp/sparse \
cp/special-f \
cp/src-base-dot \
cp/symlink-slash \
cp/thru-dangling \
df/unreadable \
dd/direct \
dd/misc \
dd/not-rewound \
dd/reblock \
dd/skip-seek \
dd/skip-seek2 \
dd/skip-seek-past-file \
dd/stderr \
dd/unblock \
dd/unblock-sync \
df/total-verify \
du/2g \
du/8gb \
du/basic \
du/deref \
du/deref-args \
du/exclude \
du/fd-leak \
du/files0-from \
du/hard-link \
du/inacc-dest \
du/inacc-dir \
du/inaccessible-cwd \
du/long-from-unreadable \
du/long-sloop \
du/max-depth \
du/no-deref \
du/no-x \
du/one-file-system \
du/restore-wd \
du/slash \
du/slink \
du/trailing-slash \
du/two-args \
id/no-context \
install/basic-1 \
install/create-leading \
install/d-slashdot \
install/install-C \
install/install-C-selinux \
install/strip-program \
install/trap \
ln/backup-1 \
ln/hard-backup \
ln/hard-to-sym \
ln/misc \
ln/sf-1 \
ln/slash-decorated-nonexistent-dest \
ln/target-1 \
ls/abmon-align \
ls/color-clear-to-eol \
ls/color-dtype-dir \
ls/color-norm \
ls/dangle \
ls/dired \
ls/file-type \
ls/follow-slink \
ls/infloop \
ls/inode \
ls/m-option \
ls/multihardlink \
ls/no-arg \
ls/no-cap \
ls/proc-selinux-segfault \
ls/readdir-mountpoint-inode \
ls/recursive \
ls/rt-1 \
ls/stat-dtype \
ls/stat-failed \
ls/stat-free-symlinks \
ls/stat-vs-dirent \
ls/symlink-slash \
ls/x-option \
mkdir/p-1 \
mkdir/p-2 \
mkdir/p-3 \
mkdir/p-slashdot \
mkdir/p-thru-slink \
mkdir/p-v \
mkdir/parents \
mkdir/perm \
mkdir/selinux \
mkdir/special-1 \
mkdir/t-slash \
mv/acl \
mv/atomic \
mv/atomic2 \
mv/backup-dir \
mv/backup-is-src \
mv/childproof \
mv/diag \
mv/dir-file \
mv/dir2dir \
mv/dup-source \
mv/force \
mv/hard-2 \
mv/hard-3 \
mv/hard-4 \
mv/hard-link-1 \
mv/hard-verbose \
mv/i-1 \
mv/i-2 \
mv/i-3 \
mv/i-4 \
mv/i-5 \
mv/i-link-no \
mv/into-self \
mv/into-self-2 \
mv/into-self-3 \
mv/into-self-4 \
mv/leak-fd \
mv/mv-n \
mv/mv-special-1 \
mv/no-target-dir \
mv/part-fail \
mv/part-hardlink \
mv/part-rename \
mv/part-symlink \
mv/partition-perm \
mv/perm-1 \
mv/to-symlink \
mv/trailing-slash \
mv/update \
readlink/can-e \
readlink/can-f \
readlink/can-m \
readlink/rl-1 \
rmdir/fail-perm \
rmdir/ignore \
rmdir/t-slash \
tail-2/assert-2 \
tail-2/big-4gb \
tail-2/flush-initial \
tail-2/follow-stdin \
tail-2/pipe-f \
tail-2/pipe-f2 \
tail-2/proc-ksyms \
tail-2/start-middle \
touch/60-seconds \
touch/dangling-symlink \
touch/dir-1 \
touch/fail-diag \
touch/fifo \
touch/no-create-missing \
touch/no-dereference \
touch/no-rights \
touch/not-owner \
touch/obsolescent \
touch/read-only \
touch/relative \
touch/trailing-slash \
$(root_tests)
pr_data = \
pr/0F \
pr/0FF \
pr/0FFnt \
pr/0FFt \
pr/0FnFnt \
pr/0FnFt \
pr/0Fnt \
pr/0Ft \
pr/2-S_f-t_notab \
pr/2-Sf-t_notab \
pr/2f-t_notab \
pr/2s_f-t_notab \
pr/2s_w60f-t_nota \
pr/2sf-t_notab \
pr/2sw60f-t_notab \
pr/2w60f-t_notab \
pr/3-0F \
pr/3-5l24f-t \
pr/3-FF \
pr/3a2l17-FF \
pr/3a3f-0F \
pr/3a3l15-t \
pr/3a3l15f-t \
pr/3b2l17-FF \
pr/3b3f-0F \
pr/3b3f-0FF \
pr/3b3f-FF \
pr/3b3l15-t \
pr/3b3l15f-t \
pr/3f-0F \
pr/3f-FF \
pr/3l24-t \
pr/3l24f-t \
pr/3ml24-FF \
pr/3ml24-t \
pr/3ml24-t-FF \
pr/3ml24f-t \
pr/4-7l24-FF \
pr/4l24-FF \
pr/FF \
pr/FFn \
pr/FFtn \
pr/FnFn \
pr/Ja3l24f-lm \
pr/Jb3l24f-lm \
pr/Jml24f-lm-lo \
pr/W-72l24f-ll \
pr/W20l24f-ll \
pr/W26l24f-ll \
pr/W27l24f-ll \
pr/W28l24f-ll \
pr/W35Ja3l24f-lm \
pr/W35Jb3l24f-lm \
pr/W35Jml24f-lmlo \
pr/W35a3l24f-lm \
pr/W35b3l24f-lm \
pr/W35ml24f-lm-lo \
pr/W72Jl24f-ll \
pr/a2l15-FF \
pr/a2l17-FF \
pr/a3-0F \
pr/a3f-0F \
pr/a3f-0FF \
pr/a3f-FF \
pr/a3l15-t \
pr/a3l15f-t \
pr/a3l24f-lm \
pr/b2l15-FF \
pr/b2l17-FF \
pr/b3-0F \
pr/b3f-0F \
pr/b3f-0FF \
pr/b3f-FF \
pr/b3l15-t \
pr/b3l15f-t \
pr/b3l24f-lm \
pr/l24-FF \
pr/l24-t \
pr/l24f-t \
pr/loli \
pr/ml20-FF-t \
pr/ml24-FF \
pr/ml24-t \
pr/ml24-t-FF \
pr/ml24f-0F \
pr/ml24f-lm-lo \
pr/ml24f-t \
pr/ml24f-t-0F \
pr/n+2-5l24f-0FF \
pr/n+2l24f-0FF \
pr/n+2l24f-bl \
pr/n+3-7l24-FF \
pr/n+3l24f-0FF \
pr/n+3l24f-bl \
pr/n+3ml20f-bl-FF \
pr/n+3ml24f-bl-tn \
pr/n+3ml24f-tn-bl \
pr/n+4-8a2l17-FF \
pr/n+4b2l17f-0FF \
pr/n+5-8b3l17f-FF \
pr/n+5a3l13f-0FF \
pr/n+6a2l17-FF \
pr/n+6b3l13f-FF \
pr/n+7l24-FF \
pr/n+8l20-FF \
pr/nJml24f-lmlmlo \
pr/nJml24f-lmlolm \
pr/nN1+3l24f-bl \
pr/nN15l24f-bl \
pr/nSml20-bl-FF \
pr/nSml20-t-t-FF \
pr/nSml20-t-tFFFF \
pr/nSml24-bl-FF \
pr/nSml24-t-t-FF \
pr/nSml24-t-tFFFF \
pr/nl24f-bl \
pr/o3Jml24f-lm-lo \
pr/o3a3Sl24f-tn \
pr/o3a3Snl24f-tn \
pr/o3a3l24f-tn \
pr/o3b3Sl24f-tn \
pr/o3b3Snl24f-tn \
pr/o3b3l24f-tn \
pr/o3mSl24f-bl-tn \
pr/o3mSnl24fbltn \
pr/o3ml24f-bl-tn \
pr/t-0FF \
pr/t-FF \
pr/t-bl \
pr/t-t \
pr/tFFn \
pr/tFFt \
pr/tFFt-bl \
pr/tFFt-ll \
pr/tFFt-lm \
pr/tFnFt \
pr/t_notab \
pr/t_tab \
pr/t_tab_ \
pr/ta3-0FF \
pr/ta3-FF \
pr/tb3-0FF \
pr/tb3-FF \
pr/tn \
pr/tn2e5o3-t_tab \
pr/tn2e8-t_tab \
pr/tn2e8o3-t_tab \
pr/tn_2e8-t_tab \
pr/tn_2e8S-t_tab \
pr/tne8-t_tab \
pr/tne8o3-t_tab \
pr/tt-0FF \
pr/tt-FF \
pr/tt-bl \
pr/tt-t \
pr/tta3-0FF \
pr/tta3-FF \
pr/ttb3-0FF \
pr/ttb3-FF \
pr/w72l24f-ll
include $(srcdir)/check.mk