mirror of
https://git.savannah.gnu.org/git/coreutils.git
synced 2026-11-19 03:33:43 +03:00
sort: use at most 8 threads by default
* src/sort.c (main): If --parallel isn't specified, restrict the number of threads to 8 by default. If the --parallel option is specified, then allow any number of threads to be set, independent of the number of processors on the system. * doc/coreutils.texi (sort invocation): Document the changes to determining the number of threads to use. Mention the memory overhead when using multiple threads. * tests/misc/sort-spinlock-abuse: Allow single core systems that support pthreads. * tests/misc/sort-stale-thread-mem: Likewise. * tests/misc/sort-unique-segv: Likewise. * NEWS: Mention the change in behaviour.
This commit is contained in:
1 parent
eea8e7530b
commit
0e181024c0
6 files changed
+28
-13
No files matched your search
@@ -27,6 +27,12 @@ GNU coreutils NEWS -*- outline -*-
|
||||
|
||||
sort -m -o f f ... f no longer dumps core when file descriptors are limited.
|
||||
|
||||
** Changes in behavior
|
||||
|
||||
sort will not create more than 8 threads by default due to diminishing
|
||||
performance gains. Also the --parallel option is no longer restricted
|
||||
to the number of available processors.
|
||||
|
||||
** New features
|
||||
|
||||
split accepts the --number option to generate a specific number of files.
|
||||
|
||||
+5
-4
@@ -4190,10 +4190,11 @@ disks and controllers.
|
||||
@item --parallel=@var{n}
|
||||
@opindex --parallel
|
||||
@cindex multithreaded sort
|
||||
Limit the number of sorts run in parallel to @var{n}. By default,
|
||||
@var{n} is set to the number of available processors, and values
|
||||
greater than that are reduced to that limit. Also see
|
||||
@ref{nproc invocation}.
|
||||
Set the number of sorts run in parallel to @var{n}. By default,
|
||||
@var{n} is set to the number of available processors, but limited
|
||||
to 8, as there are diminishing performance gains after that.
|
||||
Note also that using @var{n} threads increases the memory usage by
|
||||
a factor of log @var{n}. Also see @ref{nproc invocation}.
|
||||
|
||||
@item -u
|
||||
@itemx --unique
|
||||
|
||||
+11
-6
@@ -116,6 +116,10 @@ struct rlimit { size_t rlim_cur; };
|
||||
this number has any practical effect. */
|
||||
enum { SUBTHREAD_LINES_HEURISTIC = 4 };
|
||||
|
||||
/* The number of threads after which there are
|
||||
diminishing performance gains. */
|
||||
enum { DEFAULT_MAX_THREADS = 8 };
|
||||
|
||||
/* Exit statuses. */
|
||||
enum
|
||||
{
|
||||
@@ -455,7 +459,7 @@ Other options:\n\
|
||||
-t, --field-separator=SEP use SEP instead of non-blank to blank transition\n\
|
||||
-T, --temporary-directory=DIR use DIR for temporaries, not $TMPDIR or %s;\n\
|
||||
multiple options specify multiple directories\n\
|
||||
--parallel=N limit the number of sorts run concurrently to N\n\
|
||||
--parallel=N change the number sorts run concurrently to N\n\
|
||||
-u, --unique with -c, check for strict ordering;\n\
|
||||
without -c, output only the first of an equal run\n\
|
||||
"), DEFAULT_TMPDIR);
|
||||
@@ -4595,14 +4599,15 @@ main (int argc, char **argv)
|
||||
}
|
||||
else
|
||||
{
|
||||
unsigned long int np2 = num_processors (NPROC_CURRENT_OVERRIDABLE);
|
||||
if (!nthreads || nthreads > np2)
|
||||
nthreads = np2;
|
||||
if (!nthreads)
|
||||
{
|
||||
nthreads = MIN (DEFAULT_MAX_THREADS,
|
||||
num_processors (NPROC_CURRENT_OVERRIDABLE));
|
||||
}
|
||||
|
||||
/* Avoid integer overflow later. */
|
||||
size_t nthreads_max = SIZE_MAX / (2 * sizeof (struct merge_node));
|
||||
if (nthreads_max < nthreads)
|
||||
nthreads = nthreads_max;
|
||||
nthreads = MIN (nthreads, nthreads_max);
|
||||
|
||||
sort (files, nfiles, outfile, nthreads);
|
||||
}
|
||||
|
||||
@@ -20,7 +20,8 @@
|
||||
. "${srcdir=.}/init.sh"; path_prepend_ ../src
|
||||
print_ver_ sort
|
||||
|
||||
test "$(nproc)" = 1 && skip_ "requires a multi-core system"
|
||||
grep '^#define HAVE_PTHREAD_T 1' "$CONFIG_HEADER" > /dev/null ||
|
||||
skip_test_ 'requires pthreads'
|
||||
|
||||
seq 100000 > in || framework_failure_
|
||||
mkfifo fifo || framework_failure_
|
||||
|
||||
@@ -24,7 +24,8 @@ print_ver_ sort
|
||||
very_expensive_
|
||||
|
||||
valgrind --help >/dev/null || skip_ "requires valgrind"
|
||||
test "$(nproc)" = 1 && skip_ "requires a multi-core system"
|
||||
grep '^#define HAVE_PTHREAD_T 1' "$CONFIG_HEADER" > /dev/null ||
|
||||
skip_test_ 'requires pthreads'
|
||||
|
||||
# gensort output seems to trigger the failure more often,
|
||||
# so prefer gensort if it is available.
|
||||
|
||||
@@ -19,7 +19,8 @@
|
||||
. "${srcdir=.}/init.sh"; path_prepend_ ../src
|
||||
print_ver_ sort
|
||||
|
||||
test "$(nproc)" = 1 && skip_ "requires a multi-core system"
|
||||
grep '^#define HAVE_PTHREAD_T 1' "$CONFIG_HEADER" > /dev/null ||
|
||||
skip_test_ 'requires pthreads'
|
||||
|
||||
cat <<\EOF > in || framework_failure_
|
||||
|
||||
|
||||
Reference in new issue
Block a user