database/postgres
database/postgres
1
0
Fork 0
mirror of https://github.com/postgres/postgres.git synced 2025-05-02 11:44:50 +03:00
Code

Introduce io_max_combine_limit.

Browse Source 
The existing io_combine_limit can be changed by users.  The new
io_max_combine_limit is fixed at server startup time, and functions as a
silent clamp on the user setting.  That in itself is probably quite
useful, but the primary motivation is:

aio_init.c allocates shared memory for all asynchronous IOs including
some per-block data, and we didn't want to waste memory you'd never used
by assuming they could be up to PG_IOV_MAX.  This commit already halves
the size of 'AioHandleIov' and 'AioHandleData'.  A follow-up commit can
now expand PG_IOV_MAX without affecting that.

Since our GUC system doesn't support dependencies or cross-checks
between GUCs, the user-settable one now assigns a "raw" value to
io_combine_limit_guc, and the lower of io_combine_limit_guc and
io_max_combine_limit is maintained in io_combine_limit.

Reviewed-by: Andres Freund <andres@anarazel.de> (earlier version)
Discussion: https://postgr.es/m/CA%2BhUKG%2B2T9p-%2BzM6Eeou-RAJjTML6eit1qn26f9twznX59qtCA%40mail.gmail.com
This commit is contained in:
Thomas Munro 2025-03-19 11:40:56 +13:00
parent 17d8bba6da
commit 10f6646847
8 changed files with 72 additions and 15 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

23
doc/src/sgml/config.sgml
View File

@ -2625,6 +2625,24 @@ include_dir 'conf.d'
</listitem>
</varlistentry>
<varlistentry id="guc-io-max-combine-limit" xreflabel="io_max_combine_limit">
<term><varname>io_max_combine_limit</varname> (<type>integer</type>)
<indexterm>
<primary><varname>io_max_combine_limit</varname> configuration parameter</primary>
</indexterm>
</term>
<listitem>
<para>
Controls the largest I/O size in operations that combine I/O, and silently
limits the user-settable parameter <varname>io_combine_limit</varname>.
This parameter can only be set in
the <filename>postgresql.conf</filename> file or on the server
command line.
The default is 128kB.
</para>
</listitem>
</varlistentry>
<varlistentry id="guc-io-combine-limit" xreflabel="io_combine_limit">
<term><varname>io_combine_limit</varname> (<type>integer</type>)
<indexterm>
@ -2633,7 +2651,10 @@ include_dir 'conf.d'
</term>
<listitem>
<para>
Controls the largest I/O size in operations that combine I/O.
Controls the largest I/O size in operations that combine I/O. If set
higher than the <varname>io_max_combine_limit</varname> parameter, the
lower value will silently be used instead, so both may need to be raised
to increase the I/O size.
The default is 128kB.
</para>
</listitem>

18
src/backend/commands/variable.c
View File

@ -1156,6 +1156,24 @@ assign_maintenance_io_concurrency(int newval, void *extra)
#endif
}
/*
* GUC assign hooks that recompute io_combine_limit whenever
* io_combine_limit_guc and io_max_combine_limit are changed. These are needed
* because the GUC subsystem doesn't support dependencies between GUCs, and
* they may be assigned in either order.
*/
void
assign_io_max_combine_limit(int newval, void *extra)
{
io_max_combine_limit = newval;
io_combine_limit = Min(io_max_combine_limit, io_combine_limit_guc);
}
void
assign_io_combine_limit(int newval, void *extra)
{
io_combine_limit_guc = newval;
io_combine_limit = Min(io_max_combine_limit, io_combine_limit_guc);
}
/*
* These show hooks just exist because we want to show the values in octal.

17
src/backend/storage/aio/aio_init.c
View File

@ -18,6 +18,7 @@
#include "storage/aio.h"
#include "storage/aio_internal.h"
#include "storage/aio_subsys.h"
#include "storage/bufmgr.h"
#include "storage/io_worker.h"
#include "storage/ipc.h"
#include "storage/proc.h"
@ -72,15 +73,9 @@ AioHandleShmemSize(void)
static Size
AioHandleIOVShmemSize(void)
{
/*
* Each IO handle can have an PG_IOV_MAX long iovec.
*
* XXX: Right now the amount of space available for each IO is PG_IOV_MAX.
* While it's tempting to use the io_combine_limit GUC, that's
* PGC_USERSET, so we can't allocate shared memory based on that.
*/
/* each IO handle can have up to io_max_combine_limit iovec objects */
return mul_size(sizeof(struct iovec),
mul_size(mul_size(PG_IOV_MAX, AioProcs()),
mul_size(mul_size(io_max_combine_limit, AioProcs()),
io_max_concurrency));
}
@ -89,7 +84,7 @@ AioHandleDataShmemSize(void)
{
/* each buffer referenced by an iovec can have associated data */
return mul_size(sizeof(uint64),
mul_size(mul_size(PG_IOV_MAX, AioProcs()),
mul_size(mul_size(io_max_combine_limit, AioProcs()),
io_max_concurrency));
}
@ -160,7 +155,7 @@ AioShmemInit(void)
bool found;
uint32 io_handle_off = 0;
uint32 iovec_off = 0;
uint32 per_backend_iovecs = io_max_concurrency * PG_IOV_MAX;
uint32 per_backend_iovecs = io_max_concurrency * io_max_combine_limit;
pgaio_ctl = (PgAioCtl *)
ShmemInitStruct("AioCtl", AioCtlShmemSize(), &found);
@ -213,7 +208,7 @@ AioShmemInit(void)
ConditionVariableInit(&ioh->cv);
dclist_push_tail(&bs->idle_ios, &ioh->node);
iovec_off += PG_IOV_MAX;
iovec_off += io_max_combine_limit;
}
}

5
src/backend/storage/buffer/bufmgr.c
View File

@ -160,9 +160,12 @@ int maintenance_io_concurrency = DEFAULT_MAINTENANCE_IO_CONCURRENCY;
/*
* Limit on how many blocks should be handled in single I/O operations.
* StartReadBuffers() callers should respect it, as should other operations
* that call smgr APIs directly.
* that call smgr APIs directly. It is computed as the minimum of underlying
* GUCs io_combine_limit_guc and io_max_combine_limit.
*/
int io_combine_limit = DEFAULT_IO_COMBINE_LIMIT;
int io_combine_limit_guc = DEFAULT_IO_COMBINE_LIMIT;
int io_max_combine_limit = DEFAULT_IO_COMBINE_LIMIT;
/*
* GUC variables about triggering kernel writeback for buffers written; OS

16
src/backend/utils/misc/guc_tables.c
View File

@ -3252,6 +3252,20 @@ struct config_int ConfigureNamesInt[] =
NULL
},
{
{"io_max_combine_limit",
PGC_POSTMASTER,
RESOURCES_IO,
gettext_noop("Server-wide limit that clamps io_combine_limit."),
NULL,
GUC_UNIT_BLOCKS
},
&io_max_combine_limit,
DEFAULT_IO_COMBINE_LIMIT,
1, MAX_IO_COMBINE_LIMIT,
NULL, assign_io_max_combine_limit, NULL
},
{
{"io_combine_limit",
PGC_USERSET,
@ -3263,7 +3277,7 @@ struct config_int ConfigureNamesInt[] =
&io_combine_limit,
DEFAULT_IO_COMBINE_LIMIT,
1, MAX_IO_COMBINE_LIMIT,
NULL, NULL, NULL
NULL, assign_io_combine_limit, NULL
},
{

2
src/backend/utils/misc/postgresql.conf.sample
View File

@ -200,6 +200,8 @@
#backend_flush_after = 0 # measured in pages, 0 disables
#effective_io_concurrency = 16 # 1-1000; 0 disables prefetching
#maintenance_io_concurrency = 16 # 1-1000; 0 disables prefetching
#io_max_combine_limit = 128kB # usually 1-32 blocks (depends on OS)
# (change requires restart)
#io_combine_limit = 128kB # usually 1-32 blocks (depends on OS)
#io_method = worker # worker, sync (change requires restart)

4
src/include/storage/bufmgr.h
View File

@ -163,7 +163,9 @@ extern PGDLLIMPORT int maintenance_io_concurrency;
#define MAX_IO_COMBINE_LIMIT PG_IOV_MAX
#define DEFAULT_IO_COMBINE_LIMIT Min(MAX_IO_COMBINE_LIMIT, (128 * 1024) / BLCKSZ)
extern PGDLLIMPORT int io_combine_limit;
extern PGDLLIMPORT int io_combine_limit; /* min of the two GUCs below */
extern PGDLLIMPORT int io_combine_limit_guc;
extern PGDLLIMPORT int io_max_combine_limit;
extern PGDLLIMPORT int checkpoint_flush_after;
extern PGDLLIMPORT int backend_flush_after;

2
src/include/utils/guc_hooks.h
View File

@ -86,6 +86,8 @@ extern const char *show_log_timezone(void);
extern bool check_maintenance_io_concurrency(int *newval, void **extra,
GucSource source);
extern void assign_maintenance_io_concurrency(int newval, void *extra);
extern void assign_io_max_combine_limit(int newval, void *extra);
extern void assign_io_combine_limit(int newval, void *extra);
extern bool check_max_slot_wal_keep_size(int *newval, void **extra,
GucSource source);
extern void assign_max_wal_size(int newval, void *extra);