database/postgres
1
0
Fork 0
mirror of https://github.com/postgres/postgres.git synced 2025-07-28 23:42:10 +03:00
Code Activity

Fix running out of file descriptors for spill files.

Browse Source 
Currently while decoding changes, if the number of changes exceeds a
certain threshold, we spill those to disk.  And this happens for each
(sub)transaction.  Now, while reading all these files, we don't close them
until we read all the files.  While reading these files, if the number of
such files exceeds the maximum number of file descriptors, the operation
errors out.

Use PathNameOpenFile interface to open these files as that internally has
the mechanism to release kernel FDs as needed to get us under the
max_safe_fds limit.

Reported-by: Amit Khandekar
Author: Amit Khandekar
Reviewed-by: Amit Kapila
Backpatch-through: 9.4
Discussion: https://postgr.es/m/CAJ3gD9c-sECEn79zXw4yBnBdOttacoE-6gAyP0oy60nfs_sabQ@mail.gmail.com
This commit is contained in:
Amit Kapila
2020-01-02 10:40:32 +05:30
parent 8f3e44a164
commit f8a6d8e71b
2 changed files with 90 additions and 25 deletions
Download Patch File Download Diff File Expand all files Collapse all files

39
src/test/recovery/t/006_logical_decoding.pl
View File

@ -7,7 +7,7 @@ use strict;
use warnings;
use PostgresNode;
use TestLib;
use Test::More tests => 10;
use Test::More tests => 11;
use Config;
# Initialize master node
@ -135,5 +135,42 @@ is($node_master->psql('postgres', 'DROP DATABASE otherdb'),
is($node_master->slot('otherdb_slot')->{'slot_name'},
undef, 'logical slot was actually dropped with DB');
# Test to ensure that we don't run out of file descriptors even if there
# are more spill files than maxAllocatedDescs.
# Set max_files_per_process to a small value to make it more likely to run out
# of max open file descriptors.
$node_master->safe_psql('postgres',
'ALTER SYSTEM SET max_files_per_process = 26;');
$node_master->restart;
$node_master->safe_psql(
'postgres', q{
do $$
BEGIN
FOR i IN 1..10 LOOP
BEGIN
INSERT INTO decoding_test(x) SELECT generate_series(1,5000);
EXCEPTION
when division_by_zero then perform 'dummy';
END;
END LOOP;
END $$;
});
$result = $node_master->safe_psql('postgres',
qq[
SELECT data from pg_logical_slot_get_changes('test_slot', NULL, NULL)
WHERE data LIKE '%INSERT%' ORDER BY lsn LIMIT 1;
]);
$expected = q{table public.decoding_test: INSERT: x[integer]:1 y[text]:null};
is($result, $expected, 'got expected output from spilling subxacts session');
# Reset back max_files_per_process
$node_master->safe_psql('postgres',
'ALTER SYSTEM SET max_files_per_process = DEFAULT;');
$node_master->restart;
# done with the node
$node_master->stop;