Files
postfixadmin/tests/BroadcastQueueTest.php
Rodrigo Cortés Cano 907117b4b8 Queue broadcast messages for background sending (#1054)
When doing a broadcast message, create a list of pending recipients in new db tables, and processes each message in turn. Jobs can be cancelled/retried. Locking is in place to try and stop multiple concurrent jobs being run. 

Adds db tables: broadcast_job, broadcast_job_domain and broadcast_recipient. 

thanks @TrapoSAMA
2026-11-20 20:20:34 +00:00

466 lines
19 KiB
PHP

<?php
class BroadcastQueueTest extends \PHPUnit\Framework\TestCase
{
private array $domains = [];
protected function setUp(): void
{
parent::setUp();
$suffix = uniqid('', true);
$this->domains = [
"broadcast-a-$suffix.test",
"broadcast-b-$suffix.test",
];
$this->cleanBroadcastTables();
db_query('DELETE FROM alias');
db_query('DELETE FROM mailbox');
db_query('DELETE FROM domain');
BroadcastQueue::setWorkerMode(BroadcastQueue::MODE_LIVE);
}
protected function tearDown(): void
{
BroadcastQueue::setWorkerMode(BroadcastQueue::MODE_LIVE);
$this->cleanBroadcastTables();
db_query('DELETE FROM alias');
db_query('DELETE FROM mailbox');
db_query('DELETE FROM domain');
parent::tearDown();
}
public function testBuildRecipientsIncludesActiveMailboxesAndAliases(): void
{
$this->createDomain($this->domains[0]);
$this->createMailbox("one@{$this->domains[0]}", $this->domains[0]);
$this->createMailbox("inactive@{$this->domains[0]}", $this->domains[0], false);
$this->createAlias("alias@{$this->domains[0]}", $this->domains[0], "alias-target@example.net,one@{$this->domains[0]}");
$recipients = BroadcastQueue::buildRecipients([$this->domains[0]], false);
$this->assertSame(["alias-target@example.net", "one@{$this->domains[0]}"], $recipients);
$this->assertSame(["one@{$this->domains[0]}"], BroadcastQueue::buildRecipients([$this->domains[0]], true));
}
public function testAllSelectedDomainsCanIncludeAliases(): void
{
foreach ($this->domains as $domain) {
$this->createDomain($domain);
$this->createMailbox("one@$domain", $domain);
$this->createAlias("alias@$domain", $domain, "external@$domain");
}
$this->assertCount(4, BroadcastQueue::buildRecipients($this->domains, false));
$this->assertCount(2, BroadcastQueue::buildRecipients($this->domains, true));
}
public function testDomainAdminListFindsOwnJobAfterManyForeignJobs(): void
{
$this->createDomain($this->domains[0]);
$own = BroadcastQueue::createJob(
'owner@example.test', 'sender@example.test', 'PostfixAdmin', 'Own job', 'Body',
[$this->domains[0]], true, []
);
for ($i = 0; $i < 201; $i++) {
BroadcastQueue::createJob(
'other@example.test', 'sender@example.test', 'PostfixAdmin', 'Other job', 'Body',
[$this->domains[0]], true, []
);
}
$this->createDomain($this->domains[1]);
BroadcastQueue::createJob(
'owner@example.test', 'sender@example.test', 'PostfixAdmin', 'Shared job', 'Body',
$this->domains, true, []
);
$visible = BroadcastQueue::getJobs(20, [$this->domains[0]], false, 'owner@example.test');
$this->assertSame([$own], array_map('intval', array_column($visible, 'id')));
$this->assertSame([], BroadcastQueue::getJobs(20, [], false, 'owner@example.test'));
}
public function testRecipientReportCanReadRowsPastFirstPage(): void
{
$this->createDomain($this->domains[0]);
$recipients = [];
for ($i = 0; $i < 205; $i++) {
$recipients[] = "user$i@{$this->domains[0]}";
}
$jobId = BroadcastQueue::createJob(
'admin@example.test', 'sender@example.test', 'PostfixAdmin', 'Paged report', 'Body',
[$this->domains[0]], true, $recipients
);
$this->assertCount(200, BroadcastQueue::getRecipients($jobId));
$page = BroadcastQueue::getRecipients($jobId, 200, 200);
$this->assertCount(5, $page);
$this->assertSame($recipients[200], $page[0]['recipient']);
$this->assertSame($recipients[204], $page[4]['recipient']);
}
public function testInvalidAliasDestinationFailsWithoutStoppingOtherRecipients(): void
{
$this->createDomain($this->domains[0]);
$this->createMailbox("one@{$this->domains[0]}", $this->domains[0]);
$invalid = '@test.in@"><svg/onload=alert(1)>@test';
$this->createAlias("webmaster@{$this->domains[0]}", $this->domains[0], $invalid);
$recipients = BroadcastQueue::buildRecipients([$this->domains[0]], false);
$this->assertContains($invalid, $recipients);
$jobId = BroadcastQueue::createJob(
'admin@example.test', 'noreply@example.test', 'PostfixAdmin', 'Invalid alias', 'Body',
[$this->domains[0]], false, $recipients
);
$result = BroadcastQueue::processNext(10, true, $jobId);
$this->assertSame('finished', $result['status']);
$this->assertSame(1, $result['failed']);
$this->assertSame(1, $result['sent']);
$rows = array_column(BroadcastQueue::getRecipients($jobId), null, 'recipient');
$this->assertSame('failed', $rows[$invalid]['status']);
$this->assertSame('Invalid recipient address', $rows[$invalid]['error']);
$this->assertNull($rows[$invalid]['smtp_response']);
$this->assertSame('sent', $rows["one@{$this->domains[0]}"]['status']);
}
public function testCreateJobBusyDomainsAndDryRunProcessing(): void
{
$this->createDomain($this->domains[0]);
$recipients = ["one@{$this->domains[0]}", "two@{$this->domains[0]}"];
$jobId = BroadcastQueue::createJob(
'admin@example.test',
'noreply@example.test',
'PostfixAdmin',
'Test subject',
'Test body',
[$this->domains[0]],
true,
$recipients
);
$this->assertGreaterThan(0, $jobId);
$this->assertSame([$this->domains[0]], BroadcastQueue::getBusyDomains($this->domains));
$job = BroadcastQueue::getJob($jobId);
$this->assertSame('pending', $job['status']);
$this->assertSame(2, (int)$job['total_count']);
$this->assertSame([$this->domains[0]], BroadcastQueue::getJobDomains($jobId));
$result = BroadcastQueue::processNext(10, true);
$this->assertSame($jobId, $result['job_id']);
$this->assertSame(2, $result['processed']);
$this->assertSame(2, $result['sent']);
$this->assertSame('finished', $result['status']);
$job = BroadcastQueue::getJob($jobId);
$this->assertSame('finished', $job['status']);
$this->assertSame(2, (int)$job['sent_count']);
$this->assertSame([], BroadcastQueue::getBusyDomains($this->domains));
$statuses = array_column(BroadcastQueue::getRecipients($jobId), 'status');
$this->assertSame(['sent', 'sent'], $statuses);
}
public function testCancelAndResetInactiveJobs(): void
{
$this->createDomain($this->domains[0]);
$jobId = BroadcastQueue::createJob(
'admin@example.test',
'noreply@example.test',
'PostfixAdmin',
'Cancel subject',
'Cancel body',
[$this->domains[0]],
true,
["one@{$this->domains[0]}"]
);
BroadcastQueue::requestCancel($jobId, [], true, '', false);
$this->assertSame('cancelling', BroadcastQueue::getJob($jobId)['status']);
$result = BroadcastQueue::processNext(10, true);
$this->assertSame('cancelled', $result['status']);
$this->assertSame('cancelled', BroadcastQueue::getJob($jobId)['status']);
$this->assertSame('cancelled', BroadcastQueue::getRecipients($jobId)[0]['status']);
BroadcastQueue::resetInactive();
$this->assertSame([], BroadcastQueue::getJobs());
$this->assertSame([], BroadcastQueue::getRecipients($jobId));
$this->assertSame([], BroadcastQueue::getJobDomains($jobId));
}
public function testCancelWithNoBatchRunningFinalisesImmediately(): void
{
$this->createDomain($this->domains[0]);
$jobId = BroadcastQueue::createJob(
'admin@example.test',
'noreply@example.test',
'PostfixAdmin',
'Cancel subject',
'Cancel body',
[$this->domains[0]],
true,
["one@{$this->domains[0]}"]
);
$this->assertTrue(BroadcastQueue::requestCancel($jobId));
$this->assertSame('cancelled', BroadcastQueue::getJob($jobId)['status']);
$this->assertSame('cancelled', BroadcastQueue::getRecipients($jobId)[0]['status']);
}
public function testResetInactiveRollsBackAllDeletesOnFailure(): void
{
if (!db_sqlite()) {
$this->markTestSkipped('SQLite failure injection');
}
$this->createDomain($this->domains[0]);
$jobId = BroadcastQueue::createJob(
'admin@example.test', 'noreply@example.test', 'PostfixAdmin', 'Rollback', 'Body',
[$this->domains[0]], true, ["one@{$this->domains[0]}"]
);
$jobTable = table_by_key('broadcast_job');
$domainTable = table_by_key('broadcast_job_domain');
db_execute("UPDATE $jobTable SET status = 'finished' WHERE id = :id", ['id' => $jobId], true);
db_connect()->exec("CREATE TRIGGER reject_broadcast_domain_delete BEFORE DELETE ON $domainTable BEGIN SELECT RAISE(ABORT, 'test rollback'); END");
try {
$this->expectException(PDOException::class);
BroadcastQueue::resetInactive();
} finally {
db_connect()->exec('DROP TRIGGER reject_broadcast_domain_delete');
$this->assertNotEmpty(BroadcastQueue::getJob($jobId));
$this->assertCount(1, BroadcastQueue::getRecipients($jobId));
$this->assertCount(1, BroadcastQueue::getJobDomains($jobId));
}
}
public function testResetInactiveIncludesJobsBeyondDisplayLimit(): void
{
$this->createDomain($this->domains[0]);
$jobTable = table_by_key('broadcast_job');
for ($i = 0; $i < 101; $i++) {
$jobId = BroadcastQueue::createJob(
'admin@example.test', 'noreply@example.test', 'PostfixAdmin', 'Old job', 'Body',
[$this->domains[0]], true, ["one@{$this->domains[0]}"]
);
db_execute("UPDATE $jobTable SET status = 'finished' WHERE id = :id", ['id' => $jobId], true);
}
BroadcastQueue::resetInactive($this->domains, false, 'admin@example.test');
$this->assertSame([], BroadcastQueue::getJobs(0));
}
public function testCancellationKeepsInterruptedSubmissionUnconfirmed(): void
{
$this->createDomain($this->domains[0]);
$jobId = BroadcastQueue::createJob(
'admin@example.test',
'noreply@example.test',
'PostfixAdmin',
'Cancel interrupted',
'Body',
[$this->domains[0]],
true,
["one@{$this->domains[0]}", "two@{$this->domains[0]}"]
);
$recipientTable = table_by_key('broadcast_recipient');
db_execute(
"UPDATE $recipientTable SET status = 'sending', smtp_response = 'Message-ID: <test@invalid>' WHERE job_id = :job_id AND recipient = :recipient",
['job_id' => $jobId, 'recipient' => "one@{$this->domains[0]}"]
);
$this->assertTrue(BroadcastQueue::requestCancel($jobId));
$this->assertSame('cancelled', BroadcastQueue::getJob($jobId)['status']);
$this->assertSame(['unknown', 'cancelled'], array_column(BroadcastQueue::getRecipients($jobId), 'status'));
$this->assertSame(1, BroadcastQueue::getUnknownCount($jobId));
$unknown = BroadcastQueue::getUnknownRecipients($jobId)[0];
$this->assertTrue(BroadcastQueue::resolveUnknown($jobId, (int)$unknown['id'], 'retry'));
$this->assertSame('pending', BroadcastQueue::getJob($jobId)['status']);
$this->assertSame('finished', BroadcastQueue::processNext(1, true, $jobId)['status']);
}
public function testCancelWhileBatchRunningLeavesJobCancelling(): void
{
$this->createDomain($this->domains[0]);
$jobId = BroadcastQueue::createJob(
'admin@example.test',
'noreply@example.test',
'PostfixAdmin',
'Cancel subject',
'Cancel body',
[$this->domains[0]],
true,
["one@{$this->domains[0]}"]
);
$lock = BroadcastQueue::acquireProcessLock();
$this->assertNotNull($lock);
try {
$this->assertTrue(BroadcastQueue::requestCancel($jobId));
$this->assertSame('cancelling', BroadcastQueue::getJob($jobId)['status']);
} finally {
BroadcastQueue::releaseProcessLock($lock);
}
}
public function testInterruptedSubmissionIsReportedWithoutRetryAndJobsStayIndependent(): void
{
$this->createDomain($this->domains[0]);
$this->createDomain($this->domains[1]);
$first = BroadcastQueue::createJob(
'admin@example.test',
'noreply@example.test',
'PostfixAdmin',
'First',
'Body',
[$this->domains[0]],
true,
["one@{$this->domains[0]}", "two@{$this->domains[0]}"]
);
$second = BroadcastQueue::createJob(
'admin@example.test',
'noreply@example.test',
'PostfixAdmin',
'Second',
'Body',
[$this->domains[1]],
true,
["one@{$this->domains[1]}"]
);
$recipientTable = table_by_key('broadcast_recipient');
db_execute(
"UPDATE $recipientTable SET status = 'sending', smtp_response = 'Message-ID: <test@invalid>' WHERE job_id = :job_id AND recipient = :recipient",
['job_id' => $first, 'recipient' => "one@{$this->domains[0]}"]
);
BroadcastQueue::recoverInterrupted($first);
$result = BroadcastQueue::processNext(1, true, $second);
$this->assertSame($second, $result['job_id']);
$this->assertSame('finished', $result['status']);
$result = BroadcastQueue::processNext(1, true, $first);
$this->assertSame('finished', $result['status']);
$job = BroadcastQueue::getJob($first);
$this->assertSame(1, (int)$job['sent_count']);
$this->assertSame(0, (int)$job['failed_count']);
$this->assertSame(1, BroadcastQueue::getUnknownCount($first));
$statuses = array_column(BroadcastQueue::getRecipients($first), 'status');
$this->assertSame(['unknown', 'sent'], $statuses);
$this->assertSame('Message-ID: <test@invalid>', BroadcastQueue::getRecipients($first)[0]['smtp_response']);
}
public function testResolveUnknownRecipientsIndividually(): void
{
$this->createDomain($this->domains[0]);
$jobId = BroadcastQueue::createJob(
'admin@example.test',
'noreply@example.test',
'PostfixAdmin',
'Review',
'Body',
[$this->domains[0]],
true,
["one@{$this->domains[0]}", "two@{$this->domains[0]}"]
);
$recipientTable = table_by_key('broadcast_recipient');
$jobTable = table_by_key('broadcast_job');
db_execute("UPDATE $recipientTable SET status = 'unknown', smtp_response = 'Message-ID: <test@invalid>' WHERE job_id = :job_id", ['job_id' => $jobId]);
db_execute("UPDATE $jobTable SET status = 'finished' WHERE id = :id", ['id' => $jobId]);
$recipients = BroadcastQueue::getUnknownRecipients($jobId);
$this->assertCount(2, $recipients);
BroadcastQueue::resetInactive();
$this->assertCount(2, BroadcastQueue::getUnknownRecipients($jobId));
$this->assertFalse(BroadcastQueue::resolveUnknown($jobId, (int)$recipients[0]['id'], 'retry', [], false, 'other@example.test'));
$this->assertTrue(BroadcastQueue::resolveUnknown($jobId, (int)$recipients[0]['id'], 'retry', $this->domains, false, 'admin@example.test'));
$this->assertSame('pending', BroadcastQueue::getJob($jobId)['status']);
$this->assertTrue(BroadcastQueue::resolveUnknown($jobId, (int)$recipients[1]['id'], 'discard', $this->domains, false, 'admin@example.test'));
$this->assertSame(0, BroadcastQueue::getUnknownCount($jobId));
$this->assertSame(1, (int)BroadcastQueue::getJob($jobId)['cancelled_count']);
$result = BroadcastQueue::processNext(1, true, $jobId);
$this->assertSame('finished', $result['status']);
$this->assertSame(['sent', 'cancelled'], array_column(BroadcastQueue::getRecipients($jobId), 'status'));
BroadcastQueue::resetInactive();
$this->assertSame([], BroadcastQueue::getJob($jobId));
}
public function testWorkerModeControlsDefaultProcessing(): void
{
$this->createDomain($this->domains[0]);
$jobId = BroadcastQueue::createJob(
'admin@example.test',
'noreply@example.test',
'PostfixAdmin',
'Mode subject',
'Mode body',
[$this->domains[0]],
true,
["one@{$this->domains[0]}"]
);
BroadcastQueue::setWorkerMode(BroadcastQueue::MODE_DRY_RUN);
$this->assertSame(BroadcastQueue::MODE_DRY_RUN, BroadcastQueue::workerMode());
$result = BroadcastQueue::processNext(10);
$this->assertSame($jobId, $result['job_id']);
$this->assertSame('finished', $result['status']);
$recipients = BroadcastQueue::getRecipients($jobId);
$this->assertSame('sent', $recipients[0]['status']);
$this->assertStringStartsWith('dry-run; Message-ID: <broadcast-', $recipients[0]['smtp_response']);
BroadcastQueue::setWorkerMode(BroadcastQueue::MODE_LIVE);
$this->assertSame(BroadcastQueue::MODE_LIVE, BroadcastQueue::workerMode());
}
private function createDomain(string $domain): void
{
db_insert('domain', [
'domain' => $domain,
'description' => 'broadcast test',
'aliases' => 10,
'mailboxes' => 10,
'transport' => 'virtual',
'active' => 1,
]);
}
private function createMailbox(string $username, string $domain, bool $active = true): void
{
db_insert('mailbox', [
'username' => $username,
'password' => 'test',
'name' => 'Broadcast Test',
'maildir' => $username . '/',
'local_part' => substr($username, 0, strpos($username, '@')),
'domain' => $domain,
'active' => $active ? 1 : 0,
]);
}
private function createAlias(string $address, string $domain, string $goto, bool $active = true): void
{
db_insert('alias', [
'address' => $address,
'goto' => $goto,
'domain' => $domain,
'active' => $active ? 1 : 0,
]);
}
private function cleanBroadcastTables(): void
{
db_query('DELETE FROM broadcast_recipient');
db_query('DELETE FROM broadcast_job_domain');
db_query('DELETE FROM broadcast_job');
}
}
/* vim: set expandtab softtabstop=4 tabstop=4 shiftwidth=4: */