Refactor all TAP test suites doing connection checks

This commit refactors more TAP tests to adapt with the recent
introduction of connect_ok() and connect_fails() in PostgresNode,
introduced by 0d1a3343.  This changes the following test suites to use
the same code paths for connection checks:
- Kerberos
- LDAP
- SSL
- Authentication

Those routines are extended to be able to handle optional parameters
that are set depending on each suite's needs, as of:
- custom SQL query.
- expected stderr matching pattern.
- expected stdout matching pattern.
The new design is extensible with more parameters, and there are some
plans for those routines in the future with checks based on the contents
of the backend logs.

Author: Jacob Champion, Michael Paquier
Discussion: https://postgr.es/m/d17b919e27474abfa55d97786cb9cfadfe2b59e9.camel@vmware.com

src/test/authentication/t/001_password.pl

@@ -46,12 +46,19 @@ sub test_role
 
 	$status_string = 'success' if ($expected_res eq 0);
 
-	local $Test::Builder::Level = $Test::Builder::Level + 1;
-
-	my $res = $node->psql('postgres', undef, extra_params => [ '-U', $role, '-w' ]);
-	is($res, $expected_res,
-		"authentication $status_string for method $method, role $role");
-	return;
+	my $connstr = "user=$role";
+	my $testname =
+	  "authentication $status_string for method $method, role $role";
+
+	if ($expected_res eq 0)
+	{
+		$node->connect_ok($connstr, $testname);
+	}
+	else
+	{
+		# No checks of the error message, only the status code.
+		$node->connect_fails($connstr, $testname);
+	}
 }
 
 # Initialize primary node

src/test/authentication/t/002_saslprep.pl

@@ -41,12 +41,20 @@ sub test_login
 
 	$status_string = 'success' if ($expected_res eq 0);
 
+	my $connstr = "user=$role";
+	my $testname =
+	  "authentication $status_string for role $role with password $password";
+
 	$ENV{"PGPASSWORD"} = $password;
-	my $res = $node->psql('postgres', undef, extra_params => [ '-U', $role ]);
-	is($res, $expected_res,
-		"authentication $status_string for role $role with password $password"
-	);
-	return;
+	if ($expected_res eq 0)
+	{
+		$node->connect_ok($connstr, $testname);
+	}
+	else
+	{
+		# No checks of the error message, only the status code.
+		$node->connect_fails($connstr, $testname);
+	}
 }
 
 # Initialize primary node. Force UTF-8 encoding, so that we can use non-ASCII

src/test/kerberos/t/001_auth.pl

@@ -20,7 +20,7 @@ use Time::HiRes qw(usleep);
 
 if ($ENV{with_gssapi} eq 'yes')
 {
-	plan tests => 26;
+	plan tests => 30;
 }
 else
 {
@@ -182,28 +182,25 @@ note "running tests";
 # Test connection success or failure, and if success, that query returns true.
 sub test_access
 {
-	my ($node, $role, $query, $expected_res, $gssencmode, $test_name, $expect_log_msg) = @_;
+	my ($node, $role, $query, $expected_res, $gssencmode, $test_name,
+		$expect_log_msg)
+	  = @_;
 
 	# need to connect over TCP/IP for Kerberos
-	my ($res, $stdoutres, $stderrres) = $node->psql(
-		'postgres',
-		"$query",
-		extra_params => [
-			'-XAtd',
-			$node->connstr('postgres')
-			  . " host=$host hostaddr=$hostaddr $gssencmode",
-			'-U',
-			$role
-		]);
-
-	# If we get a query result back, it should be true.
-	if ($res == $expected_res and $res eq 0)
+	my $connstr = $node->connstr('postgres')
+	  . " user=$role host=$host hostaddr=$hostaddr $gssencmode";
+
+	if ($expected_res eq 0)
 	{
-		is($stdoutres, "t", $test_name);
+		# The result is assumed to match "true", or "t", here.
+		$node->connect_ok(
+			$connstr, $test_name,
+			sql             => $query,
+			expected_stdout => qr/t/);
 	}
 	else
 	{
-		is($res, $expected_res, $test_name);
+		$node->connect_fails($connstr, $test_name);
 	}
 
 	# Verify specified log message is logged in the log file.
@@ -227,20 +224,12 @@ sub test_query
 	my ($node, $role, $query, $expected, $gssencmode, $test_name) = @_;
 
 	# need to connect over TCP/IP for Kerberos
-	my ($res, $stdoutres, $stderrres) = $node->psql(
-		'postgres',
-		"$query",
-		extra_params => [
-			'-XAtd',
-			$node->connstr('postgres')
-			  . " host=$host hostaddr=$hostaddr $gssencmode",
-			'-U',
-			$role
-		]);
-
-	is($res, 0, $test_name);
-	like($stdoutres, $expected, $test_name);
-	is($stderrres, "", $test_name);
+	my $connstr = $node->connstr('postgres')
+	  . " user=$role host=$host hostaddr=$hostaddr $gssencmode";
+
+	my ($stdoutres, $stderrres);
+
+	$node->connect_ok($connstr, $test_name, $query, $expected);
 	return;
 }
 

src/test/ldap/t/001_auth.pl

@@ -163,12 +163,17 @@ note "running tests";
 sub test_access
 {
 	my ($node, $role, $expected_res, $test_name) = @_;
-
-	my $res =
-	  $node->psql('postgres', undef,
-				  extra_params => [ '-U', $role, '-c', 'SELECT 1' ]);
-	is($res, $expected_res, $test_name);
-	return;
+	my $connstr = "user=$role";
+
+	if ($expected_res eq 0)
+	{
+		$node->connect_ok($connstr, $test_name);
+	}
+	else
+	{
+		# No checks of the error message, only the status code.
+		$node->connect_fails($connstr, $test_name);
+	}
 }
 
 note "simple bind";

src/test/perl/PostgresNode.pm

@@ -1860,47 +1860,94 @@ sub interactive_psql
 
 =pod
 
-=item $node->connect_ok($connstr, $test_name)
+=item $node->connect_ok($connstr, $test_name, %params)
 
 Attempt a connection with a custom connection string.  This is expected
 to succeed.
 
+=over
+
+=item sql => B<value>
+
+If this parameter is set, this query is used for the connection attempt
+instead of the default.
+
+=item expected_stdout => B<value>
+
+If this regular expression is set, matches it with the output generated.
+
+=back
+
 =cut
 
 sub connect_ok
 {
 	local $Test::Builder::Level = $Test::Builder::Level + 1;
-	my ($self, $connstr, $test_name) = @_;
+	my ($self, $connstr, $test_name, %params) = @_;
+
+	my $sql;
+	if (defined($params{sql}))
+	{
+		$sql = $params{sql};
+	}
+	else
+	{
+		$sql = "SELECT \$\$connected with $connstr\$\$";
+	}
+
+	# Never prompt for a password, any callers of this routine should
+	# have set up things properly, and this should not block.
 	my ($ret, $stdout, $stderr) = $self->psql(
 		'postgres',
-		"SELECT \$\$connected with $connstr\$\$",
+		$sql,
+		extra_params  => ['-w'],
 		connstr       => "$connstr",
 		on_error_stop => 0);
 
-	ok($ret == 0, $test_name);
+	is($ret, 0, $test_name);
+
+	if (defined($params{expected_stdout}))
+	{
+		like($stdout, $params{expected_stdout}, "$test_name: matches");
+	}
 }
 
 =pod
 
-=item $node->connect_fails($connstr, $expected_stderr, $test_name)
+=item $node->connect_fails($connstr, $test_name, %params)
 
 Attempt a connection with a custom connection string.  This is expected
-to fail with a message that matches the regular expression
-$expected_stderr.
+to fail.
+
+=over
+
+=item expected_stderr => B<value>
+
+If this regular expression is set, matches it with the output generated.
+
+=back
 
 =cut
 
 sub connect_fails
 {
 	local $Test::Builder::Level = $Test::Builder::Level + 1;
-	my ($self, $connstr, $expected_stderr, $test_name) = @_;
+	my ($self, $connstr, $test_name, %params) = @_;
+
+	# Never prompt for a password, any callers of this routine should
+	# have set up things properly, and this should not block.
 	my ($ret, $stdout, $stderr) = $self->psql(
 		'postgres',
 		"SELECT \$\$connected with $connstr\$\$",
+		extra_params => ['-w'],
 		connstr      => "$connstr");
 
-	ok($ret != 0, $test_name);
-	like($stderr, $expected_stderr, "$test_name: matches");
+	isnt($ret, 0, $test_name);
+
+	if (defined($params{expected_stderr}))
+	{
+		like($stderr, $params{expected_stderr}, "$test_name: matches");
+	}
 }
 
 =pod

src/test/ssl/t/001_ssltests.pl

@@ -137,8 +137,8 @@ $common_connstr =
 # The server should not accept non-SSL connections.
 $node->connect_fails(
 	"$common_connstr sslmode=disable",
-	qr/\Qno pg_hba.conf entry\E/,
-	"server doesn't accept non-SSL connections");
+	"server doesn't accept non-SSL connections",
+	expected_stderr => qr/\Qno pg_hba.conf entry\E/);
 
 # Try without a root cert. In sslmode=require, this should work. In verify-ca
 # or verify-full mode it should fail.
@@ -147,34 +147,34 @@ $node->connect_ok(
 	"connect without server root cert sslmode=require");
 $node->connect_fails(
 	"$common_connstr sslrootcert=invalid sslmode=verify-ca",
-	qr/root certificate file "invalid" does not exist/,
-	"connect without server root cert sslmode=verify-ca");
+	"connect without server root cert sslmode=verify-ca",
+	expected_stderr => qr/root certificate file "invalid" does not exist/);
 $node->connect_fails(
 	"$common_connstr sslrootcert=invalid sslmode=verify-full",
-	qr/root certificate file "invalid" does not exist/,
-	"connect without server root cert sslmode=verify-full");
+	"connect without server root cert sslmode=verify-full",
+	expected_stderr => qr/root certificate file "invalid" does not exist/);
 
 # Try with wrong root cert, should fail. (We're using the client CA as the
 # root, but the server's key is signed by the server CA.)
 $node->connect_fails(
 	"$common_connstr sslrootcert=ssl/client_ca.crt sslmode=require",
-	qr/SSL error: certificate verify failed/,
-	"connect with wrong server root cert sslmode=require");
+	"connect with wrong server root cert sslmode=require",
+	expected_stderr => qr/SSL error: certificate verify failed/);
 $node->connect_fails(
 	"$common_connstr sslrootcert=ssl/client_ca.crt sslmode=verify-ca",
-	qr/SSL error: certificate verify failed/,
-	"connect with wrong server root cert sslmode=verify-ca");
+	"connect with wrong server root cert sslmode=verify-ca",
+	expected_stderr => qr/SSL error: certificate verify failed/);
 $node->connect_fails(
 	"$common_connstr sslrootcert=ssl/client_ca.crt sslmode=verify-full",
-	qr/SSL error: certificate verify failed/,
-	"connect with wrong server root cert sslmode=verify-full");
+	"connect with wrong server root cert sslmode=verify-full",
+	expected_stderr => qr/SSL error: certificate verify failed/);
 
 # Try with just the server CA's cert. This fails because the root file
 # must contain the whole chain up to the root CA.
 $node->connect_fails(
 	"$common_connstr sslrootcert=ssl/server_ca.crt sslmode=verify-ca",
-	qr/SSL error: certificate verify failed/,
-	"connect with server CA cert, without root CA");
+	"connect with server CA cert, without root CA",
+	expected_stderr => qr/SSL error: certificate verify failed/);
 
 # And finally, with the correct root cert.
 $node->connect_ok(
@@ -206,14 +206,14 @@ $node->connect_ok(
 # A CRL belonging to a different CA is not accepted, fails
 $node->connect_fails(
 	"$common_connstr sslrootcert=ssl/root+server_ca.crt sslmode=verify-ca sslcrl=ssl/client.crl",
-	qr/SSL error: certificate verify failed/,
-	"CRL belonging to a different CA");
+	"CRL belonging to a different CA",
+	expected_stderr => qr/SSL error: certificate verify failed/);
 
 # The same for CRL directory
 $node->connect_fails(
 	"$common_connstr sslrootcert=ssl/root+server_ca.crt sslmode=verify-ca sslcrldir=ssl/client-crldir",
-	qr/SSL error: certificate verify failed/,
-	"directory CRL belonging to a different CA");
+	"directory CRL belonging to a different CA",
+	expected_stderr => qr/SSL error: certificate verify failed/);
 
 # With the correct CRL, succeeds (this cert is not revoked)
 $node->connect_ok(
@@ -237,8 +237,10 @@ $node->connect_ok(
 	"mismatch between host name and server certificate sslmode=verify-ca");
 $node->connect_fails(
 	"$common_connstr sslmode=verify-full host=wronghost.test",
-	qr/\Qserver certificate for "common-name.pg-ssltest.test" does not match host name "wronghost.test"\E/,
-	"mismatch between host name and server certificate sslmode=verify-full");
+	"mismatch between host name and server certificate sslmode=verify-full",
+	expected_stderr =>
+	  qr/\Qserver certificate for "common-name.pg-ssltest.test" does not match host name "wronghost.test"\E/
+);
 
 # Test Subject Alternative Names.
 switch_server_cert($node, 'server-multiple-alt-names');
@@ -257,12 +259,16 @@ $node->connect_ok("$common_connstr host=foo.wildcard.pg-ssltest.test",
 
 $node->connect_fails(
 	"$common_connstr host=wronghost.alt-name.pg-ssltest.test",
-	qr/\Qserver certificate for "dns1.alt-name.pg-ssltest.test" (and 2 other names) does not match host name "wronghost.alt-name.pg-ssltest.test"\E/,
-	"host name not matching with X.509 Subject Alternative Names");
+	"host name not matching with X.509 Subject Alternative Names",
+	expected_stderr =>
+	  qr/\Qserver certificate for "dns1.alt-name.pg-ssltest.test" (and 2 other names) does not match host name "wronghost.alt-name.pg-ssltest.test"\E/
+);
 $node->connect_fails(
 	"$common_connstr host=deep.subdomain.wildcard.pg-ssltest.test",
-	qr/\Qserver certificate for "dns1.alt-name.pg-ssltest.test" (and 2 other names) does not match host name "deep.subdomain.wildcard.pg-ssltest.test"\E/,
-	"host name not matching with X.509 Subject Alternative Names wildcard");
+	"host name not matching with X.509 Subject Alternative Names wildcard",
+	expected_stderr =>
+	  qr/\Qserver certificate for "dns1.alt-name.pg-ssltest.test" (and 2 other names) does not match host name "deep.subdomain.wildcard.pg-ssltest.test"\E/
+);
 
 # Test certificate with a single Subject Alternative Name. (this gives a
 # slightly different error message, that's all)
@@ -277,12 +283,15 @@ $node->connect_ok(
 
 $node->connect_fails(
 	"$common_connstr host=wronghost.alt-name.pg-ssltest.test",
-	qr/\Qserver certificate for "single.alt-name.pg-ssltest.test" does not match host name "wronghost.alt-name.pg-ssltest.test"\E/,
-	"host name not matching with a single X.509 Subject Alternative Name");
+	"host name not matching with a single X.509 Subject Alternative Name",
+	expected_stderr =>
+	  qr/\Qserver certificate for "single.alt-name.pg-ssltest.test" does not match host name "wronghost.alt-name.pg-ssltest.test"\E/
+);
 $node->connect_fails(
 	"$common_connstr host=deep.subdomain.wildcard.pg-ssltest.test",
-	qr/\Qserver certificate for "single.alt-name.pg-ssltest.test" does not match host name "deep.subdomain.wildcard.pg-ssltest.test"\E/,
-	"host name not matching with a single X.509 Subject Alternative Name wildcard"
+	"host name not matching with a single X.509 Subject Alternative Name wildcard",
+	expected_stderr =>
+	  qr/\Qserver certificate for "single.alt-name.pg-ssltest.test" does not match host name "deep.subdomain.wildcard.pg-ssltest.test"\E/
 );
 
 # Test server certificate with a CN and SANs. Per RFCs 2818 and 6125, the CN
@@ -298,8 +307,10 @@ $node->connect_ok("$common_connstr host=dns2.alt-name.pg-ssltest.test",
 	"certificate with both a CN and SANs 2");
 $node->connect_fails(
 	"$common_connstr host=common-name.pg-ssltest.test",
-	qr/\Qserver certificate for "dns1.alt-name.pg-ssltest.test" (and 1 other name) does not match host name "common-name.pg-ssltest.test"\E/,
-	"certificate with both a CN and SANs ignores CN");
+	"certificate with both a CN and SANs ignores CN",
+	expected_stderr =>
+	  qr/\Qserver certificate for "dns1.alt-name.pg-ssltest.test" (and 1 other name) does not match host name "common-name.pg-ssltest.test"\E/
+);
 
 # Finally, test a server certificate that has no CN or SANs. Of course, that's
 # not a very sensible certificate, but libpq should handle it gracefully.
@@ -313,8 +324,9 @@ $node->connect_ok(
 $node->connect_fails(
 	$common_connstr . " "
 	  . "sslmode=verify-full host=common-name.pg-ssltest.test",
-	qr/could not get server's host name from server certificate/,
-	"server certificate without CN or SANs sslmode=verify-full");
+	"server certificate without CN or SANs sslmode=verify-full",
+	expected_stderr =>
+	  qr/could not get server's host name from server certificate/);
 
 # Test that the CRL works
 switch_server_cert($node, 'server-revoked');
@@ -328,12 +340,12 @@ $node->connect_ok(
 	"connects without client-side CRL");
 $node->connect_fails(
 	"$common_connstr sslrootcert=ssl/root+server_ca.crt sslmode=verify-ca sslcrl=ssl/root+server.crl",
-	qr/SSL error: certificate verify failed/,
-	"does not connect with client-side CRL file");
+	"does not connect with client-side CRL file",
+	expected_stderr => qr/SSL error: certificate verify failed/);
 $node->connect_fails(
 	"$common_connstr sslrootcert=ssl/root+server_ca.crt sslmode=verify-ca sslcrldir=ssl/root+server-crldir",
-	qr/SSL error: certificate verify failed/,
-	"does not connect with client-side CRL directory");
+	"does not connect with client-side CRL directory",
+	expected_stderr => qr/SSL error: certificate verify failed/);
 
 # pg_stat_ssl
 command_like(
@@ -355,16 +367,16 @@ $node->connect_ok(
 	"connection success with correct range of TLS protocol versions");
 $node->connect_fails(
 	"$common_connstr sslrootcert=ssl/root+server_ca.crt sslmode=require ssl_min_protocol_version=TLSv1.2 ssl_max_protocol_version=TLSv1.1",
-	qr/invalid SSL protocol version range/,
-	"connection failure with incorrect range of TLS protocol versions");
+	"connection failure with incorrect range of TLS protocol versions",
+	expected_stderr => qr/invalid SSL protocol version range/);
 $node->connect_fails(
 	"$common_connstr sslrootcert=ssl/root+server_ca.crt sslmode=require ssl_min_protocol_version=incorrect_tls",
-	qr/invalid ssl_min_protocol_version value/,
-	"connection failure with an incorrect SSL protocol minimum bound");
+	"connection failure with an incorrect SSL protocol minimum bound",
+	expected_stderr => qr/invalid ssl_min_protocol_version value/);
 $node->connect_fails(
 	"$common_connstr sslrootcert=ssl/root+server_ca.crt sslmode=require ssl_max_protocol_version=incorrect_tls",
-	qr/invalid ssl_max_protocol_version value/,
-	"connection failure with an incorrect SSL protocol maximum bound");
+	"connection failure with an incorrect SSL protocol maximum bound",
+	expected_stderr => qr/invalid ssl_max_protocol_version value/);
 
 ### Server-side tests.
 ###
@@ -378,8 +390,8 @@ $common_connstr =
 # no client cert
 $node->connect_fails(
 	"$common_connstr user=ssltestuser sslcert=invalid",
-	qr/connection requires a valid client certificate/,
-	"certificate authorization fails without client cert");
+	"certificate authorization fails without client cert",
+	expected_stderr => qr/connection requires a valid client certificate/);
 
 # correct client cert in unencrypted PEM
 $node->connect_ok(
@@ -408,8 +420,9 @@ $node->connect_ok(
 # correct client cert in encrypted PEM with wrong password
 $node->connect_fails(
 	"$common_connstr user=ssltestuser sslcert=ssl/client.crt sslkey=ssl/client-encrypted-pem_tmp.key sslpassword='wrong'",
-	qr!\Qprivate key file "ssl/client-encrypted-pem_tmp.key": bad decrypt\E!,
-	"certificate authorization fails with correct client cert and wrong password in encrypted PEM format"
+	"certificate authorization fails with correct client cert and wrong password in encrypted PEM format",
+	expected_stderr =>
+	  qr!\Qprivate key file "ssl/client-encrypted-pem_tmp.key": bad decrypt\E!
 );
 
 
@@ -446,6 +459,7 @@ TODO:
 	# correct client cert in encrypted PEM with empty password
 	$node->connect_fails(
 		"$common_connstr user=ssltestuser sslcert=ssl/client.crt sslkey=ssl/client-encrypted-pem_tmp.key sslpassword=''",
+		expected_stderr =>
 		  qr!\Qprivate key file "ssl/client-encrypted-pem_tmp.key": processing error\E!,
 		"certificate authorization fails with correct client cert and empty password in encrypted PEM format"
 	);
@@ -453,6 +467,7 @@ TODO:
 	# correct client cert in encrypted PEM with no password
 	$node->connect_fails(
 		"$common_connstr user=ssltestuser sslcert=ssl/client.crt sslkey=ssl/client-encrypted-pem_tmp.key",
+		expected_stderr =>
 		  qr!\Qprivate key file "ssl/client-encrypted-pem_tmp.key": processing error\E!,
 		"certificate authorization fails with correct client cert and no password in encrypted PEM format"
 	);
@@ -485,22 +500,24 @@ SKIP:
 
 	$node->connect_fails(
 		"$common_connstr user=ssltestuser sslcert=ssl/client.crt sslkey=ssl/client_wrongperms_tmp.key",
-		qr!\Qprivate key file "ssl/client_wrongperms_tmp.key" has group or world access\E!,
-		"certificate authorization fails because of file permissions");
+		"certificate authorization fails because of file permissions",
+		expected_stderr =>
+		  qr!\Qprivate key file "ssl/client_wrongperms_tmp.key" has group or world access\E!
+	);
 }
 
 # client cert belonging to another user
 $node->connect_fails(
 	"$common_connstr user=anotheruser sslcert=ssl/client.crt sslkey=ssl/client_tmp.key",
-	qr/certificate authentication failed for user "anotheruser"/,
-	"certificate authorization fails with client cert belonging to another user"
-);
+	"certificate authorization fails with client cert belonging to another user",
+	expected_stderr =>
+	  qr/certificate authentication failed for user "anotheruser"/);
 
 # revoked client cert
 $node->connect_fails(
 	"$common_connstr user=ssltestuser sslcert=ssl/client-revoked.crt sslkey=ssl/client-revoked_tmp.key",
-	qr/SSL error: sslv3 alert certificate revoked/,
-	"certificate authorization fails with revoked client cert");
+	"certificate authorization fails with revoked client cert",
+	expected_stderr => qr/SSL error: sslv3 alert certificate revoked/);
 
 # Check that connecting with auth-option verify-full in pg_hba:
 # works, iff username matches Common Name
@@ -515,9 +532,9 @@ $node->connect_ok(
 
 $node->connect_fails(
 	"$common_connstr user=anotheruser sslcert=ssl/client.crt sslkey=ssl/client_tmp.key",
-	qr/FATAL: .* "trust" authentication failed for user "anotheruser"/,
-	"auth_option clientcert=verify-full fails with mismatching username and Common Name"
-);
+	"auth_option clientcert=verify-full fails with mismatching username and Common Name",
+	expected_stderr =>
+	  qr/FATAL: .* "trust" authentication failed for user "anotheruser"/,);
 
 # Check that connecting with auth-optionverify-ca in pg_hba :
 # works, when username doesn't match Common Name
@@ -536,16 +553,18 @@ $node->connect_ok(
 	"intermediate client certificate is provided by client");
 $node->connect_fails(
 	$common_connstr . " " . "sslmode=require sslcert=ssl/client.crt",
-	qr/SSL error: tlsv1 alert unknown ca/, "intermediate client certificate is missing");
+	"intermediate client certificate is missing",
+	expected_stderr => qr/SSL error: tlsv1 alert unknown ca/);
 
 # test server-side CRL directory
-switch_server_cert($node, 'server-cn-only', undef, undef, 'root+client-crldir');
+switch_server_cert($node, 'server-cn-only', undef, undef,
+	'root+client-crldir');
 
 # revoked client cert
 $node->connect_fails(
 	"$common_connstr user=ssltestuser sslcert=ssl/client-revoked.crt sslkey=ssl/client-revoked_tmp.key",
-	qr/SSL error: sslv3 alert certificate revoked/,
-	"certificate authorization fails with revoked client cert with server-side CRL directory");
+	"certificate authorization fails with revoked client cert with server-side CRL directory",
+	expected_stderr => qr/SSL error: sslv3 alert certificate revoked/);
 
 # clean up
 foreach my $key (@keys)

src/test/ssl/t/002_scram.pl

@@ -60,8 +60,8 @@ $node->connect_ok(
 # Test channel_binding
 $node->connect_fails(
 	"$common_connstr user=ssltestuser channel_binding=invalid_value",
-	qr/invalid channel_binding value: "invalid_value"/,
-	"SCRAM with SSL and channel_binding=invalid_value");
+	"SCRAM with SSL and channel_binding=invalid_value",
+	expected_stderr => qr/invalid channel_binding value: "invalid_value"/);
 $node->connect_ok("$common_connstr user=ssltestuser channel_binding=disable",
 	"SCRAM with SSL and channel_binding=disable");
 if ($supports_tls_server_end_point)
@@ -74,15 +74,19 @@ else
 {
 	$node->connect_fails(
 		"$common_connstr user=ssltestuser channel_binding=require",
-		qr/channel binding is required, but server did not offer an authentication method that supports channel binding/,
-		"SCRAM with SSL and channel_binding=require");
+		"SCRAM with SSL and channel_binding=require",
+		expected_stderr =>
+		  qr/channel binding is required, but server did not offer an authentication method that supports channel binding/
+	);
 }
 
 # Now test when the user has an MD5-encrypted password; should fail
 $node->connect_fails(
 	"$common_connstr user=md5testuser channel_binding=require",
-	qr/channel binding required but not supported by server's authentication request/,
-	"MD5 with SSL and channel_binding=require");
+	"MD5 with SSL and channel_binding=require",
+	expected_stderr =>
+	  qr/channel binding required but not supported by server's authentication request/
+);
 
 # Now test with auth method 'cert' by connecting to 'certdb'. Should fail,
 # because channel binding is not performed.  Note that ssl/client.key may
@@ -93,8 +97,10 @@ copy("ssl/client.key", $client_tmp_key);
 chmod 0600, $client_tmp_key;
 $node->connect_fails(
 	"sslcert=ssl/client.crt sslkey=$client_tmp_key sslrootcert=invalid hostaddr=$SERVERHOSTADDR dbname=certdb user=ssltestuser channel_binding=require",
-	qr/channel binding required, but server authenticated client without channel binding/,
-	"Cert authentication and channel_binding=require");
+	"Cert authentication and channel_binding=require",
+	expected_stderr =>
+	  qr/channel binding required, but server authenticated client without channel binding/
+);
 
 # clean up
 unlink($client_tmp_key);