diff --git a/src/test/ssl/t/001_ssltests.pl b/src/test/ssl/t/001_ssltests.pl
index a454bb0274..c0680f39d6 100644
--- a/src/test/ssl/t/001_ssltests.pl
+++ b/src/test/ssl/t/001_ssltests.pl
@@ -52,9 +52,11 @@ foreach my $key (@keys)
 
 # Also make a copy of that explicitly world-readable.  We can't
 # necessarily rely on the file in the source tree having those
-# permissions.
+# permissions.  Add it to @keys to include it in the final clean
+# up phase.
 copy("ssl/client.key", "ssl/client_wrongperms_tmp.key");
 chmod 0644, "ssl/client_wrongperms_tmp.key";
+push @keys, 'client_wrongperms';
 
 #### Set up the server.
 
diff --git a/src/test/ssl/t/002_scram.pl b/src/test/ssl/t/002_scram.pl
index ee6e26d732..a1ab911988 100644
--- a/src/test/ssl/t/002_scram.pl
+++ b/src/test/ssl/t/002_scram.pl
@@ -89,14 +89,20 @@ test_connect_fails(
 	qr/channel binding required but not supported by server's authentication request/,
 	"MD5 with SSL and channel_binding=require");
 
-# Now test with auth method 'cert' by connecting to 'certdb'. Should
-# fail, because channel binding is not performed.
-copy("ssl/client.key", "ssl/client_tmp.key");
-chmod 0600, "ssl/client_tmp.key";
+# Now test with auth method 'cert' by connecting to 'certdb'. Should fail,
+# because channel binding is not performed.  Note that ssl/client.key may
+# be used in a different test, so the name of this temporary client key
+# is chosen here to be unique.
+my $client_tmp_key = "ssl/client_scram_tmp.key";
+copy("ssl/client.key", $client_tmp_key);
+chmod 0600, $client_tmp_key;
 test_connect_fails(
-	"sslcert=ssl/client.crt sslkey=ssl/client_tmp.key hostaddr=$SERVERHOSTADDR",
+	"sslcert=ssl/client.crt sslkey=$client_tmp_key hostaddr=$SERVERHOSTADDR",
 	"dbname=certdb user=ssltestuser channel_binding=require",
 	qr/channel binding required, but server authenticated client without channel binding/,
 	"Cert authentication and channel_binding=require");
 
+# clean up
+unlink($client_tmp_key);
+
 done_testing($number_of_tests);