Deprecate extract-certs-pem.zeek and add log-certs-base64.zeek

Extract-certs-pem writes pem files to a dedicated file; since it does
not really work in cluster-environments it was never super helpful.

This commit deprecates this file and, instead, adds
log-certs-base64.zeek, which adds the base64-encoded certificate (which
is basically equivalent with a PEM) to the log-file. Since, nowadays,
the log-files are deduplicates this should not add a huge overhead.

scripts/policy/protocols/ssl/extract-certs-pem.zeek

@@ -1,3 +1,5 @@
+@deprecated "Remove in v5.1. Use log-certs-base64.zeek instead."
+
 ##! This script is used to extract host certificates seen on the wire to disk
 ##! after being converted to PEM files.  The certificates will be stored in
 ##! a single file, one for local certificates and one for remote certificates.

scripts/policy/protocols/ssl/log-certs-base64.zeek

@@ -0,0 +1,19 @@
+##! This script is used to extract certificates seen on the wire to Zeek log files.
+##! The certificates are base64-encoded and written to ssl.log, to the newly added cert
+##! field.
+
+@load base/protocols/ssl
+@load base/files/x509
+
+redef record X509::Info += {
+	## Base64 endoded X.509 certificate.
+	cert: string &log &optional;
+};
+
+event x509_certificate(f: fa_file, cert_ref: opaque of x509, cert: X509::Certificate) &priority=1
+	{
+	if ( ! f$info?$x509 )
+		return;
+
+	f$info$x509$cert = encode_base64(x509_get_certificate_string(cert_ref));
+	}

scripts/test-all-policy.zeek

@@ -102,9 +102,10 @@
 @load protocols/ssh/interesting-hostnames.zeek
 @load protocols/ssh/software.zeek
 @load protocols/ssl/expiring-certs.zeek
-@load protocols/ssl/extract-certs-pem.zeek
+# @load protocols/ssl/extract-certs-pem.zeek
 @load protocols/ssl/heartbleed.zeek
 @load protocols/ssl/known-certs.zeek
+@load protocols/ssl/log-certs-base64.zeek
 @load protocols/ssl/log-hostcerts-only.zeek
 #@load protocols/ssl/notary.zeek
 @load protocols/ssl/validate-certs.zeek

scripts/zeekygen/__load__.zeek

@@ -7,6 +7,7 @@
 @load frameworks/files/extract-all-files.zeek
 @load policy/misc/dump-events.zeek
 @load policy/protocols/conn/speculative-service.zeek
+@load policy/protocols/ssl/extract-certs-pem.zeek
 
 @load ./example.zeek
 

testing/btest/Baseline/coverage.bare-mode-errors/errors

@@ -1,5 +1,7 @@
 ### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63.
 ### NOTE: This file has been sorted with diff-sort.
+warning in <...>/extract-certs-pem.zeek, line 1: deprecated script loaded from <...>/__load__.zeek:10 "Remove in v5.1. Use log-certs-base64.zeek instead."
+warning in <...>/extract-certs-pem.zeek, line 1: deprecated script loaded from command line arguments "Remove in v5.1. Use log-certs-base64.zeek instead."
 warning in <...>/log-ocsp.zeek, line 1: deprecated script loaded from <...>/test-all-policy.zeek:44 ("Remove in v5.1. OCSP logging is now disabled by default")
 warning in <...>/log-ocsp.zeek, line 1: deprecated script loaded from <...>/test-all-policy.zeek:44 ("Remove in v5.1. OCSP logging is now disabled by default")
 warning in <...>/log-ocsp.zeek, line 1: deprecated script loaded from command line arguments ("Remove in v5.1. OCSP logging is now disabled by default")

testing/btest/Baseline/scripts.policy.protocols.ssl.log-certs-base64/x509.log

@@ -0,0 +1,13 @@
+### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63.
+#separator \x09
+#set_separator	,
+#empty_field	(empty)
+#unset_field	-
+#path	x509
+#open XXXX-XX-XX-XX-XX-XX
+#fields	ts	fp	certificate.version	certificate.serial	certificate.subject	certificate.issuer	certificate.not_valid_before	certificate.not_valid_after	certificate.key_alg	certificate.sig_alg	certificate.key_type	certificate.key_length	certificate.exponent	certificate.curve	san.dns	san.uri	san.email	san.ip	basic_constraints.ca	basic_constraints.path_len	host_cert	client_cert	cert
+#types	time	string	count	string	string	string	time	time	string	string	string	count	string	string	vector[string]	vector[string]	vector[string]	vector[addr]	bool	count	bool	bool	string
+XXXXXXXXXX.XXXXXX	a97a22b691caf62c1623d14abf8a31ac915f14d87f77d8a37c47eb4785b484ad	3	1E58FDC12DE4C703	CN=*.google.com,O=Google Inc,L=Mountain View,ST=California,C=US	CN=Google Internet Authority G2,O=Google Inc,C=US	XXXXXXXXXX.XXXXXX	XXXXXXXXXX.XXXXXX	rsaEncryption	sha1WithRSAEncryption	rsa	2048	65537	-	*.google.com,*.android.com,*.appengine.google.com,*.cloud.google.com,*.google-analytics.com,*.google.ca,*.google.cl,*.google.co.in,*.google.co.jp,*.google.co.uk,*.google.com.ar,*.google.com.au,*.google.com.br,*.google.com.co,*.google.com.mx,*.google.com.tr,*.google.com.vn,*.google.de,*.google.es,*.google.fr,*.google.hu,*.google.it,*.google.nl,*.google.pl,*.google.pt,*.googleapis.cn,*.googlecommerce.com,*.googlevideo.com,*.gstatic.com,*.gvt1.com,*.urchin.com,*.url.google.com,*.youtube-nocookie.com,*.youtube.com,*.youtubeeducation.com,*.ytimg.com,android.com,g.co,goo.gl,google-analytics.com,google.com,googlecommerce.com,urchin.com,youtu.be,youtube.com,youtubeeducation.com	-	-	-	F	-	T	F	MIIHPzCCBiegAwIBAgIIHlj9wS3kxwMwDQYJKoZIhvcNAQEFBQAwSTELMAkGA1UEBhMCVVMxEzARBgNVBAoTCkdvb2dsZSBJbmMxJTAjBgNVBAMTHEdvb2dsZSBJbnRlcm5ldCBBdXRob3JpdHkgRzIwHhcNMTQwNDA5MTIwNTA4WhcNMTQwNzA4MDAwMDAwWjBmMQswCQYDVQQGEwJVUzETMBEGA1UECAwKQ2FsaWZvcm5pYTEWMBQGA1UEBwwNTW91bnRhaW4gVmlldzETMBEGA1UECgwKR29vZ2xlIEluYzEVMBMGA1UEAwwMKi5nb29nbGUuY29tMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAz3izpQrGZwvH70JAvgzaS+hOUIlx3JNEZnl3gde66I54riDhWRSn56I1dWQVL8DPSOIjoHIHRDb6GYmQVa9CiKBNvof+GGmif3+rG1FBLa2jAkOWxp83ReuJw/oUd3LqgleFDbVQ1MG8+6YGrbhAWDGib/z3k9GSurmP63h6qZuQTZLruQcau0MRnOKbyV9TKPz93dfqetoPcTZ5XrbYP8PmA9L90bV5i9MBZXCEOs/w//SgDYuE6pu0p4fo0RdW4jk5wqRhQlueja5yoUEIU/d4WnsvvJAIG5BoCMY1y8ngbCimDKshG3Hg0PlCoqMwBxPAJrWAB2dA9JnsxSfNxwIDAQABo4IEDDCCBAgwHQYDVR0lBBYwFAYIKwYBBQUHAwEGCCsGAQUFBwMCMIIC4gYDVR0RBIIC2TCCAtWCDCouZ29vZ2xlLmNvbYINKi5hbmRyb2lkLmNvbYIWKi5hcHBlbmdpbmUuZ29vZ2xlLmNvbYISKi5jbG91ZC5nb29nbGUuY29tghYqLmdvb2dsZS1hbmFseXRpY3MuY29tggsqLmdvb2dsZS5jYYILKi5nb29nbGUuY2yCDiouZ29vZ2xlLmNvLmlugg4qLmdvb2dsZS5jby5qcIIOKi5nb29nbGUuY28udWuCDyouZ29vZ2xlLmNvbS5hcoIPKi5nb29nbGUuY29tLmF1gg8qLmdvb2dsZS5jb20uYnKCDyouZ29vZ2xlLmNvbS5jb4IPKi5nb29nbGUuY29tLm14gg8qLmdvb2dsZS5jb20udHKCDyouZ29vZ2xlLmNvbS52boILKi5nb29nbGUuZGWCCyouZ29vZ2xlLmVzggsqLmdvb2dsZS5mcoILKi5nb29nbGUuaHWCCyouZ29vZ2xlLml0ggsqLmdvb2dsZS5ubIILKi5nb29nbGUucGyCCyouZ29vZ2xlLnB0gg8qLmdvb2dsZWFwaXMuY26CFCouZ29vZ2xlY29tbWVyY2UuY29tghEqLmdvb2dsZXZpZGVvLmNvbYINKi5nc3RhdGljLmNvbYIKKi5ndnQxLmNvbYIMKi51cmNoaW4uY29tghAqLnVybC5nb29nbGUuY29tghYqLnlvdXR1YmUtbm9jb29raWUuY29tgg0qLnlvdXR1YmUuY29tghYqLnlvdXR1YmVlZHVjYXRpb24uY29tggsqLnl0aW1nLmNvbYILYW5kcm9pZC5jb22CBGcuY2+CBmdvby5nbIIUZ29vZ2xlLWFuYWx5dGljcy5jb22CCmdvb2dsZS5jb22CEmdvb2dsZWNvbW1lcmNlLmNvbYIKdXJjaGluLmNvbYIIeW91dHUuYmWCC3lvdXR1YmUuY29tghR5b3V0dWJlZWR1Y2F0aW9uLmNvbTBoBggrBgEFBQcBAQRcMFowKwYIKwYBBQUHMAKGH2h0dHA6Ly9wa2kuZ29vZ2xlLmNvbS9HSUFHMi5jcnQwKwYIKwYBBQUHMAGGH2h0dHA6Ly9jbGllbnRzMS5nb29nbGUuY29tL29jc3AwHQYDVR0OBBYEFMSXIDaEplRtz9GCgkXGI4hW+6XYMAwGA1UdEwEB/wQCMAAwHwYDVR0jBBgwFoAUSt0GFhu89mi1dvWBtrtiGrpagS8wFwYDVR0gBBAwDjAMBgorBgEEAdZ5AgUBMDAGA1UdHwQpMCcwJaAjoCGGH2h0dHA6Ly9wa2kuZ29vZ2xlLmNvbS9HSUFHMi5jcmwwDQYJKoZIhvcNAQEFBQADggEBAFFTcdvX6AVVhSbNO8pinIArK65sVRHsq6z0UGNvCKuHGL+HajVpzAlCctL4D0xxf/j9O48k504ALPxSxOfthEA04UErkOwuQB/E3Vax9yrniAcVoMDI/393EJCtcXysNca0x/DxbsEukr1ICNmhAmaJa2xS/jhRlc3DeyOUQMqSTFYqvM36ynaper5gDL/V4dDK5hb/jcpzMKFmLJIazcTxz+lIEGtsDRGp6m5Y91M1kQTo4J/nbcIjOTZa4z/M6MT3/NGr9IlKmudqhB51mJnuYTliwRR2hADzRmSfRGsRzBYWUI5N7QlBMxwXex0T89Hdm2lprSaWgSlJTMNiliA=
+XXXXXXXXXX.XXXXXX	a047a37fa2d2e118a4f5095fe074d6cfe0e352425a7632bf8659c03919a6c81d	3	023A69	CN=Google Internet Authority G2,O=Google Inc,C=US	CN=GeoTrust Global CA,O=GeoTrust Inc.,C=US	XXXXXXXXXX.XXXXXX	XXXXXXXXXX.XXXXXX	rsaEncryption	sha1WithRSAEncryption	rsa	2048	65537	-	-	-	-	-	T	0	F	F	MIIEBDCCAuygAwIBAgIDAjppMA0GCSqGSIb3DQEBBQUAMEIxCzAJBgNVBAYTAlVTMRYwFAYDVQQKEw1HZW9UcnVzdCBJbmMuMRswGQYDVQQDExJHZW9UcnVzdCBHbG9iYWwgQ0EwHhcNMTMwNDA1MTUxNTU1WhcNMTUwNDA0MTUxNTU1WjBJMQswCQYDVQQGEwJVUzETMBEGA1UEChMKR29vZ2xlIEluYzElMCMGA1UEAxMcR29vZ2xlIEludGVybmV0IEF1dGhvcml0eSBHMjCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEBAJwqBHdc2FCROgajguDYUEi8iT/xGXAaiEZ+4I/F8YnOIe5a/mENtzJEiaB0C1NPVaTOgmKV7utZX8bhBYASxF6UP7xbSDj0U/ck5vuR6RXEz/RTDfRK/J9U3n2+oGtvh8DQUB8oMANA2ghzUWx//zo8pzcGjr1LEQTrfSTe5vn8MXH7lNVg8y5Kr0LSy+rEahqyzFPdFUuLH8gZYR/Nnag+YyuENWllhMgZxUYi+FOVvuOAShDGKuy6lyARxzmZEASg8GF6lSWMTlJ14rbtCMoU/M4iarNOz0YDl5cDfsCx3nuvRTPPuj5xt970JSXCDTWJnZ37DhF5iR43xa+OcmkCAwEAAaOB+zCB+DAfBgNVHSMEGDAWgBTAephojYn7qwVkDBF9qn1luMrMTjAdBgNVHQ4EFgQUSt0GFhu89mi1dvWBtrtiGrpagS8wEgYDVR0TAQH/BAgwBgEB/wIBADAOBgNVHQ8BAf8EBAMCAQYwOgYDVR0fBDMwMTAvoC2gK4YpaHR0cDovL2NybC5nZW90cnVzdC5jb20vY3Jscy9ndGdsb2JhbC5jcmwwPQYIKwYBBQUHAQEEMTAvMC0GCCsGAQUFBzABhiFodHRwOi8vZ3RnbG9iYWwtb2NzcC5nZW90cnVzdC5jb20wFwYDVR0gBBAwDjAMBgorBgEEAdZ5AgUBMA0GCSqGSIb3DQEBBQUAA4IBAQA21waAESetKhSbOHezI6B1WLuxfoNCunLaHtiONgaX4PCVOzf9G0JY/iLIa704XtE7JW4S615ndkZAkNoUyHgN7ZVm2o6Gb4ChulYylYbc3GrKBIxbf/a/zG+FA1jDaFETzf3I93k9mTXwVqO94FntT0QJo544evZG0R0SnU++0ED8Vf4GXjzaHFa9llF7b1cq26KqltyMdMKVvvBulRP/F/A8rLIQjcxz++iPAsbw+zOzlTvjwstoWHPbqCRiOwY1nQ2pM714A5AuTHhdUDqB1O6gyHA43LL5Z/qHQF1hwFGPa4NrzQU6yuGnBXj8ytqU0CwIPX4WecigUCAkVDNx
+XXXXXXXXXX.XXXXXX	3c35cc963eb004451323d3275d05b353235053490d9cd83729a2faf5e7ca1cc0	3	12BBE6	CN=GeoTrust Global CA,O=GeoTrust Inc.,C=US	OU=Equifax Secure Certificate Authority,O=Equifax,C=US	XXXXXXXXXX.XXXXXX	XXXXXXXXXX.XXXXXX	rsaEncryption	sha1WithRSAEncryption	rsa	2048	65537	-	-	-	-	-	T	-	F	F	MIIDfTCCAuagAwIBAgIDErvmMA0GCSqGSIb3DQEBBQUAME4xCzAJBgNVBAYTAlVTMRAwDgYDVQQKEwdFcXVpZmF4MS0wKwYDVQQLEyRFcXVpZmF4IFNlY3VyZSBDZXJ0aWZpY2F0ZSBBdXRob3JpdHkwHhcNMDIwNTIxMDQwMDAwWhcNMTgwODIxMDQwMDAwWjBCMQswCQYDVQQGEwJVUzEWMBQGA1UEChMNR2VvVHJ1c3QgSW5jLjEbMBkGA1UEAxMSR2VvVHJ1c3QgR2xvYmFsIENBMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEA2swYYzD99BcjGlZ+W988bDjkcbd4kdS8odhM+KhDtgPpTSEHCIjaWC9mOSm9BXiLnTjoBbdqfnGk5sRgprDvgOSJKA+eJdbtg/OtppHHmMlCGDUUna2YRpIuT8rxh0PBFpVXLVDviS2Aelet8u5fa9IAjbkU+BQVNdnARqN7csiRv8lVK83Qlz6cJmTM386DGXHKTubU1XupGc1V3sjs0l44U+VcT4wt/lAjNvxm5suOpDkZALeVAjmRCw7+OC7RHQWa9k0+bw8HHa8sHo9gOeL6NlMTOdReJivbPagUvTLrGAMoUgRx5aszPeE4uwc2hGKceeoWMPRfwCvocWvk+QIDAQABo4HwMIHtMB8GA1UdIwQYMBaAFEjmaPkr0rKV10fYIyAQTzOYkJ/UMB0GA1UdDgQWBBTAephojYn7qwVkDBF9qn1luMrMTjAPBgNVHRMBAf8EBTADAQH/MA4GA1UdDwEB/wQEAwIBBjA6BgNVHR8EMzAxMC+gLaArhilodHRwOi8vY3JsLmdlb3RydXN0LmNvbS9jcmxzL3NlY3VyZWNhLmNybDBOBgNVHSAERzBFMEMGBFUdIAAwOzA5BggrBgEFBQcCARYtaHR0cHM6Ly93d3cuZ2VvdHJ1c3QuY29tL3Jlc291cmNlcy9yZXBvc2l0b3J5MA0GCSqGSIb3DQEBBQUAA4GBAHbhEm5OSxYShjAGsoEIz/AIx8dxfmbuwu3UOx//8PDITtZDOLC5MH0Y0FWDomrLNhGc6Ehmo21/uBPUR/6LWlxz/K7ZGzIZOKuXNBSqltLroxwUCEm2u+WR74M26x1Wb8ravHNjkOR/ez4iyz0H7V84dJzjA1BOoa+Y7mHyhD8S
+#close XXXX-XX-XX-XX-XX-XX

testing/btest/scripts/policy/protocols/ssl/log-certs-base64.zeek

@@ -0,0 +1,5 @@
+# @TEST-EXEC: zeek -b -r $TRACES/tls/ecdhe.pcap %INPUT
+# @TEST-EXEC: btest-diff x509.log
+
+@load protocols/ssl/log-certs-base64
+