Merge remote-tracking branch 'origin/topic/johanna/cert-validation'

* origin/topic/johanna/cert-validation:
  and still use the hash for notice suppression.
  add knob to revert to old validation behavior
  Update certificate validation script - new version will cache valid intermediate chains that it encounters on the wire and use those to try to validate chains that might be missing intermediate certificates.

BIT-1332 #merged
This commit is contained in:
Robin Sommer 2015-03-17 09:07:51 -07:00
commit 0cfe431f15
11 changed files with 234 additions and 32 deletions

11
CHANGES
View file

@ -1,4 +1,11 @@
2.3-547 | 2015-03-17 09:07:51 -0700
* Update certificate validation script to cache valid intermediate
chains that it encounters on the wire and use those to try to
validate chains that might be missing intermediate certificates.
(Johanna Amann)
2.3-541 | 2015-03-13 15:44:08 -0500
* Make INSTALL a symlink to doc/install/install.rst (Jon siwek)
@ -352,7 +359,7 @@
2.3-328 | 2014-12-02 08:13:10 -0500
* Update windows-version-detection.bro to add support for
* Update windows-version-detection.bro to add support for
Windows 10. (Michal Purzynski)
2.3-326 | 2014-12-01 12:10:27 -0600
@ -422,7 +429,7 @@
2.3-280 | 2014-11-05 09:46:33 -0500
* Add Windows detection based on CryptoAPI HTTP traffic as a
* Add Windows detection based on CryptoAPI HTTP traffic as a
software framework policy script. (Vlad Grigorescu)
2.3-278 | 2014-11-03 18:55:18 -0800

View file

@ -1 +1 @@
2.3-541
2.3-547

View file

@ -1,4 +1,7 @@
##! Perform full certificate chain validation for SSL certificates.
#
# Also caches all intermediate certificates encountered so far and use them
# for future validations.
@load base/frameworks/notice
@load base/protocols/ssl
@ -19,12 +22,107 @@ export {
};
## MD5 hash values for recently validated chains along with the
## validation status message are kept in this table to avoid constant
## validation status are kept in this table to avoid constant
## validation every time the same certificate chain is seen.
global recently_validated_certs: table[string] of string = table()
&read_expire=5mins &synchronized &redef;
&read_expire=5mins &redef;
## Use intermediate CA certificate caching when trying to validate
## certificates. When this is enabled, Bro keeps track of all valid
## intermediate CA certificates that it has seen in the past. When
## encountering a host certificate that cannot be validated because
## of missing intermediate CA certificate, the cached list is used
## to try to validate the cert. This is similar to how Firefox is
## doing certificate validation.
##
## Disabling this will usually greatly increase the number of validation warnings
## that you encounter. Only disable if you want to find misconfigured servers.
global ssl_cache_intermediate_ca: bool = T &redef;
## Event from a worker to the manager that it has encountered a new
## valid intermediate.
global intermediate_add: event(key: string, value: vector of opaque of x509);
## Event from the manager to the workers that a new intermediate chain
## is to be added.
global new_intermediate: event(key: string, value: vector of opaque of x509);
}
global intermediate_cache: table[string] of vector of opaque of x509;
@if ( Cluster::is_enabled() )
@load base/frameworks/cluster
redef Cluster::manager2worker_events += /SSL::intermediate_add/;
redef Cluster::worker2manager_events += /SSL::new_intermediate/;
@endif
function add_to_cache(key: string, value: vector of opaque of x509)
{
intermediate_cache[key] = value;
@if ( Cluster::is_enabled() )
event SSL::new_intermediate(key, value);
@endif
}
@if ( Cluster::is_enabled() && Cluster::local_node_type() != Cluster::MANAGER )
event SSL::intermediate_add(key: string, value: vector of opaque of x509)
{
intermediate_cache[key] = value;
}
@endif
@if ( Cluster::is_enabled() && Cluster::local_node_type() == Cluster::MANAGER )
event SSL::new_intermediate(key: string, value: vector of opaque of x509)
{
if ( key in intermediate_cache )
return;
intermediate_cache[key] = value;
event SSL::intermediate_add(key, value);
}
@endif
function cache_validate(chain: vector of opaque of x509): string
{
local chain_hash: vector of string = vector();
for ( i in chain )
chain_hash[i] = sha1_hash(x509_get_certificate_string(chain[i]));
local chain_id = join_string_vec(chain_hash, ".");
# If we tried this certificate recently, just return the cached result.
if ( chain_id in recently_validated_certs )
return recently_validated_certs[chain_id];
local result = x509_verify(chain, root_certs);
recently_validated_certs[chain_id] = result$result_string;
# if we have a working chain where we did not store the intermediate certs
# in our cache yet - do so
if ( ssl_cache_intermediate_ca &&
result$result_string == "ok" &&
result?$chain_certs &&
|result$chain_certs| > 2 )
{
local result_chain = result$chain_certs;
local icert = x509_parse(result_chain[1]);
if ( icert$subject !in intermediate_cache )
{
local cachechain: vector of opaque of x509;
for ( i in result_chain )
{
if ( i >=1 && i<=|result_chain|-2 )
cachechain[i-1] = result_chain[i];
}
add_to_cache(icert$subject, cachechain);
}
}
return result$result_string;
}
event ssl_established(c: connection) &priority=3
{
# If there aren't any certs we can't very well do certificate validation.
@ -32,9 +130,31 @@ event ssl_established(c: connection) &priority=3
! c$ssl$cert_chain[0]?$x509 )
return;
local chain_id = join_string_vec(c$ssl$cert_chain_fuids, ".");
local intermediate_chain: vector of opaque of x509 = vector();
local issuer = c$ssl$cert_chain[0]$x509$certificate$issuer;
local hash = c$ssl$cert_chain[0]$sha1;
local result: string;
# Look if we already have a working chain for the issuer of this cert.
# If yes, try this chain first instead of using the chain supplied from
# the server.
if ( ssl_cache_intermediate_ca && issuer in intermediate_cache )
{
intermediate_chain[0] = c$ssl$cert_chain[0]$x509$handle;
for ( i in intermediate_cache[issuer] )
intermediate_chain[i+1] = intermediate_cache[issuer][i];
result = cache_validate(intermediate_chain);
if ( result == "ok" )
{
c$ssl$validation_status = result;
return;
}
}
# Validation with known chains failed or there was no fitting intermediate
# in our store.
# Fall back to validating the certificate with the server-supplied chain.
local chain: vector of opaque of x509 = vector();
for ( i in c$ssl$cert_chain )
{
@ -42,18 +162,10 @@ event ssl_established(c: connection) &priority=3
chain[i] = c$ssl$cert_chain[i]$x509$handle;
}
if ( chain_id in recently_validated_certs )
{
c$ssl$validation_status = recently_validated_certs[chain_id];
}
else
{
local result = x509_verify(chain, root_certs);
c$ssl$validation_status = result$result_string;
recently_validated_certs[chain_id] = result$result_string;
}
result = cache_validate(chain);
c$ssl$validation_status = result;
if ( c$ssl$validation_status != "ok" )
if ( result != "ok" )
{
local message = fmt("SSL certificate validation failed with (%s)", c$ssl$validation_status);
NOTICE([$note=Invalid_Server_Cert, $msg=message,
@ -61,5 +173,3 @@ event ssl_established(c: connection) &priority=3
$identifier=cat(c$id$resp_h,c$id$resp_p,hash,c$ssl$validation_status)]);
}
}

View file

@ -0,0 +1,15 @@
#separator \x09
#set_separator ,
#empty_field (empty)
#unset_field -
#path ssl
#open 2015-03-09-19-32-44
#fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p version cipher curve server_name resumed last_alert next_protocol established cert_chain_fuids client_cert_chain_fuids subject issuer client_subject client_issuer validation_status
#types time string addr port addr port string string string string bool string string bool vector[string] vector[string] string string string string string
1425929564.247511 CXWv6p3arKYeMETxOg 192.168.4.149 58529 128.32.169.140 443 TLSv10 TLS_RSA_WITH_RC4_128_MD5 - - F - - T FTzCuuqU5y7w85H89 (empty) CN=www.cviis.org,OU=Domain Control Validated CN=Starfield Secure Certificate Authority - G2,OU=http://certs.starfieldtech.com/repository/,O=Starfield Technologies\, Inc.,L=Scottsdale,ST=Arizona,C=US - - unable to get local issuer certificate
1425929565.270104 CXWv6p3arKYeMETxOg 192.168.4.149 58529 128.32.169.140 443 TLSv10 TLS_RSA_WITH_RC4_128_MD5 - - F - - T FXzQOu1ZSKSF7H8Ez6 (empty) CN=www.cviis.org,OU=Domain Control Validated CN=Starfield Secure Certificate Authority - G2,OU=http://certs.starfieldtech.com/repository/,O=Starfield Technologies\, Inc.,L=Scottsdale,ST=Arizona,C=US - - unable to get local issuer certificate
1425929566.843026 CjhGID4nQcgTWjvg4c 192.168.4.149 58530 72.167.102.91 443 TLSv12 TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384 secp256r1 - F - - T F5l2dVkZHiwiOWR67,Fkw2ETDXfIXIvatba,Fbgf8A3V6m8v33wTcj (empty) CN=valid.sfig2.catest.starfieldtech.com,O=Starfield Technologies\, LLC,L=Scottsdale,ST=Arizona,C=US,serialNumber=R-1724741-6,businessCategory=Private Organization,jurisdictionST=Arizona,jurisdictionC=US CN=Starfield Secure Certificate Authority - G2,OU=http://certs.starfieldtech.com/repository/,O=Starfield Technologies\, Inc.,L=Scottsdale,ST=Arizona,C=US - - ok
1425929571.372511 CCvvfg3TEfuqmmG4bh 192.168.4.149 58532 128.32.169.140 443 TLSv10 TLS_RSA_WITH_RC4_128_MD5 - - F - - T FhEtvg4pQ90832J56f (empty) CN=www.cviis.org,OU=Domain Control Validated CN=Starfield Secure Certificate Authority - G2,OU=http://certs.starfieldtech.com/repository/,O=Starfield Technologies\, Inc.,L=Scottsdale,ST=Arizona,C=US - - ok
1425929567.865619 CjhGID4nQcgTWjvg4c 192.168.4.149 58530 72.167.102.91 443 TLSv12 TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384 secp256r1 - F - - T Fyc6cQ2rMCAhpIGcM5,FoJ8j735m9ogDYopYj,FHaYhA3ykzVlKPnnsc (empty) CN=valid.sfig2.catest.starfieldtech.com,O=Starfield Technologies\, LLC,L=Scottsdale,ST=Arizona,C=US,serialNumber=R-1724741-6,businessCategory=Private Organization,jurisdictionST=Arizona,jurisdictionC=US CN=Starfield Secure Certificate Authority - G2,OU=http://certs.starfieldtech.com/repository/,O=Starfield Technologies\, Inc.,L=Scottsdale,ST=Arizona,C=US - - ok
1425929572.395104 CCvvfg3TEfuqmmG4bh 192.168.4.149 58532 128.32.169.140 443 TLSv10 TLS_RSA_WITH_RC4_128_MD5 - - F - - T FwZZ8034tgyXSponwg (empty) CN=www.cviis.org,OU=Domain Control Validated CN=Starfield Secure Certificate Authority - G2,OU=http://certs.starfieldtech.com/repository/,O=Starfield Technologies\, Inc.,L=Scottsdale,ST=Arizona,C=US - - ok
#close 2015-03-09-19-32-53

View file

@ -0,0 +1,12 @@
#separator \x09
#set_separator ,
#empty_field (empty)
#unset_field -
#path ssl
#open 2015-03-09-19-51-25
#fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p version cipher curve server_name resumed last_alert next_protocol established cert_chain_fuids client_cert_chain_fuids subject issuer client_subject client_issuer validation_status
#types time string addr port addr port string string string string bool string string bool vector[string] vector[string] string string string string string
1417039703.224578 CXWv6p3arKYeMETxOg 192.168.4.149 58529 128.32.169.140 443 TLSv10 TLS_RSA_WITH_RC4_128_MD5 - - F - - T FghNi02cFL9n6ttuMa (empty) CN=www.cviis.org,OU=Domain Control Validated CN=Starfield Secure Certificate Authority - G2,OU=http://certs.starfieldtech.com/repository/,O=Starfield Technologies\, Inc.,L=Scottsdale,ST=Arizona,C=US - - unable to get local issuer certificate
1417039705.820093 CjhGID4nQcgTWjvg4c 192.168.4.149 58530 72.167.102.91 443 TLSv12 TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384 secp256r1 - F - - T Fz7gr4fSm2T2sEyDl,FhjNBG25vvoBO6CS79,FQFHJA20WL56NP6LXk (empty) CN=valid.sfig2.catest.starfieldtech.com,O=Starfield Technologies\, LLC,L=Scottsdale,ST=Arizona,C=US,serialNumber=R-1724741-6,businessCategory=Private Organization,jurisdictionST=Arizona,jurisdictionC=US CN=Starfield Secure Certificate Authority - G2,OU=http://certs.starfieldtech.com/repository/,O=Starfield Technologies\, Inc.,L=Scottsdale,ST=Arizona,C=US - - ok
1417039710.349578 CCvvfg3TEfuqmmG4bh 192.168.4.149 58532 128.32.169.140 443 TLSv10 TLS_RSA_WITH_RC4_128_MD5 - - F - - T FRcFYq3e3hgYkZ8dS1 (empty) CN=www.cviis.org,OU=Domain Control Validated CN=Starfield Secure Certificate Authority - G2,OU=http://certs.starfieldtech.com/repository/,O=Starfield Technologies\, Inc.,L=Scottsdale,ST=Arizona,C=US - - unable to get local issuer certificate
#close 2015-03-09-19-51-25

View file

@ -0,0 +1,23 @@
#separator \x09
#set_separator ,
#empty_field (empty)
#unset_field -
#path ssl
#open 2015-03-09-19-44-42
#fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p version cipher curve server_name resumed last_alert next_protocol established cert_chain_fuids client_cert_chain_fuids subject issuer client_subject client_issuer validation_status
#types time string addr port addr port string string string string bool string string bool vector[string] vector[string] string string string string string
1394745602.951961 CXWv6p3arKYeMETxOg 192.168.4.149 60539 87.98.220.10 443 TLSv10 TLS_DHE_RSA_WITH_AES_256_CBC_SHA - - F - - T F1fX1R2cDOzbvg17ye,FqPEQR2eytAQybroyl (empty) CN=www.spidh.org,OU=COMODO SSL,OU=Domain Control Validated CN=COMODO SSL CA,O=COMODO CA Limited,L=Salford,ST=Greater Manchester,C=GB - - certificate has expired
1394745618.791420 CjhGID4nQcgTWjvg4c 192.168.4.149 60540 122.1.240.204 443 TLSv10 TLS_RSA_WITH_AES_256_CBC_SHA - - F - - T F6NAbK127LhNBaEe5c,FDhmPt28vyXlGMTxP7,F0ROCKibhE1KntJ1h (empty) CN=www.tobu-estate.com,OU=Terms of use at www.verisign.com/rpa (c)05,O=TOBU RAILWAY Co.\,Ltd.,L=Sumida-ku,ST=Tokyo,C=JP CN=VeriSign Class 3 Secure Server CA - G3,OU=Terms of use at https://www.verisign.com/rpa (c)10,OU=VeriSign Trust Network,O=VeriSign\, Inc.,C=US - - ok
#close 2015-03-09-19-44-42
#separator \x09
#set_separator ,
#empty_field (empty)
#unset_field -
#path ssl
#open 2015-03-09-19-44-42
#fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p version cipher curve server_name resumed last_alert next_protocol established cert_chain_fuids client_cert_chain_fuids subject issuer client_subject client_issuer validation_status
#types time string addr port addr port string string string string bool string string bool vector[string] vector[string] string string string string string
1417039703.224578 CXWv6p3arKYeMETxOg 192.168.4.149 58529 128.32.169.140 443 TLSv10 TLS_RSA_WITH_RC4_128_MD5 - - F - - T FghNi02cFL9n6ttuMa (empty) CN=www.cviis.org,OU=Domain Control Validated CN=Starfield Secure Certificate Authority - G2,OU=http://certs.starfieldtech.com/repository/,O=Starfield Technologies\, Inc.,L=Scottsdale,ST=Arizona,C=US - - unable to get local issuer certificate
1417039705.820093 CjhGID4nQcgTWjvg4c 192.168.4.149 58530 72.167.102.91 443 TLSv12 TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384 secp256r1 - F - - T Fz7gr4fSm2T2sEyDl,FhjNBG25vvoBO6CS79,FQFHJA20WL56NP6LXk (empty) CN=valid.sfig2.catest.starfieldtech.com,O=Starfield Technologies\, LLC,L=Scottsdale,ST=Arizona,C=US,serialNumber=R-1724741-6,businessCategory=Private Organization,jurisdictionST=Arizona,jurisdictionC=US CN=Starfield Secure Certificate Authority - G2,OU=http://certs.starfieldtech.com/repository/,O=Starfield Technologies\, Inc.,L=Scottsdale,ST=Arizona,C=US - - ok
1417039710.349578 CCvvfg3TEfuqmmG4bh 192.168.4.149 58532 128.32.169.140 443 TLSv10 TLS_RSA_WITH_RC4_128_MD5 - - F - - T FRcFYq3e3hgYkZ8dS1 (empty) CN=www.cviis.org,OU=Domain Control Validated CN=Starfield Secure Certificate Authority - G2,OU=http://certs.starfieldtech.com/repository/,O=Starfield Technologies\, Inc.,L=Scottsdale,ST=Arizona,C=US - - ok
#close 2015-03-09-19-44-42

View file

@ -1,11 +0,0 @@
#separator \x09
#set_separator ,
#empty_field (empty)
#unset_field -
#path ssl
#open 2014-08-08-17-13-58
#fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p version cipher curve server_name resumed last_alert next_protocol established cert_chain_fuids client_cert_chain_fuids subject issuer client_subject client_issuer validation_status
#types time string addr port addr port string string string string bool string string bool vector[string] vector[string] string string string string string
1394745602.951961 CXWv6p3arKYeMETxOg 192.168.4.149 60539 87.98.220.10 443 TLSv10 TLS_DHE_RSA_WITH_AES_256_CBC_SHA - - F - - T F1fX1R2cDOzbvg17ye,FqPEQR2eytAQybroyl (empty) CN=www.spidh.org,OU=COMODO SSL,OU=Domain Control Validated CN=COMODO SSL CA,O=COMODO CA Limited,L=Salford,ST=Greater Manchester,C=GB - - certificate has expired
1394745618.791420 CjhGID4nQcgTWjvg4c 192.168.4.149 60540 122.1.240.204 443 TLSv10 TLS_RSA_WITH_AES_256_CBC_SHA - - F - - T F6NAbK127LhNBaEe5c,FDhmPt28vyXlGMTxP7,F0ROCKibhE1KntJ1h (empty) CN=www.tobu-estate.com,OU=Terms of use at www.verisign.com/rpa (c)05,O=TOBU RAILWAY Co.\,Ltd.,L=Sumida-ku,ST=Tokyo,C=JP CN=VeriSign Class 3 Secure Server CA - G3,OU=Terms of use at https://www.verisign.com/rpa (c)10,OU=VeriSign Trust Network,O=VeriSign\, Inc.,C=US - - ok
#close 2014-08-08-17-13-58

Binary file not shown.

View file

@ -0,0 +1,37 @@
# @TEST-SERIALIZE: comm
#
# @TEST-EXEC: btest-bg-run manager-1 "cp ../cluster-layout.bro . && CLUSTER_NODE=manager-1 bro %INPUT"
# @TEST-EXEC: sleep 1
# @TEST-EXEC: btest-bg-run proxy-1 "cp ../cluster-layout.bro . && CLUSTER_NODE=proxy-1 bro %INPUT"
# @TEST-EXEC: btest-bg-run proxy-2 "cp ../cluster-layout.bro . && CLUSTER_NODE=proxy-2 bro %INPUT"
# @TEST-EXEC: sleep 1
# @TEST-EXEC: btest-bg-run worker-1 "cp ../cluster-layout.bro . && CLUSTER_NODE=worker-1 bro --pseudo-realtime -C -r $TRACES/tls/missing-intermediate.pcap %INPUT"
# @TEST-EXEC: btest-bg-run worker-2 "cp ../cluster-layout.bro . && CLUSTER_NODE=worker-2 bro --pseudo-realtime -C -r $TRACES/tls/missing-intermediate.pcap %INPUT"
# @TEST-EXEC: btest-bg-wait 10
# @TEST-EXEC: cat manager-1/ssl*.log > ssl.log
# @TEST-EXEC: TEST_DIFF_CANONIFIER=$SCRIPTS/diff-remove-file-ids btest-diff ssl.log
#
redef Log::default_rotation_interval = 0secs;
@TEST-START-FILE cluster-layout.bro
redef Cluster::nodes = {
["manager-1"] = [$node_type=Cluster::MANAGER, $ip=127.0.0.1, $p=37757/tcp, $workers=set("worker-1", "worker-2")],
["proxy-1"] = [$node_type=Cluster::PROXY, $ip=127.0.0.1, $p=37758/tcp, $manager="manager-1", $workers=set("worker-1", "worker-2")],
["proxy-2"] = [$node_type=Cluster::PROXY, $ip=127.0.0.1, $p=37759/tcp, $manager="manager-1", $workers=set("worker-2")],
["worker-1"] = [$node_type=Cluster::WORKER, $ip=127.0.0.1, $p=37760/tcp, $manager="manager-1", $proxy="proxy-1", $interface="eth0"],
["worker-2"] = [$node_type=Cluster::WORKER, $ip=127.0.0.1, $p=37761/tcp, $manager="manager-1", $proxy="proxy-1", $interface="eth1"],
};
@TEST-END-FILE
event terminate_me() {
terminate();
}
event remote_connection_closed(p: event_peer) {
schedule 1sec { terminate_me() };
}
@load base/frameworks/cluster
@load protocols/ssl/validate-certs.bro

View file

@ -0,0 +1,6 @@
# @TEST-EXEC: bro -C -r $TRACES/tls/missing-intermediate.pcap %INPUT
# @TEST-EXEC: btest-diff ssl.log
@load protocols/ssl/validate-certs.bro
redef SSL::ssl_cache_intermediate_ca = F;

View file

@ -1,4 +1,7 @@
# @TEST-EXEC: bro -r $TRACES/tls/tls-expired-cert.trace %INPUT
# @TEST-EXEC: btest-diff ssl.log
# @TEST-EXEC: cat ssl.log > ssl-all.log
# @TEST-EXEC: bro -C -r $TRACES/tls/missing-intermediate.pcap %INPUT
# @TEST-EXEC: cat ssl.log >> ssl-all.log
# @TEST-EXEC: btest-diff ssl-all.log
@load protocols/ssl/validate-certs
@load protocols/ssl/validate-certs.bro