diff --git a/CHANGES b/CHANGES index 5ba0f2b702..111c557b49 100644 --- a/CHANGES +++ b/CHANGES @@ -1,4 +1,17 @@ +2.3-518 | 2015-03-04 13:13:50 -0800 + + * Add bytes_recvd to stats.log recording the number of bytes + received, according to packet headers. (Mike Smiley) + +2.3-516 | 2015-03-04 12:30:06 -0800 + + * Extract most specific Common Name from SSL certificates (Johanna + Amann) + + * Send CN and SAN fields of SSL certificates to the Intel framework. + (Johanna Amann) + 2.3-511 | 2015-03-02 18:07:17 -0800 * Changes to plugin meta hooks for function calls. (Gilbert Clark) diff --git a/NEWS b/NEWS index be899e44d3..ec94bd10fe 100644 --- a/NEWS +++ b/NEWS @@ -61,6 +61,8 @@ New Functionality - The SSL analysis now alert when encountering SSL connections with old protocol versions or unsafe cipher suites. +- [TODO] Add new BroControl features. + Changed Functionality --------------------- @@ -89,6 +91,8 @@ Changed Functionality - conn.log gained a new field local_resp that works like local_orig, just for the responder address of the connection. +- [TODO] Add changed BroControl features. + Deprecated Functionality ------------------------ diff --git a/VERSION b/VERSION index 9c9bf55b86..5a9ae5afca 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -2.3-511 +2.3-518 diff --git a/aux/broctl b/aux/broctl index 95b48726af..f7b9ef4d24 160000 --- a/aux/broctl +++ b/aux/broctl @@ -1 +1 @@ -Subproject commit 95b48726af25dad03e0c89029c3a4aaa739e4fed +Subproject commit f7b9ef4d245114180c5932f362310632b8cf5174 diff --git a/scripts/base/init-bare.bro b/scripts/base/init-bare.bro index 2eff946a24..bf6366a62d 100644 --- a/scripts/base/init-bare.bro +++ b/scripts/base/init-bare.bro @@ -440,6 +440,7 @@ type NetStats: record { ## packet capture system, this value may not be available and will then ## be always set to zero. pkts_link: count &default=0; + bytes_recvd: count &default=0; ##< Bytes received by Bro. }; ## Statistics about Bro's resource consumption. @@ -2774,19 +2775,20 @@ export { module X509; export { type Certificate: record { - version: count; ##< Version number. - serial: string; ##< Serial number. - subject: string; ##< Subject. - issuer: string; ##< Issuer. - not_valid_before: time; ##< Timestamp before when certificate is not valid. - not_valid_after: time; ##< Timestamp after when certificate is not valid. - key_alg: string; ##< Name of the key algorithm - sig_alg: string; ##< Name of the signature algorithm - key_type: string &optional; ##< Key type, if key parseable by openssl (either rsa, dsa or ec) - key_length: count &optional; ##< Key length in bits - exponent: string &optional; ##< Exponent, if RSA-certificate - curve: string &optional; ##< Curve, if EC-certificate - } &log; + version: count &log; ##< Version number. + serial: string &log; ##< Serial number. + subject: string &log; ##< Subject. + issuer: string &log; ##< Issuer. + cn: string &optional; ##< Last (most specific) common name. + not_valid_before: time &log; ##< Timestamp before when certificate is not valid. + not_valid_after: time &log; ##< Timestamp after when certificate is not valid. + key_alg: string &log; ##< Name of the key algorithm + sig_alg: string &log; ##< Name of the signature algorithm + key_type: string &optional &log; ##< Key type, if key parseable by openssl (either rsa, dsa or ec) + key_length: count &optional &log; ##< Key length in bits + exponent: string &optional &log; ##< Exponent, if RSA-certificate + curve: string &optional &log; ##< Curve, if EC-certificate + }; type Extension: record { name: string; ##< Long name of extension. oid if name not known diff --git a/scripts/policy/frameworks/intel/seen/ssl.bro b/scripts/policy/frameworks/intel/seen/ssl.bro index 70c70f5b71..7bfbef4e9b 100644 --- a/scripts/policy/frameworks/intel/seen/ssl.bro +++ b/scripts/policy/frameworks/intel/seen/ssl.bro @@ -10,3 +10,16 @@ event ssl_extension_server_name(c: connection, is_orig: bool, names: string_vec) $conn=c, $where=SSL::IN_SERVER_NAME]); } + +event ssl_established(c: connection) + { + if ( ! c$ssl?$cert_chain || |c$ssl$cert_chain| == 0 || + ! c$ssl$cert_chain[0]?$x509 ) + return; + + if ( c$ssl$cert_chain[0]$x509?$certificate && c$ssl$cert_chain[0]$x509$certificate?$cn ) + Intel::seen([$indicator=c$ssl$cert_chain[0]$x509$certificate$cn, + $indicator_type=Intel::DOMAIN, + $conn=c, + $where=X509::IN_CERT]); + } diff --git a/scripts/policy/frameworks/intel/seen/x509.bro b/scripts/policy/frameworks/intel/seen/x509.bro index 11e4d57f90..3a2859b6d5 100644 --- a/scripts/policy/frameworks/intel/seen/x509.bro +++ b/scripts/policy/frameworks/intel/seen/x509.bro @@ -2,6 +2,18 @@ @load base/files/x509 @load ./where-locations +event x509_ext_subject_alternative_name(f: fa_file, ext: X509::SubjectAlternativeName) + { + if ( ext?$dns ) + { + for ( i in ext$dns ) + Intel::seen([$indicator=ext$dns[i], + $indicator_type=Intel::DOMAIN, + $f=f, + $where=X509::IN_CERT]); + } + } + event x509_certificate(f: fa_file, cert_ref: opaque of x509, cert: X509::Certificate) { if ( /emailAddress=/ in cert$subject ) diff --git a/scripts/policy/misc/stats.bro b/scripts/policy/misc/stats.bro index eb1ddb0202..a8a08bdcc1 100644 --- a/scripts/policy/misc/stats.bro +++ b/scripts/policy/misc/stats.bro @@ -39,6 +39,9 @@ export { ## Number of packets seen on the link since the last stats ## interval if reading live traffic. pkts_link: count &log &optional; + ## Number of bytes received since the last stats interval if + ## reading live traffic. + bytes_recv: count &log &optional; }; ## Event to catch stats as they are written to the logging stream. @@ -74,6 +77,7 @@ event check_stats(last_ts: time, last_ns: NetStats, last_res: bro_resources) info$pkts_recv = ns$pkts_recvd - last_ns$pkts_recvd; info$pkts_dropped = ns$pkts_dropped - last_ns$pkts_dropped; info$pkts_link = ns$pkts_link - last_ns$pkts_link; + info$bytes_recv = ns$bytes_recvd - last_ns$bytes_recvd; } Log::write(Stats::LOG, info); diff --git a/src/bro.bif b/src/bro.bif index 4e685eb84a..e7be72410c 100644 --- a/src/bro.bif +++ b/src/bro.bif @@ -1675,6 +1675,7 @@ function net_stats%(%): NetStats unsigned int recv = 0; unsigned int drop = 0; unsigned int link = 0; + unsigned int bytes_recv = 0; const iosource::Manager::PktSrcList& pkt_srcs(iosource_mgr->GetPktSrcs()); @@ -1688,12 +1689,14 @@ function net_stats%(%): NetStats recv += stat.received; drop += stat.dropped; link += stat.link; + bytes_recv += stat.bytes_received; } RecordVal* ns = new RecordVal(net_stats); ns->Assign(0, new Val(recv, TYPE_COUNT)); ns->Assign(1, new Val(drop, TYPE_COUNT)); ns->Assign(2, new Val(link, TYPE_COUNT)); + ns->Assign(3, new Val(bytes_recv, TYPE_COUNT)); return ns; %} diff --git a/src/file_analysis/analyzer/x509/X509.cc b/src/file_analysis/analyzer/x509/X509.cc index b294d448be..8c70597dca 100644 --- a/src/file_analysis/analyzer/x509/X509.cc +++ b/src/file_analysis/analyzer/x509/X509.cc @@ -104,13 +104,35 @@ RecordVal* file_analysis::X509::ParseCertificate(X509Val* cert_val) len = BIO_gets(bio, buf, sizeof(buf)); pX509Cert->Assign(2, new StringVal(len, buf)); BIO_reset(bio); + + X509_NAME *subject_name = X509_get_subject_name(ssl_cert); + // extract the most specific (last) common name from the subject + int namepos = -1; + for ( ;; ) + { + int j = X509_NAME_get_index_by_NID(subject_name, NID_commonName, namepos); + if ( j == -1 ) + break; + + namepos = j; + } + + if ( namepos != -1 ) + { + // we found a common name + ASN1_STRING_print(bio, X509_NAME_ENTRY_get_data(X509_NAME_get_entry(subject_name, namepos))); + len = BIO_gets(bio, buf, sizeof(buf)); + pX509Cert->Assign(4, new StringVal(len, buf)); + BIO_reset(bio); + } + X509_NAME_print_ex(bio, X509_get_issuer_name(ssl_cert), 0, XN_FLAG_RFC2253); len = BIO_gets(bio, buf, sizeof(buf)); pX509Cert->Assign(3, new StringVal(len, buf)); BIO_free(bio); - pX509Cert->Assign(4, new Val(GetTimeFromAsn1(X509_get_notBefore(ssl_cert)), TYPE_TIME)); - pX509Cert->Assign(5, new Val(GetTimeFromAsn1(X509_get_notAfter(ssl_cert)), TYPE_TIME)); + pX509Cert->Assign(5, new Val(GetTimeFromAsn1(X509_get_notBefore(ssl_cert)), TYPE_TIME)); + pX509Cert->Assign(6, new Val(GetTimeFromAsn1(X509_get_notAfter(ssl_cert)), TYPE_TIME)); // we only read 255 bytes because byte 256 is always 0. // if the string is longer than 255, that will be our null-termination, @@ -118,7 +140,7 @@ RecordVal* file_analysis::X509::ParseCertificate(X509Val* cert_val) if ( ! i2t_ASN1_OBJECT(buf, 255, ssl_cert->cert_info->key->algor->algorithm) ) buf[0] = 0; - pX509Cert->Assign(6, new StringVal(buf)); + pX509Cert->Assign(7, new StringVal(buf)); // Special case for RDP server certificates. For some reason some (all?) RDP server // certificates like to specify their key algorithm as md5WithRSAEncryption, which @@ -136,23 +158,23 @@ RecordVal* file_analysis::X509::ParseCertificate(X509Val* cert_val) if ( ! i2t_ASN1_OBJECT(buf, 255, ssl_cert->sig_alg->algorithm) ) buf[0] = 0; - pX509Cert->Assign(7, new StringVal(buf)); + pX509Cert->Assign(8, new StringVal(buf)); // Things we can do when we have the key... EVP_PKEY *pkey = X509_extract_key(ssl_cert); if ( pkey != NULL ) { if ( pkey->type == EVP_PKEY_DSA ) - pX509Cert->Assign(8, new StringVal("dsa")); + pX509Cert->Assign(9, new StringVal("dsa")); else if ( pkey->type == EVP_PKEY_RSA ) { - pX509Cert->Assign(8, new StringVal("rsa")); + pX509Cert->Assign(9, new StringVal("rsa")); char *exponent = BN_bn2dec(pkey->pkey.rsa->e); if ( exponent != NULL ) { - pX509Cert->Assign(10, new StringVal(exponent)); + pX509Cert->Assign(11, new StringVal(exponent)); OPENSSL_free(exponent); exponent = NULL; } @@ -160,8 +182,8 @@ RecordVal* file_analysis::X509::ParseCertificate(X509Val* cert_val) #ifndef OPENSSL_NO_EC else if ( pkey->type == EVP_PKEY_EC ) { - pX509Cert->Assign(8, new StringVal("ecdsa")); - pX509Cert->Assign(11, KeyCurve(pkey)); + pX509Cert->Assign(9, new StringVal("ecdsa")); + pX509Cert->Assign(12, KeyCurve(pkey)); } #endif @@ -172,7 +194,7 @@ RecordVal* file_analysis::X509::ParseCertificate(X509Val* cert_val) unsigned int length = KeyLength(pkey); if ( length > 0 ) - pX509Cert->Assign(9, new Val(length, TYPE_COUNT)); + pX509Cert->Assign(10, new Val(length, TYPE_COUNT)); EVP_PKEY_free(pkey); } diff --git a/src/iosource/PktSrc.h b/src/iosource/PktSrc.h index 378ac3f5ee..2400219fd0 100644 --- a/src/iosource/PktSrc.h +++ b/src/iosource/PktSrc.h @@ -38,7 +38,12 @@ public: */ unsigned int link; - Stats() { received = dropped = link = 0; } + /** + * Bytes received by source after filtering (w/o drops). + */ + uint64 bytes_received; + + Stats() { received = dropped = link = bytes_received = 0; } }; /** diff --git a/src/iosource/pcap/Source.cc b/src/iosource/pcap/Source.cc index a68f0ca322..7645903c2a 100644 --- a/src/iosource/pcap/Source.cc +++ b/src/iosource/pcap/Source.cc @@ -180,6 +180,8 @@ bool PcapSource::ExtractNextPacket(Packet* pkt) last_hdr = current_hdr; last_data = data; ++stats.received; + stats.bytes_received += current_hdr.len; + return true; } @@ -219,7 +221,7 @@ bool PcapSource::SetFilter(int index) #ifndef HAVE_LINUX // Linux doesn't clear counters when resetting filter. - stats.received = stats.dropped = stats.link = 0; + stats.received = stats.dropped = stats.link = stats.bytes_received = 0; #endif return true; @@ -230,7 +232,7 @@ void PcapSource::Statistics(Stats* s) char errbuf[PCAP_ERRBUF_SIZE]; if ( ! (props.is_live && pd) ) - s->received = s->dropped = s->link = 0; + s->received = s->dropped = s->link = s->bytes_received = 0; else { @@ -238,7 +240,7 @@ void PcapSource::Statistics(Stats* s) if ( pcap_stats(pd, &pstat) < 0 ) { PcapError(); - s->received = s->dropped = s->link = 0; + s->received = s->dropped = s->link = s->bytes_received = 0; } else @@ -249,6 +251,7 @@ void PcapSource::Statistics(Stats* s) } s->received = stats.received; + s->bytes_received = stats.bytes_received; if ( ! props.is_live ) s->dropped = 0; diff --git a/testing/btest/Baseline/bifs.net_stats_trace/output b/testing/btest/Baseline/bifs.net_stats_trace/output index 55d6693db5..ed924affa6 100644 --- a/testing/btest/Baseline/bifs.net_stats_trace/output +++ b/testing/btest/Baseline/bifs.net_stats_trace/output @@ -1 +1 @@ -[pkts_recvd=136, pkts_dropped=0, pkts_link=0] +[pkts_recvd=136, pkts_dropped=0, pkts_link=0, bytes_recvd=25260] diff --git a/testing/btest/Baseline/scripts.base.protocols.ssl.common_name/.stdout b/testing/btest/Baseline/scripts.base.protocols.ssl.common_name/.stdout new file mode 100644 index 0000000000..0642f10875 --- /dev/null +++ b/testing/btest/Baseline/scripts.base.protocols.ssl.common_name/.stdout @@ -0,0 +1,3 @@ +*.gstatic.com +Google Internet Authority +No CN diff --git a/testing/btest/Baseline/scripts.policy.frameworks.intel.seen.certs/intel-all.log b/testing/btest/Baseline/scripts.policy.frameworks.intel.seen.certs/intel-all.log new file mode 100644 index 0000000000..8782898d33 --- /dev/null +++ b/testing/btest/Baseline/scripts.policy.frameworks.intel.seen.certs/intel-all.log @@ -0,0 +1,22 @@ +#separator \x09 +#set_separator , +#empty_field (empty) +#unset_field - +#path intel +#open 2015-03-04-01-12-47 +#fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p fuid file_mime_type file_desc seen.indicator seen.indicator_type seen.where seen.node sources +#types time string addr port addr port string string string string enum enum string set[string] +1416942644.593119 CXWv6p3arKYeMETxOg 192.168.4.149 49422 23.92.19.75 443 F0txuw2pvrkZOn04a8 - 23.92.19.75:443/tcp www.pantz.org Intel::DOMAIN X509::IN_CERT bro source1 +#close 2015-03-04-01-12-47 +#separator \x09 +#set_separator , +#empty_field (empty) +#unset_field - +#path intel +#open 2015-03-04-01-12-47 +#fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p fuid file_mime_type file_desc seen.indicator seen.indicator_type seen.where seen.node sources +#types time string addr port addr port string string string string enum enum string set[string] +1170717505.934612 CXWv6p3arKYeMETxOg 192.150.187.164 58868 194.127.84.106 443 - - - www.dresdner-privat.de Intel::DOMAIN X509::IN_CERT bro source1 +1170717509.082241 CjhGID4nQcgTWjvg4c 192.150.187.164 58869 194.127.84.106 443 - - - www.dresdner-privat.de Intel::DOMAIN X509::IN_CERT bro source1 +1170717512.108799 CCvvfg3TEfuqmmG4bh 192.150.187.164 58870 194.127.84.106 443 - - - www.dresdner-privat.de Intel::DOMAIN X509::IN_CERT bro source1 +#close 2015-03-04-01-12-47 diff --git a/testing/btest/Traces/tls/cert-no-cn.pcap b/testing/btest/Traces/tls/cert-no-cn.pcap new file mode 100644 index 0000000000..d208c696b5 Binary files /dev/null and b/testing/btest/Traces/tls/cert-no-cn.pcap differ diff --git a/testing/btest/scripts/base/protocols/ssl/common_name.test b/testing/btest/scripts/base/protocols/ssl/common_name.test new file mode 100644 index 0000000000..fa14e19045 --- /dev/null +++ b/testing/btest/scripts/base/protocols/ssl/common_name.test @@ -0,0 +1,13 @@ +# This tests a normal SSL connection and the log it outputs. + +# @TEST-EXEC: bro -r $TRACES/tls/tls-conn-with-extensions.trace %INPUT +# @TEST-EXEC: bro -C -r $TRACES/tls/cert-no-cn.pcap %INPUT +# @TEST-EXEC: btest-diff .stdout + +event x509_certificate(f: fa_file, cert_ref: opaque of x509, cert: X509::Certificate) + { + if ( cert?$cn ) + print cert$cn; + else + print "No CN"; + } diff --git a/testing/btest/scripts/policy/frameworks/intel/seen/certs.bro b/testing/btest/scripts/policy/frameworks/intel/seen/certs.bro new file mode 100644 index 0000000000..afddc6b2d9 --- /dev/null +++ b/testing/btest/scripts/policy/frameworks/intel/seen/certs.bro @@ -0,0 +1,18 @@ +# @TEST-EXEC: bro -r $TRACES/tls/ecdsa-cert.pcap %INPUT +# @TEST-EXEC: cat intel.log > intel-all.log +# @TEST-EXEC: bro -r $TRACES/tls/ssl.v3.trace %INPUT +# @TEST-EXEC: cat intel.log >> intel-all.log +# @TEST-EXEC: btest-diff intel-all.log + +@TEST-START-FILE intel.dat +#fields indicator indicator_type meta.source meta.desc meta.url +www.pantz.org Intel::DOMAIN source1 test entry http://some-data-distributor.com/100000 +www.dresdner-privat.de Intel::DOMAIN source1 test entry http://some-data-distributor.com/100000 +@TEST-END-FILE + +@load base/frameworks/intel +@load base/protocols/ssl +@load frameworks/intel/seen + +redef Intel::read_files += { "intel.dat" }; +