diff --git a/CHANGES b/CHANGES index 2cbebd0255..e1d175999f 100644 --- a/CHANGES +++ b/CHANGES @@ -1,4 +1,13 @@ +2.2-120 | 2014-01-28 10:25:23 -0800 + + * Fix and extend x509_extension() event, which now actually returns + the extension. (Bernhard Amann) + + New event signauture: + + event x509_extension(c: connection, is_orig: bool, cert:X509, extension: X509_extension_info) + 2.2-117 | 2014-01-23 14:18:19 -0800 * Fixing initialization context in anonymous functions. (Robin diff --git a/NEWS b/NEWS index df0bd07f4c..07d8e53c52 100644 --- a/NEWS +++ b/NEWS @@ -36,6 +36,11 @@ Changed Functionality - Notice::end_suppression() has been removed. +- Bro now parses X.509 extensions headers and, as a result, the + corresponding event got a new signature: + + event x509_extension(c: connection, is_orig: bool, cert: X509, ext: X509_extension_info); + Bro 2.2 ======= diff --git a/VERSION b/VERSION index 33ad1f2a99..bb754958fd 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -2.2-117 +2.2-120 diff --git a/aux/broctl b/aux/broctl index c4b5fb7336..9ff2e2ced6 160000 --- a/aux/broctl +++ b/aux/broctl @@ -1 +1 @@ -Subproject commit c4b5fb7336f2b598cf69777a7ec91b4aa16cacd1 +Subproject commit 9ff2e2ced64a3bd4af1268154e261671a1153481 diff --git a/scripts/base/init-bare.bro b/scripts/base/init-bare.bro index 8d4899b785..d4e631ecf4 100644 --- a/scripts/base/init-bare.bro +++ b/scripts/base/init-bare.bro @@ -2432,6 +2432,17 @@ type X509: record { not_valid_after: time; ##< Timestamp after when certificate is not valid. }; +## An X509 extension. +## +## .. bro:see:: x509_extension +type X509_extension_info: record { + name: string; ##< Long name of extension; oid if name not known. + short_name: string &optional; ##< Short name of extension if known. + oid: string; ##< Oid of extension. + critical: bool; ##< True if extension is critical. + value: string; ##< Extension content parsed to string for known extensions. Raw data otherwise. +}; + ## HTTP session statistics. ## ## .. bro:see:: http_stats @@ -2849,6 +2860,12 @@ global load_sample_freq = 20 &redef; ## .. bro:see:: gap_report const gap_report_freq = 1.0 sec &redef; +## Whether to attempt to automatically detect SYN/FIN/RST-filtered trace +## and not report missing segments for such connections. +## If this is enabled, then missing data at the end of connections may not +## be reported via :bro:see:`content_gap`. +const detect_filtered_trace = F &redef; + ## Whether we want :bro:see:`content_gap` and :bro:see:`gap_report` for partial ## connections. A connection is partial if it is missing a full handshake. Note ## that gap reports for partial connections might not be reliable. diff --git a/src/NetVar.cc b/src/NetVar.cc index 79652112f3..05a4e16b47 100644 --- a/src/NetVar.cc +++ b/src/NetVar.cc @@ -48,6 +48,7 @@ int tcp_max_above_hole_without_any_acks; int tcp_excessive_data_without_further_acks; RecordType* x509_type; +RecordType* x509_extension_type; RecordType* socks_address; @@ -356,6 +357,7 @@ void init_net_var() opt_internal_int("tcp_excessive_data_without_further_acks"); x509_type = internal_type("X509")->AsRecordType(); + x509_extension_type = internal_type("X509_extension_info")->AsRecordType(); socks_address = internal_type("SOCKS::Address")->AsRecordType(); diff --git a/src/NetVar.h b/src/NetVar.h index 12949c0e55..8ef6571313 100644 --- a/src/NetVar.h +++ b/src/NetVar.h @@ -51,6 +51,7 @@ extern int tcp_max_above_hole_without_any_acks; extern int tcp_excessive_data_without_further_acks; extern RecordType* x509_type; +extern RecordType* x509_extension_type; extern RecordType* socks_address; diff --git a/src/analyzer/protocol/ssl/events.bif b/src/analyzer/protocol/ssl/events.bif index 01abb87745..7319d2ce3e 100644 --- a/src/analyzer/protocol/ssl/events.bif +++ b/src/analyzer/protocol/ssl/events.bif @@ -178,11 +178,13 @@ event x509_certificate%(c: connection, is_orig: bool, cert: X509, chain_idx: cou ## ## is_orig: True if event is raised for originator side of the connection. ## -## data: The raw data associated with the extension. +## cert: The parsed certificate. +## +## extension: The parsed extension. ## ## .. bro:see:: ssl_alert ssl_client_hello ssl_established ssl_extension ## ssl_server_hello x509_certificate x509_error x509_verify -event x509_extension%(c: connection, is_orig: bool, data: string%); +event x509_extension%(c: connection, is_orig: bool, cert: X509, extension: X509_extension_info%); ## Generated when errors occur during parsing an X509 certificate. ## diff --git a/src/analyzer/protocol/ssl/ssl-analyzer.pac b/src/analyzer/protocol/ssl/ssl-analyzer.pac index 18d3812742..280f3a2d32 100644 --- a/src/analyzer/protocol/ssl/ssl-analyzer.pac +++ b/src/analyzer/protocol/ssl/ssl-analyzer.pac @@ -9,6 +9,7 @@ #include "util.h" #include +#include #include %} @@ -211,7 +212,7 @@ refine connection SSL_Conn += { BifEvent::generate_ssl_server_hello(bro_analyzer(), bro_analyzer()->Conn(), - version, ts, new StringVal(server_random.length(), + version, ts, new StringVal(server_random.length(), (const char*) server_random.data()), to_string_val(session_id), ciphers->size()==0 ? 0 : ciphers->at(0), comp_method); @@ -298,27 +299,61 @@ refine connection SSL_Conn += { int num_ext = X509_get_ext_count(pTemp); for ( int k = 0; k < num_ext; ++k ) { - unsigned char *pBuffer = 0; - int length = 0; + char name[256]; + char oid[256]; + + memset(name, 0, sizeof(name)); + memset(oid, 0, sizeof(oid)); X509_EXTENSION* ex = X509_get_ext(pTemp, k); - if (ex) - { - ASN1_STRING *pString = X509_EXTENSION_get_data(ex); - length = ASN1_STRING_to_UTF8(&pBuffer, pString); - //i2t_ASN1_OBJECT(&pBuffer, length, obj) - // printf("extension length: %d\n", length); - // -1 indicates an error. - if ( length >= 0 ) - { - StringVal* value = new StringVal(length, (char*)pBuffer); - BifEvent::generate_x509_extension(bro_analyzer(), - bro_analyzer()->Conn(), ${rec.is_orig}, value); - } - OPENSSL_free(pBuffer); - } + + if ( ! ex ) + continue; + + ASN1_OBJECT* ext_asn = X509_EXTENSION_get_object(ex); + const char* short_name = OBJ_nid2sn(OBJ_obj2nid(ext_asn)); + + OBJ_obj2txt(name, sizeof(name) - 1, ext_asn, 0); + OBJ_obj2txt(oid, sizeof(oid) - 1, ext_asn, 1); + + int critical = 0; + if ( X509_EXTENSION_get_critical(ex) != 0 ) + critical = 1; + + BIO *bio = BIO_new(BIO_s_mem()); + if( ! X509V3_EXT_print(bio, ex, 0, 0)) + M_ASN1_OCTET_STRING_print(bio, ex->value); + + BIO_flush(bio); + int length = BIO_pending(bio); + + // Use OPENSSL_malloc here. Using new or anything else can lead + // to interesting, hard to debug segfaults. + char *buffer = (char*) OPENSSL_malloc(length); + BIO_read(bio, buffer, length); + StringVal* ext_val = new StringVal(length, buffer); + OPENSSL_free(buffer); + + BIO_free_all(bio); + + RecordVal* pX509Ext = new RecordVal(x509_extension_type); + pX509Ext->Assign(0, new StringVal(name)); + + if ( short_name && strlen(short_name) > 0 ) + pX509Ext->Assign(1, new StringVal(short_name)); + + pX509Ext->Assign(2, new StringVal(oid)); + pX509Ext->Assign(3, new Val(critical, TYPE_BOOL)); + pX509Ext->Assign(4, ext_val); + + BifEvent::generate_x509_extension(bro_analyzer(), + bro_analyzer()->Conn(), + ${rec.is_orig}, + pX509Cert->Ref(), + pX509Ext); } } + X509_free(pTemp); } } diff --git a/src/analyzer/protocol/tcp/TCP.cc b/src/analyzer/protocol/tcp/TCP.cc index aefc5a1808..57c4ebef18 100644 --- a/src/analyzer/protocol/tcp/TCP.cc +++ b/src/analyzer/protocol/tcp/TCP.cc @@ -373,14 +373,11 @@ void TCP_Analyzer::ProcessSYN(const IP_Hdr* ip, const struct tcphdr* tp, void TCP_Analyzer::ProcessFIN(double t, TCP_Endpoint* endpoint, int& seq_len, uint32 base_seq) { - if ( endpoint->FIN_cnt == 0 ) - { - ++seq_len; // FIN consumes a byte of sequence space - ++endpoint->FIN_cnt; // remember that we've seen a FIN - } + ++seq_len; // FIN consumes a byte of sequence space. + ++endpoint->FIN_cnt; // remember that we've seen a FIN - else if ( t < endpoint->last_time + tcp_storm_interarrival_thresh && - ++endpoint->FIN_cnt == tcp_storm_thresh ) + if ( t < endpoint->last_time + tcp_storm_interarrival_thresh && + endpoint->FIN_cnt == tcp_storm_thresh ) Weird("FIN_storm"); // Remember the relative seq in FIN_seq. diff --git a/src/analyzer/protocol/tcp/TCP_Endpoint.cc b/src/analyzer/protocol/tcp/TCP_Endpoint.cc index d596234021..ad642a46e3 100644 --- a/src/analyzer/protocol/tcp/TCP_Endpoint.cc +++ b/src/analyzer/protocol/tcp/TCP_Endpoint.cc @@ -161,6 +161,13 @@ void TCP_Endpoint::SetState(EndpointState new_state) bro_int_t TCP_Endpoint::Size() const { + if ( prev_state == TCP_ENDPOINT_SYN_SENT && state == TCP_ENDPOINT_RESET && + peer->state == TCP_ENDPOINT_INACTIVE && ! NoDataAcked() ) + // This looks like a half-open connection was discovered and aborted. + // Sequence numbers could be misleading if used in context of data size + // and there was never a chance for this endpoint to send data anyway. + return 0; + bro_int_t size; uint64 last_seq_64 = (uint64(last_seq_high) << 32) | last_seq; diff --git a/src/analyzer/protocol/tcp/TCP_Reassembler.cc b/src/analyzer/protocol/tcp/TCP_Reassembler.cc index a1e20dc0e6..49292a04a5 100644 --- a/src/analyzer/protocol/tcp/TCP_Reassembler.cc +++ b/src/analyzer/protocol/tcp/TCP_Reassembler.cc @@ -178,7 +178,7 @@ void TCP_Reassembler::Undelivered(int up_to_seq) // to this method and only if this condition is not true). reporter->InternalError("Calling Undelivered for data that has already been delivered (or has already been marked as undelivered"); - if ( last_reassem_seq == 1 && + if ( BifConst::detect_filtered_trace && last_reassem_seq == 1 && (endpoint->FIN_cnt > 0 || endpoint->RST_cnt > 0 || peer->FIN_cnt > 0 || peer->RST_cnt > 0) ) { diff --git a/src/const.bif b/src/const.bif index fd0419c7d9..0ba168ca85 100644 --- a/src/const.bif +++ b/src/const.bif @@ -5,6 +5,7 @@ const ignore_keep_alive_rexmit: bool; const skip_http_data: bool; const use_conn_size_analyzer: bool; +const detect_filtered_trace: bool; const report_gaps_for_partial: bool; const exit_only_after_terminate: bool; diff --git a/src/file_analysis/File.cc b/src/file_analysis/File.cc index 55b28763c8..deda0f9e93 100644 --- a/src/file_analysis/File.cc +++ b/src/file_analysis/File.cc @@ -103,7 +103,6 @@ File::~File() DBG_LOG(DBG_FILE_ANALYSIS, "Destroying File object %s", id.c_str()); Unref(val); - // Queue may not be empty in the case where only content gaps were seen. while ( ! fonc_queue.empty() ) { delete_vals(fonc_queue.front().second); @@ -460,20 +459,27 @@ void File::FileEvent(EventHandlerPtr h) FileEvent(h, vl); } +static void flush_file_event_queue(queue >& q) + { + while ( ! q.empty() ) + { + pair p = q.front(); + mgr.QueueEvent(p.first, p.second); + q.pop(); + } + } + void File::FileEvent(EventHandlerPtr h, val_list* vl) { + if ( h == file_state_remove ) + flush_file_event_queue(fonc_queue); + mgr.QueueEvent(h, vl); if ( h == file_new ) { did_file_new_event = true; - - while ( ! fonc_queue.empty() ) - { - pair p = fonc_queue.front(); - mgr.QueueEvent(p.first, p.second); - fonc_queue.pop(); - } + flush_file_event_queue(fonc_queue); } if ( h == file_new || h == file_timeout || h == file_extraction_limit ) diff --git a/testing/btest/Baseline/core.tcp.fin-retransmit/out b/testing/btest/Baseline/core.tcp.fin-retransmit/out new file mode 100644 index 0000000000..8afb8222c9 --- /dev/null +++ b/testing/btest/Baseline/core.tcp.fin-retransmit/out @@ -0,0 +1,2 @@ +[size=0, state=5, num_pkts=3, num_bytes_ip=156, flow_label=0] +[size=0, state=6, num_pkts=2, num_bytes_ip=92, flow_label=0] diff --git a/testing/btest/Baseline/core.tcp.miss-end-data/conn.log b/testing/btest/Baseline/core.tcp.miss-end-data/conn.log new file mode 100644 index 0000000000..723e5becc3 --- /dev/null +++ b/testing/btest/Baseline/core.tcp.miss-end-data/conn.log @@ -0,0 +1,10 @@ +#separator \x09 +#set_separator , +#empty_field (empty) +#unset_field - +#path conn +#open 2014-01-24-22-19-38 +#fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p proto service duration orig_bytes resp_bytes conn_state local_orig missed_bytes history orig_pkts orig_ip_bytes resp_pkts resp_ip_bytes tunnel_parents +#types time string addr port addr port enum string interval count count string bool count string count count count count table[string] +1331764471.664131 CXWv6p3arKYeMETxOg 192.168.122.230 60648 77.238.160.184 80 tcp http 10.048360 538 2902 SF - 2902 ShADafF 5 750 4 172 (empty) +#close 2014-01-24-22-19-38 diff --git a/testing/btest/Baseline/core.tcp.miss-end-data/out b/testing/btest/Baseline/core.tcp.miss-end-data/out new file mode 100644 index 0000000000..cd5881035f --- /dev/null +++ b/testing/btest/Baseline/core.tcp.miss-end-data/out @@ -0,0 +1 @@ +content_gap, [orig_h=192.168.122.230, orig_p=60648/tcp, resp_h=77.238.160.184, resp_p=80/tcp], F, 1, 2902 diff --git a/testing/btest/Baseline/scripts.base.protocols.ssl.x509_extensions/.stdout b/testing/btest/Baseline/scripts.base.protocols.ssl.x509_extensions/.stdout new file mode 100644 index 0000000000..3f9c8661bf --- /dev/null +++ b/testing/btest/Baseline/scripts.base.protocols.ssl.x509_extensions/.stdout @@ -0,0 +1,20 @@ +[name=X509v3 Authority Key Identifier, short_name=authorityKeyIdentifier, oid=2.5.29.35, critical=F, value=keyid:3F:D5:B5:D0:D6:44:79:50:4A:17:A3:9B:8C:4A:DC:B8:B0:22:64:6B^J] +[name=X509v3 Subject Key Identifier, short_name=subjectKeyIdentifier, oid=2.5.29.14, critical=F, value=A2:76:09:20:A8:40:FD:A1:AC:C8:E9:35:B9:11:A6:61:FF:8C:FF:A3] +[name=X509v3 Key Usage, short_name=keyUsage, oid=2.5.29.15, critical=T, value=Digital Signature, Key Encipherment] +[name=X509v3 Basic Constraints, short_name=basicConstraints, oid=2.5.29.19, critical=T, value=CA:FALSE] +[name=X509v3 Extended Key Usage, short_name=extendedKeyUsage, oid=2.5.29.37, critical=F, value=TLS Web Server Authentication, TLS Web Client Authentication] +[name=X509v3 Certificate Policies, short_name=certificatePolicies, oid=2.5.29.32, critical=F, value=Policy: 1.3.6.1.4.1.6449.1.2.1.3.4^J CPS: https://secure.comodo.com/CPS^J] +[name=X509v3 CRL Distribution Points, short_name=crlDistributionPoints, oid=2.5.29.31, critical=F, value=^JFull Name:^J URI:http://crl.comodoca.com/COMODOHigh-AssuranceSecureServerCA.crl^J] +[name=Authority Information Access, short_name=authorityInfoAccess, oid=1.3.6.1.5.5.7.1.1, critical=F, value=CA Issuers - URI:http://crt.comodoca.com/COMODOHigh-AssuranceSecureServerCA.crt^JOCSP - URI:http://ocsp.comodoca.com^J] +[name=X509v3 Subject Alternative Name, short_name=subjectAltName, oid=2.5.29.17, critical=F, value=DNS:*.taleo.net, DNS:taleo.net] +[name=X509v3 Authority Key Identifier, short_name=authorityKeyIdentifier, oid=2.5.29.35, critical=F, value=keyid:AD:BD:98:7A:34:B4:26:F7:FA:C4:26:54:EF:03:BD:E0:24:CB:54:1A^J] +[name=X509v3 Subject Key Identifier, short_name=subjectKeyIdentifier, oid=2.5.29.14, critical=F, value=3F:D5:B5:D0:D6:44:79:50:4A:17:A3:9B:8C:4A:DC:B8:B0:22:64:6B] +[name=X509v3 Key Usage, short_name=keyUsage, oid=2.5.29.15, critical=T, value=Certificate Sign, CRL Sign] +[name=X509v3 Basic Constraints, short_name=basicConstraints, oid=2.5.29.19, critical=T, value=CA:TRUE, pathlen:0] +[name=X509v3 Certificate Policies, short_name=certificatePolicies, oid=2.5.29.32, critical=F, value=Policy: X509v3 Any Policy^J] +[name=X509v3 CRL Distribution Points, short_name=crlDistributionPoints, oid=2.5.29.31, critical=F, value=^JFull Name:^J URI:http://crl.usertrust.com/AddTrustExternalCARoot.crl^J] +[name=Authority Information Access, short_name=authorityInfoAccess, oid=1.3.6.1.5.5.7.1.1, critical=F, value=CA Issuers - URI:http://crt.usertrust.com/AddTrustExternalCARoot.p7c^JCA Issuers - URI:http://crt.usertrust.com/AddTrustUTNSGCCA.crt^JOCSP - URI:http://ocsp.usertrust.com^J] +[name=X509v3 Subject Key Identifier, short_name=subjectKeyIdentifier, oid=2.5.29.14, critical=F, value=AD:BD:98:7A:34:B4:26:F7:FA:C4:26:54:EF:03:BD:E0:24:CB:54:1A] +[name=X509v3 Key Usage, short_name=keyUsage, oid=2.5.29.15, critical=F, value=Certificate Sign, CRL Sign] +[name=X509v3 Basic Constraints, short_name=basicConstraints, oid=2.5.29.19, critical=T, value=CA:TRUE] +[name=X509v3 Authority Key Identifier, short_name=authorityKeyIdentifier, oid=2.5.29.35, critical=F, value=keyid:AD:BD:98:7A:34:B4:26:F7:FA:C4:26:54:EF:03:BD:E0:24:CB:54:1A^JDirName:/C=SE/O=AddTrust AB/OU=AddTrust External TTP Network/CN=AddTrust External CA Root^Jserial:01^J] diff --git a/testing/btest/Traces/tcp/fin_retransmission.pcap b/testing/btest/Traces/tcp/fin_retransmission.pcap new file mode 100644 index 0000000000..1e17844af5 Binary files /dev/null and b/testing/btest/Traces/tcp/fin_retransmission.pcap differ diff --git a/testing/btest/Traces/tcp/miss_end_data.pcap b/testing/btest/Traces/tcp/miss_end_data.pcap new file mode 100644 index 0000000000..ae5aecbaac Binary files /dev/null and b/testing/btest/Traces/tcp/miss_end_data.pcap differ diff --git a/testing/btest/core/tcp/fin-retransmit.bro b/testing/btest/core/tcp/fin-retransmit.bro new file mode 100644 index 0000000000..42bf062f5a --- /dev/null +++ b/testing/btest/core/tcp/fin-retransmit.bro @@ -0,0 +1,8 @@ +# @TEST-EXEC: bro -b -r $TRACES/tcp/fin_retransmission.pcap %INPUT >out +# @TEST-EXEC: btest-diff out + +event connection_state_remove(c: connection) + { + print c$orig; + print c$resp; + } diff --git a/testing/btest/core/tcp/miss-end-data.bro b/testing/btest/core/tcp/miss-end-data.bro new file mode 100644 index 0000000000..6cee7577d9 --- /dev/null +++ b/testing/btest/core/tcp/miss-end-data.bro @@ -0,0 +1,10 @@ +# @TEST-EXEC: bro -r $TRACES/tcp/miss_end_data.pcap %INPUT >out +# @TEST-EXEC: btest-diff out +# @TEST-EXEC: btest-diff conn.log + +redef report_gaps_for_partial = T; + +event content_gap(c: connection, is_orig: bool, seq: count, length: count) + { + print "content_gap", c$id, is_orig, seq, length; + } diff --git a/testing/btest/scripts/base/protocols/ssl/x509_extensions.test b/testing/btest/scripts/base/protocols/ssl/x509_extensions.test new file mode 100644 index 0000000000..4db3233b27 --- /dev/null +++ b/testing/btest/scripts/base/protocols/ssl/x509_extensions.test @@ -0,0 +1,7 @@ +# @TEST-EXEC: bro -r $TRACES/tls1.2.trace %INPUT +# @TEST-EXEC: btest-diff .stdout + +event x509_extension(c: connection, is_orig: bool, cert:X509, extension: X509_extension_info) +{ + print extension; +}