diff --git a/CHANGES b/CHANGES index 8d40038d5b..08289bf25b 100644 --- a/CHANGES +++ b/CHANGES @@ -1,4 +1,8 @@ +2.6-711 | 2019-07-29 20:15:27 -0700 + + * Fix duplicate TCP packets not being detected as retransmissions (Jeff Barber) + 2.6-708 | 2019-07-30 02:46:39 +0000 * Add an additional license file, COPYING.3rdparty, that collects diff --git a/VERSION b/VERSION index ed6a0f92aa..73531621a5 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -2.6-708 +2.6-711 diff --git a/src/analyzer/protocol/tcp/TCP.cc b/src/analyzer/protocol/tcp/TCP.cc index 74e73b80e2..5be893fd8e 100644 --- a/src/analyzer/protocol/tcp/TCP.cc +++ b/src/analyzer/protocol/tcp/TCP.cc @@ -891,6 +891,8 @@ static void init_endpoint(TCP_Endpoint* endpoint, TCP_Flags flags, // numbering consistent. endpoint->InitAckSeq(first_seg_seq - 1); endpoint->InitStartSeq(first_seg_seq - 1); + // But ensure first packet is not marked duplicate + last_seq = first_seg_seq; } endpoint->InitLastSeq(last_seq); @@ -1019,7 +1021,7 @@ static int32 update_last_seq(TCP_Endpoint* endpoint, uint32 last_seq, // ## endpoint->last_seq = last_seq; endpoint->UpdateLastSeq(last_seq); - else if ( delta_last < 0 && len > 0 ) + else if ( delta_last <= 0 && len > 0 ) endpoint->DidRxmit(); return delta_last; diff --git a/testing/btest/Baseline/core.pppoe-over-qinq/conn.log b/testing/btest/Baseline/core.pppoe-over-qinq/conn.log index 028dd982fb..2c2b146f4c 100644 --- a/testing/btest/Baseline/core.pppoe-over-qinq/conn.log +++ b/testing/btest/Baseline/core.pppoe-over-qinq/conn.log @@ -3,8 +3,8 @@ #empty_field (empty) #unset_field - #path conn -#open 2018-08-01-20-09-03 +#open 2019-07-26-20-04-59 #fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p proto service duration orig_bytes resp_bytes conn_state local_orig local_resp missed_bytes history orig_pkts orig_ip_bytes resp_pkts resp_ip_bytes tunnel_parents #types time string addr port addr port enum string interval count count string bool bool count string count count count count set[string] -1523351398.449222 CHhAvVGS1DHFjwGM9 1.1.1.1 20394 2.2.2.2 443 tcp - 273.626833 11352 4984 SF - - 0 ShADdtaTTFf 44 25283 42 13001 - -#close 2018-08-01-20-09-03 +1523351398.449222 CHhAvVGS1DHFjwGM9 1.1.1.1 20394 2.2.2.2 443 tcp - 273.626833 11352 4984 SF - - 0 ShADdtaTTtFf 44 25283 42 13001 - +#close 2019-07-26-20-05-00 diff --git a/testing/btest/Baseline/core.tcp.tcp-dups/out b/testing/btest/Baseline/core.tcp.tcp-dups/out new file mode 100644 index 0000000000..600bef0294 --- /dev/null +++ b/testing/btest/Baseline/core.tcp.tcp-dups/out @@ -0,0 +1,3 @@ +RETRANSMITS:, [orig_h=192.168.0.102, orig_p=53206/tcp, resp_h=192.168.0.112, resp_p=22/tcp], T, 10, ShADTadtT +RETRANSMITS:, [orig_h=192.168.0.102, orig_p=53206/tcp, resp_h=192.168.0.112, resp_p=22/tcp], F, 10, ShADTadtTt +REMOVE:, [orig_h=192.168.0.102, orig_p=53206/tcp, resp_h=192.168.0.112, resp_p=22/tcp], ShADTadtTtFf diff --git a/testing/btest/Baseline/scripts.base.protocols.http.http-connect/conn.log b/testing/btest/Baseline/scripts.base.protocols.http.http-connect/conn.log index af890ad64a..d4eff0f151 100644 --- a/testing/btest/Baseline/scripts.base.protocols.http.http-connect/conn.log +++ b/testing/btest/Baseline/scripts.base.protocols.http.http-connect/conn.log @@ -3,8 +3,8 @@ #empty_field (empty) #unset_field - #path conn -#open 2016-07-13-16-16-21 +#open 2019-07-26-20-05-28 #fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p proto service duration orig_bytes resp_bytes conn_state local_orig local_resp missed_bytes history orig_pkts orig_ip_bytes resp_pkts resp_ip_bytes tunnel_parents #types time string addr port addr port enum string interval count count string bool bool count string count count count count set[string] -1078232251.833846 CHhAvVGS1DHFjwGM9 79.26.245.236 3378 254.228.86.79 8240 tcp smtp,http 6.722274 1685 223 SF - - 0 ShADadfF 14 2257 16 944 - -#close 2016-07-13-16-16-21 +1078232251.833846 CHhAvVGS1DHFjwGM9 79.26.245.236 3378 254.228.86.79 8240 tcp smtp,http 6.722274 1685 223 SF - - 0 ShADadtTfF 14 2257 16 944 - +#close 2019-07-26-20-05-29 diff --git a/testing/btest/Baseline/scripts.base.protocols.irc.basic/conn.log b/testing/btest/Baseline/scripts.base.protocols.irc.basic/conn.log index d7064790ae..b2abaa7126 100644 --- a/testing/btest/Baseline/scripts.base.protocols.irc.basic/conn.log +++ b/testing/btest/Baseline/scripts.base.protocols.irc.basic/conn.log @@ -3,9 +3,9 @@ #empty_field (empty) #unset_field - #path conn -#open 2016-07-13-16-16-28 +#open 2019-07-26-20-10-57 #fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p proto service duration orig_bytes resp_bytes conn_state local_orig local_resp missed_bytes history orig_pkts orig_ip_bytes resp_pkts resp_ip_bytes tunnel_parents #types time string addr port addr port enum string interval count count string bool bool count string count count count count set[string] 1311189318.898709 ClEkJM2Vm5giqnMf4h 192.168.1.77 57655 209.197.168.151 1024 tcp irc-dcc-data 2.256935 124 42208 SF - - 0 ShAdDaFf 28 1592 43 44452 - -1311189164.064603 CHhAvVGS1DHFjwGM9 192.168.1.77 57640 66.198.80.67 6667 tcp irc 178.237017 453 25404 S3 - - 0 ShADdaf 63 3761 52 28194 - -#close 2016-07-13-16-16-28 +1311189164.064603 CHhAvVGS1DHFjwGM9 192.168.1.77 57640 66.198.80.67 6667 tcp irc 178.237017 453 25404 S3 - - 0 ShADdTtaf 63 3761 52 28194 - +#close 2019-07-26-20-10-58 diff --git a/testing/btest/Baseline/scripts.policy.misc.dump-events/all-events.log b/testing/btest/Baseline/scripts.policy.misc.dump-events/all-events.log index 753df3ecca..f6a43528ff 100644 --- a/testing/btest/Baseline/scripts.policy.misc.dump-events/all-events.log +++ b/testing/btest/Baseline/scripts.policy.misc.dump-events/all-events.log @@ -1043,7 +1043,7 @@ [0] c: connection = [id=[orig_h=192.168.133.100, orig_p=49648/tcp, resp_h=192.168.133.102, resp_p=25/tcp], orig=[size=969, state=4, num_pkts=17, num_bytes_ip=1865, flow_label=0, l2_addr=58:b0:35:86:54:8d], resp=[size=162, state=4, num_pkts=10, num_bytes_ip=690, flow_label=0, l2_addr=00:08:ca:cc:ad:4c], start_time=1437831787.856895, duration=0.05732, service={\x0aSMTP\x0a}, history=ShAdDa, uid=CmES5u32sYpV7JYN, tunnel=, vlan=, inner_vlan=, dpd=, dpd_state=, conn=, extract_orig=F, extract_resp=F, thresholds=, dce_rpc=, dce_rpc_state=, dce_rpc_backing=, dhcp=, dnp3=, dns=, dns_state=, ftp=, ftp_data_reuse=F, ssl=, http=, http_state=, irc=, krb=, modbus=, mysql=, ntlm=, ntp=, radius=, rdp=, rfb=, sip=, sip_state=, snmp=, smb_state=, smtp=[ts=1437831787.914113, uid=CmES5u32sYpV7JYN, id=[orig_h=192.168.133.100, orig_p=49648/tcp, resp_h=192.168.133.102, resp_p=25/tcp], trans_depth=2, helo=[192.168.133.100], mailfrom=, rcptto=, date=, from=, to=, cc=, reply_to=, msg_id=, in_reply_to=, subject=, x_originating_ip=, first_received=, second_received=, last_reply=, path=[192.168.133.102, 192.168.133.100], user_agent=, tls=F, process_received_from=T, has_client_activity=F, entity=, fuids=[]], smtp_state=[helo=[192.168.133.100], messages_transferred=1, pending_messages=, mime_depth=1], socks=, ssh=, syslog=] 1437831800.217854 connection_state_remove - [0] c: connection = [id=[orig_h=192.168.133.100, orig_p=49336/tcp, resp_h=74.125.71.189, resp_p=443/tcp], orig=[size=0, state=3, num_pkts=3, num_bytes_ip=156, flow_label=0, l2_addr=58:b0:35:86:54:8d], resp=[size=85, state=3, num_pkts=3, num_bytes_ip=411, flow_label=0, l2_addr=cc:b2:55:f4:62:92], start_time=1437831798.533593, duration=0.000221, service={\x0a\x0a}, history=^dA, uid=CP5puj4I8PtEU4qzYg, tunnel=, vlan=, inner_vlan=, dpd=, dpd_state=, conn=, extract_orig=F, extract_resp=F, thresholds=, dce_rpc=, dce_rpc_state=, dce_rpc_backing=, dhcp=, dnp3=, dns=, dns_state=, ftp=, ftp_data_reuse=F, ssl=, http=, http_state=, irc=, krb=, modbus=, mysql=, ntlm=, ntp=, radius=, rdp=, rfb=, sip=, sip_state=, snmp=, smb_state=, smtp=, smtp_state=, socks=, ssh=, syslog=] + [0] c: connection = [id=[orig_h=192.168.133.100, orig_p=49336/tcp, resp_h=74.125.71.189, resp_p=443/tcp], orig=[size=0, state=3, num_pkts=3, num_bytes_ip=156, flow_label=0, l2_addr=58:b0:35:86:54:8d], resp=[size=85, state=3, num_pkts=3, num_bytes_ip=411, flow_label=0, l2_addr=cc:b2:55:f4:62:92], start_time=1437831798.533593, duration=0.000221, service={\x0a\x0a}, history=^dtA, uid=CP5puj4I8PtEU4qzYg, tunnel=, vlan=, inner_vlan=, dpd=, dpd_state=, conn=, extract_orig=F, extract_resp=F, thresholds=, dce_rpc=, dce_rpc_state=, dce_rpc_backing=, dhcp=, dnp3=, dns=, dns_state=, ftp=, ftp_data_reuse=F, ssl=, http=, http_state=, irc=, krb=, modbus=, mysql=, ntlm=, ntp=, radius=, rdp=, rfb=, sip=, sip_state=, snmp=, smb_state=, smtp=, smtp_state=, socks=, ssh=, syslog=] 1437831800.217854 connection_state_remove [0] c: connection = [id=[orig_h=192.168.133.100, orig_p=49655/tcp, resp_h=17.167.150.73, resp_p=443/tcp], orig=[size=2249, state=4, num_pkts=15, num_bytes_ip=2873, flow_label=0, l2_addr=58:b0:35:86:54:8d], resp=[size=3653, state=4, num_pkts=13, num_bytes_ip=4185, flow_label=0, l2_addr=cc:b2:55:f4:62:92], start_time=1437831799.461152, duration=0.756702, service={\x0aSSL\x0a}, history=ShADda, uid=C3eiCBGOLw3VtHfOj, tunnel=, vlan=, inner_vlan=, dpd=, dpd_state=, conn=, extract_orig=F, extract_resp=F, thresholds=, dce_rpc=, dce_rpc_state=, dce_rpc_backing=, dhcp=, dnp3=, dns=, dns_state=, ftp=, ftp_data_reuse=F, ssl=[ts=1437831799.611764, uid=C3eiCBGOLw3VtHfOj, id=[orig_h=192.168.133.100, orig_p=49655/tcp, resp_h=17.167.150.73, resp_p=443/tcp], version_num=771, version=TLSv12, cipher=TLS_RSA_WITH_RC4_128_MD5, curve=, server_name=p31-keyvalueservice.icloud.com, session_id=, resumed=F, client_ticket_empty_session_seen=F, client_key_exchange_seen=T, server_appdata=0, client_appdata=F, last_alert=, next_protocol=, analyzer_id=, established=T, logged=T, delay_tokens=, cert_chain=[[ts=1437831799.764576, fuid=F1vce92FT1oRjKI328, tx_hosts={\x0a\x0917.167.150.73\x0a}, rx_hosts={\x0a\x09192.168.133.100\x0a}, conn_uids={\x0aC3eiCBGOLw3VtHfOj\x0a}, source=SSL, depth=0, analyzers={\x0aMD5,\x0aSHA1,\x0aX509\x0a}, mime_type=application/x-x509-user-cert, filename=, duration=0 secs, local_orig=, is_orig=F, seen_bytes=1406, total_bytes=, missing_bytes=0, overflow_bytes=0, timedout=F, parent_fuid=, md5=1bf9696d9f337805383427e88781d001, sha1=f5ccb1a724133607548b00d8eb402efca3076d58, sha256=, x509=[ts=1437831799.764576, id=F1vce92FT1oRjKI328, certificate=[version=3, serial=053FCE9BA6805B00, subject=C=US,ST=California,O=Apple Inc.,OU=management:idms.group.506364,CN=*.icloud.com, issuer=C=US,O=Apple Inc.,OU=Certification Authority,CN=Apple IST CA 2 - G1, cn=*.icloud.com, not_valid_before=1424184331.0, not_valid_after=1489848331.0, key_alg=rsaEncryption, sig_alg=sha256WithRSAEncryption, key_type=rsa, key_length=2048, exponent=65537, curve=], handle=, extensions=[[name=Authority Information Access, short_name=authorityInfoAccess, oid=1.3.6.1.5.5.7.1.1, critical=F, value=OCSP - URI:http://ocsp.apple.com/ocsp04-appleistca2g101\x0a], [name=X509v3 Subject Key Identifier, short_name=subjectKeyIdentifier, oid=2.5.29.14, critical=F, value=8E:51:A1:0E:0A:9B:1C:04:F7:59:D3:69:2E:23:16:91:0E:AD:06:FB], [name=X509v3 Basic Constraints, short_name=basicConstraints, oid=2.5.29.19, critical=T, value=CA:FALSE], [name=X509v3 Authority Key Identifier, short_name=authorityKeyIdentifier, oid=2.5.29.35, critical=F, value=keyid:D8:7A:94:44:7C:90:70:90:16:9E:DD:17:9C:01:44:03:86:D6:2A:29\x0a], [name=X509v3 Certificate Policies, short_name=certificatePolicies, oid=2.5.29.32, critical=F, value=Policy: 1.2.840.113635.100.5.11.4\x0a User Notice:\x0a Explicit Text: Reliance on this certificate by any party assumes acceptance of any applicable terms and conditions of use and/or certification practice statements.\x0a CPS: http://www.apple.com/certificateauthority/rpa\x0a], [name=X509v3 CRL Distribution Points, short_name=crlDistributionPoints, oid=2.5.29.31, critical=F, value=\x0aFull Name:\x0a URI:http://crl.apple.com/appleistca2g1.crl\x0a], [name=X509v3 Key Usage, short_name=keyUsage, oid=2.5.29.15, critical=T, value=Digital Signature, Key Encipherment], [name=X509v3 Extended Key Usage, short_name=extendedKeyUsage, oid=2.5.29.37, critical=F, value=TLS Web Server Authentication, TLS Web Client Authentication], [name=X509v3 Subject Alternative Name, short_name=subjectAltName, oid=2.5.29.17, critical=F, value=DNS:*.icloud.com]], san=[dns=[*.icloud.com], uri=, email=, ip=, other_fields=F], basic_constraints=[ca=F, path_len=]], extracted=, extracted_cutoff=, extracted_size=], [ts=1437831799.764576, fuid=Fxp53s3wA5G3zdEJg8, tx_hosts={\x0a\x0917.167.150.73\x0a}, rx_hosts={\x0a\x09192.168.133.100\x0a}, conn_uids={\x0aC3eiCBGOLw3VtHfOj\x0a}, source=SSL, depth=0, analyzers={\x0aMD5,\x0aSHA1,\x0aX509\x0a}, mime_type=application/x-x509-ca-cert, filename=, duration=0 secs, local_orig=, is_orig=F, seen_bytes=1092, total_bytes=, missing_bytes=0, overflow_bytes=0, timedout=F, parent_fuid=, md5=48f0e38385112eeca5fc9ffd402eaecd, sha1=8e8321ca08b08e3726fe1d82996884eeb5f0d655, sha256=, x509=[ts=1437831799.764576, id=Fxp53s3wA5G3zdEJg8, certificate=[version=3, serial=023A74, subject=C=US,O=Apple Inc.,OU=Certification Authority,CN=Apple IST CA 2 - G1, issuer=CN=GeoTrust Global CA,O=GeoTrust Inc.,C=US, cn=Apple IST CA 2 - G1, not_valid_before=1402933322.0, not_valid_after=1653061322.0, key_alg=rsaEncryption, sig_alg=sha256WithRSAEncryption, key_type=rsa, key_length=2048, exponent=65537, curve=], handle=, extensions=[[name=X509v3 Authority Key Identifier, short_name=authorityKeyIdentifier, oid=2.5.29.35, critical=F, value=keyid:C0:7A:98:68:8D:89:FB:AB:05:64:0C:11:7D:AA:7D:65:B8:CA:CC:4E\x0a], [name=X509v3 Subject Key Identifier, short_name=subjectKeyIdentifier, oid=2.5.29.14, critical=F, value=D8:7A:94:44:7C:90:70:90:16:9E:DD:17:9C:01:44:03:86:D6:2A:29], [name=X509v3 Basic Constraints, short_name=basicConstraints, oid=2.5.29.19, critical=T, value=CA:TRUE, pathlen:0], [name=X509v3 Key Usage, short_name=keyUsage, oid=2.5.29.15, critical=T, value=Certificate Sign, CRL Sign], [name=X509v3 CRL Distribution Points, short_name=crlDistributionPoints, oid=2.5.29.31, critical=F, value=\x0aFull Name:\x0a URI:http://g.symcb.com/crls/gtglobal.crl\x0a], [name=Authority Information Access, short_name=authorityInfoAccess, oid=1.3.6.1.5.5.7.1.1, critical=F, value=OCSP - URI:http://g.symcd.com\x0a], [name=X509v3 Certificate Policies, short_name=certificatePolicies, oid=2.5.29.32, critical=F, value=Policy: 2.16.840.1.113733.1.7.54\x0a CPS: http://www.geotrust.com/resources/cps\x0a]], san=, basic_constraints=[ca=T, path_len=0]], extracted=, extracted_cutoff=, extracted_size=]], cert_chain_fuids=[F1vce92FT1oRjKI328, Fxp53s3wA5G3zdEJg8], client_cert_chain=[], client_cert_chain_fuids=[], subject=C=US,ST=California,O=Apple Inc.,OU=management:idms.group.506364,CN=*.icloud.com, issuer=C=US,O=Apple Inc.,OU=Certification Authority,CN=Apple IST CA 2 - G1, client_subject=, client_issuer=, server_depth=0, client_depth=0], http=, http_state=, irc=, krb=, modbus=, mysql=, ntlm=, ntp=, radius=, rdp=, rfb=, sip=, sip_state=, snmp=, smb_state=, smtp=, smtp_state=, socks=, ssh=, syslog=] diff --git a/testing/btest/Traces/tcp/ssh-dups.pcap b/testing/btest/Traces/tcp/ssh-dups.pcap new file mode 100644 index 0000000000..cbe8ba8f35 Binary files /dev/null and b/testing/btest/Traces/tcp/ssh-dups.pcap differ diff --git a/testing/btest/core/tcp/tcp-dups.zeek b/testing/btest/core/tcp/tcp-dups.zeek new file mode 100644 index 0000000000..4857160561 --- /dev/null +++ b/testing/btest/core/tcp/tcp-dups.zeek @@ -0,0 +1,12 @@ +# @TEST-EXEC: zeek -C -r $TRACES/tcp/ssh-dups.pcap %INPUT >out +# @TEST-EXEC: btest-diff out + +event tcp_multiple_retransmissions(c: connection, is_orig: bool, threshold: count) + { + print "RETRANSMITS:", c$id, is_orig, threshold, c$history; + } + +event connection_state_remove(c: connection) + { + print "REMOVE:", c$id, c$history; + } diff --git a/testing/external/commit-hash.zeek-testing b/testing/external/commit-hash.zeek-testing index 7fd8982987..9ac9662c8e 100644 --- a/testing/external/commit-hash.zeek-testing +++ b/testing/external/commit-hash.zeek-testing @@ -1 +1 @@ -84239d2fdd2f491f436f8597e8b6ca5fb93f7a5f +4e78e7e6f9baf56ec6303d2580f380628fd31e36 diff --git a/testing/external/commit-hash.zeek-testing-private b/testing/external/commit-hash.zeek-testing-private index 098a51358b..f937b4504b 100644 --- a/testing/external/commit-hash.zeek-testing-private +++ b/testing/external/commit-hash.zeek-testing-private @@ -1 +1 @@ -e485d5c6ce4407c9b62880e075b1ba86d8d563cd +d4500752b4359db494d4f24b04543986e76eefec