diff --git a/scripts/base/protocols/conn/main.zeek b/scripts/base/protocols/conn/main.zeek index e2209b6e22..ed28bd6104 100644 --- a/scripts/base/protocols/conn/main.zeek +++ b/scripts/base/protocols/conn/main.zeek @@ -107,6 +107,7 @@ export { ## f packet with FIN bit set ## r packet with RST bit set ## c packet with a bad checksum (applies to UDP too) + ## g a content gap ## t packet with retransmitted payload ## w packet with a zero window advertisement ## i inconsistent packet (e.g. FIN+RST bits set) @@ -122,7 +123,7 @@ export { ## 's' can be recorded multiple times for either direction ## if the associated sequence number differs from the ## last-seen packet of the same flag type. - ## 'c', 't' and 'w' are recorded in a logarithmic fashion: + ## 'c', 'g', 't' and 'w' are recorded in a logarithmic fashion: ## the second instance represents that the event was seen ## (at least) 10 times; the third instance, 100 times; etc. history: string &log &optional; diff --git a/src/analyzer/protocol/tcp/TCP.cc b/src/analyzer/protocol/tcp/TCP.cc index 9329b103ed..595fe8e6b6 100644 --- a/src/analyzer/protocol/tcp/TCP.cc +++ b/src/analyzer/protocol/tcp/TCP.cc @@ -1321,6 +1321,14 @@ void TCP_Analyzer::DeliverPacket(int len, const u_char* data, bool is_orig, PacketWithRST(); } + int32 delta_last = update_last_seq(endpoint, seq_one_past_segment, flags, len); + endpoint->last_time = current_timestamp; + + int do_close; + int gen_event; + UpdateStateMachine(current_timestamp, endpoint, peer, base_seq, ack_seq, + len, delta_last, is_orig, flags, do_close, gen_event); + uint64 rel_ack = 0; if ( flags.ACK() ) @@ -1350,21 +1358,13 @@ void TCP_Analyzer::DeliverPacket(int len, const u_char* data, bool is_orig, Weird("TCP_ack_underflow_or_misorder"); } else if ( ! flags.RST() ) - // Don't trust ack's in RSt packets. + // Don't trust ack's in RST packets. update_ack_seq(peer, ack_seq); } peer->AckReceived(rel_ack); } - int32 delta_last = update_last_seq(endpoint, seq_one_past_segment, flags, len); - endpoint->last_time = current_timestamp; - - int do_close; - int gen_event; - UpdateStateMachine(current_timestamp, endpoint, peer, base_seq, ack_seq, - len, delta_last, is_orig, flags, do_close, gen_event); - if ( tcp_packet ) GeneratePacketEvent(rel_seq, rel_ack, data, len, caplen, is_orig, flags); diff --git a/src/analyzer/protocol/tcp/TCP_Endpoint.cc b/src/analyzer/protocol/tcp/TCP_Endpoint.cc index 7e7b316e10..99551cd211 100644 --- a/src/analyzer/protocol/tcp/TCP_Endpoint.cc +++ b/src/analyzer/protocol/tcp/TCP_Endpoint.cc @@ -32,8 +32,8 @@ TCP_Endpoint::TCP_Endpoint(TCP_Analyzer* arg_analyzer, int arg_is_orig) tcp_analyzer = arg_analyzer; is_orig = arg_is_orig; - chk_cnt = rxmt_cnt = win0_cnt = 0; - chk_thresh = rxmt_thresh = win0_thresh = 1; + gap_cnt = chk_cnt = rxmt_cnt = win0_cnt = 0; + gap_thresh = chk_thresh = rxmt_thresh = win0_thresh = 1; hist_last_SYN = hist_last_FIN = hist_last_RST = 0; @@ -313,3 +313,11 @@ void TCP_Endpoint::ZeroWindow() Conn()->HistoryThresholdEvent(tcp_multiple_zero_windows, IsOrig(), t); } + +void TCP_Endpoint::Gap(uint64 seq, uint64 len) + { + uint32 t = gap_thresh; + if ( Conn()->ScaledHistoryEntry(IsOrig() ? 'G' : 'g', + gap_cnt, gap_thresh) ) + Conn()->HistoryThresholdEvent(tcp_multiple_gap, IsOrig(), t); + } diff --git a/src/analyzer/protocol/tcp/TCP_Endpoint.h b/src/analyzer/protocol/tcp/TCP_Endpoint.h index 4c38aadd93..4c1cf64d6c 100644 --- a/src/analyzer/protocol/tcp/TCP_Endpoint.h +++ b/src/analyzer/protocol/tcp/TCP_Endpoint.h @@ -175,6 +175,9 @@ public: // Called to inform endpoint that it has offered a zero window. void ZeroWindow(); + // Called to inform endpoint that it a gap occurred. + void Gap(uint64 seq, uint64 len); + // Returns true if the data was used (and hence should be recorded // in the save file), false otherwise. int DataSent(double t, uint64 seq, int len, int caplen, const u_char* data, @@ -240,6 +243,7 @@ protected: uint32 chk_cnt, chk_thresh; uint32 rxmt_cnt, rxmt_thresh; uint32 win0_cnt, win0_thresh; + uint32 gap_cnt, gap_thresh; }; #define ENDIAN_UNKNOWN 0 diff --git a/src/analyzer/protocol/tcp/TCP_Reassembler.cc b/src/analyzer/protocol/tcp/TCP_Reassembler.cc index ef68f621b5..5a82197054 100644 --- a/src/analyzer/protocol/tcp/TCP_Reassembler.cc +++ b/src/analyzer/protocol/tcp/TCP_Reassembler.cc @@ -134,6 +134,9 @@ void TCP_Reassembler::Gap(uint64 seq, uint64 len) // The one opportunity we lose here is on clean FIN // handshakes, but Oh Well. + if ( established(endp, endp->peer) ) + endp->Gap(seq, len); + if ( report_gap(endp, endp->peer) ) { val_list* vl = new val_list; diff --git a/src/analyzer/protocol/tcp/events.bif b/src/analyzer/protocol/tcp/events.bif index d93ebe4819..390dadec0f 100644 --- a/src/analyzer/protocol/tcp/events.bif +++ b/src/analyzer/protocol/tcp/events.bif @@ -300,7 +300,7 @@ event tcp_rexmit%(c: connection, is_orig: bool, seq: count, len: count, data_in_ ## threshold: the threshold that was crossed ## ## .. bro:see:: udp_multiple_checksum_errors -## tcp_multiple_zero_windows tcp_multiple_retransmissions +## tcp_multiple_zero_windows tcp_multiple_retransmissions tcp_multiple_gap event tcp_multiple_checksum_errors%(c: connection, is_orig: bool, threshold: count%); ## Generated if a TCP flow crosses a zero-window threshold, per @@ -312,7 +312,7 @@ event tcp_multiple_checksum_errors%(c: connection, is_orig: bool, threshold: cou ## ## threshold: the threshold that was crossed ## -## .. bro:see:: tcp_multiple_checksum_errors tcp_multiple_retransmissions +## .. bro:see:: tcp_multiple_checksum_errors tcp_multiple_retransmissions tcp_multiple_gap event tcp_multiple_zero_windows%(c: connection, is_orig: bool, threshold: count%); ## Generated if a TCP flow crosses a retransmission threshold, per @@ -324,9 +324,21 @@ event tcp_multiple_zero_windows%(c: connection, is_orig: bool, threshold: count% ## ## threshold: the threshold that was crossed ## -## .. bro:see:: tcp_multiple_checksum_errors tcp_multiple_zero_windows +## .. bro:see:: tcp_multiple_checksum_errors tcp_multiple_zero_windows tcp_multiple_gap event tcp_multiple_retransmissions%(c: connection, is_orig: bool, threshold: count%); +## Generated if a TCP flow crosses a gap threshold, per 'G'/'g' history +## reporting. +## +## c: The connection record for the TCP connection. +## +## is_orig: True if the event is raised for the originator side. +## +## threshold: the threshold that was crossed +## +## .. bro:see:: tcp_multiple_checksum_errors tcp_multiple_zero_windows tcp_multiple_retransmissions +event tcp_multiple_gap%(c: connection, is_orig: bool, threshold: count%); + ## Generated when failing to write contents of a TCP stream to a file. ## ## c: The connection whose contents are being recorded. diff --git a/testing/btest/Baseline/core.tcp.large-file-reassembly/conn.log b/testing/btest/Baseline/core.tcp.large-file-reassembly/conn.log index 8da44df913..3a997687d1 100644 --- a/testing/btest/Baseline/core.tcp.large-file-reassembly/conn.log +++ b/testing/btest/Baseline/core.tcp.large-file-reassembly/conn.log @@ -3,10 +3,10 @@ #empty_field (empty) #unset_field - #path conn -#open 2016-07-13-16-13-01 +#open 2019-04-17-20-41-29 #fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p proto service duration orig_bytes resp_bytes conn_state local_orig local_resp missed_bytes history orig_pkts orig_ip_bytes resp_pkts resp_ip_bytes tunnel_parents #types time string addr port addr port enum string interval count count string bool bool count string count count count count set[string] 1395939406.175845 ClEkJM2Vm5giqnMf4h 192.168.56.1 59763 192.168.56.101 63988 tcp ftp-data 0.001676 0 270 SF - - 0 ShAdfFa 5 272 4 486 - -1395939411.361078 C4J4Th3PJpwUYZZ6gc 192.168.56.1 59764 192.168.56.101 37150 tcp ftp-data 150.496065 0 5416666670 SF - - 4675708816 ShAdfFa 13 688 12 24454 - +1395939411.361078 C4J4Th3PJpwUYZZ6gc 192.168.56.1 59764 192.168.56.101 37150 tcp ftp-data 150.496065 0 5416666670 SF - - 4675708816 ShAdgfFa 13 688 12 24454 - 1395939399.984671 CHhAvVGS1DHFjwGM9 192.168.56.1 59762 192.168.56.101 21 tcp ftp 169.634297 104 1041 SF - - 0 ShAdDaFf 31 1728 18 1985 - -#close 2016-07-13-16-13-01 +#close 2019-04-17-20-41-29 diff --git a/testing/btest/Baseline/core.tcp.large-file-reassembly/files.log b/testing/btest/Baseline/core.tcp.large-file-reassembly/files.log index 31087d58cc..15de6047b6 100644 --- a/testing/btest/Baseline/core.tcp.large-file-reassembly/files.log +++ b/testing/btest/Baseline/core.tcp.large-file-reassembly/files.log @@ -3,9 +3,9 @@ #empty_field (empty) #unset_field - #path files -#open 2017-01-25-07-03-11 +#open 2019-04-17-20-41-29 #fields ts fuid tx_hosts rx_hosts conn_uids source depth analyzers mime_type filename duration local_orig is_orig seen_bytes total_bytes missing_bytes overflow_bytes timedout parent_fuid md5 sha1 sha256 extracted extracted_cutoff extracted_size #types time string set[addr] set[addr] set[string] string count set[string] string string interval bool bool count count count count bool string string string string string bool count 1395939406.177079 FAb5m22Dhe2Zi95anf 192.168.56.101 192.168.56.1 ClEkJM2Vm5giqnMf4h FTP_DATA 0 DATA_EVENT text/plain - 0.000000 - F 270 - 0 0 F - - - - - - - 1395939411.364462 FhI0ao2FNTjabdfSBd 192.168.56.101 192.168.56.1 C4J4Th3PJpwUYZZ6gc FTP_DATA 0 DATA_EVENT text/plain - 150.490904 - F 23822 - 5416642848 0 F - - - - - - - -#close 2017-01-25-07-03-11 +#close 2019-04-17-20-41-29 diff --git a/testing/btest/Baseline/core.tcp.rxmit-history/conn-1.log b/testing/btest/Baseline/core.tcp.rxmit-history/conn-1.log index 43daf101a3..466f882257 100644 --- a/testing/btest/Baseline/core.tcp.rxmit-history/conn-1.log +++ b/testing/btest/Baseline/core.tcp.rxmit-history/conn-1.log @@ -3,8 +3,8 @@ #empty_field (empty) #unset_field - #path conn -#open 2018-01-12-21-43-34 +#open 2019-04-17-20-42-43 #fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p proto service duration orig_bytes resp_bytes conn_state local_orig local_resp missed_bytes history orig_pkts orig_ip_bytes resp_pkts resp_ip_bytes tunnel_parents #types time string addr port addr port enum string interval count count string bool bool count string count count count count set[string] -1285862902.700271 CHhAvVGS1DHFjwGM9 10.0.88.85 50368 192.168.0.27 80 tcp - 60.991770 474 23783 RSTO - - 24257 ShADadtR 17 1250 22 28961 - -#close 2018-01-12-21-43-34 +1285862902.700271 CHhAvVGS1DHFjwGM9 10.0.88.85 50368 192.168.0.27 80 tcp - 60.991770 474 23783 RSTO - - 24257 ShADaGdgtR 17 1250 22 28961 - +#close 2019-04-17-20-42-43 diff --git a/testing/btest/Baseline/core.tcp.rxmit-history/conn-2.log b/testing/btest/Baseline/core.tcp.rxmit-history/conn-2.log index 22d4ec3ab9..e75d9487d0 100644 --- a/testing/btest/Baseline/core.tcp.rxmit-history/conn-2.log +++ b/testing/btest/Baseline/core.tcp.rxmit-history/conn-2.log @@ -3,7 +3,7 @@ #empty_field (empty) #unset_field - #path conn -#open 2018-01-12-21-43-35 +#open 2019-04-17-20-42-44 #fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p proto service duration orig_bytes resp_bytes conn_state local_orig local_resp missed_bytes history orig_pkts orig_ip_bytes resp_pkts resp_ip_bytes tunnel_parents #types time string addr port addr port enum string interval count count string bool bool count string count count count count set[string] 1300475167.096535 CHhAvVGS1DHFjwGM9 141.142.220.202 5353 224.0.0.251 5353 udp dns - - - S0 - - 0 D 1 73 0 0 - @@ -40,4 +40,4 @@ 1300475168.859163 Ck51lg1bScffFj34Ri 141.142.220.118 49998 208.80.152.3 80 tcp http 0.215893 1130 734 S1 - - 0 ShADad 6 1450 4 950 - 1300475168.892936 CtxTCR2Yer0FR1tIBg 141.142.220.118 50000 208.80.152.3 80 tcp http 0.229603 1148 734 S1 - - 0 ShADad 6 1468 4 950 - 1300475168.895267 CLNN1k2QMum1aexUK7 141.142.220.118 50001 208.80.152.3 80 tcp http 0.227284 1178 734 S1 - - 0 ShADad 6 1498 4 950 - -#close 2018-01-12-21-43-35 +#close 2019-04-17-20-42-44 diff --git a/testing/btest/Baseline/scripts.base.frameworks.netcontrol.packetfilter/conn.log b/testing/btest/Baseline/scripts.base.frameworks.netcontrol.packetfilter/conn.log index 9a673f80e2..614a90a0f7 100644 --- a/testing/btest/Baseline/scripts.base.frameworks.netcontrol.packetfilter/conn.log +++ b/testing/btest/Baseline/scripts.base.frameworks.netcontrol.packetfilter/conn.log @@ -3,13 +3,13 @@ #empty_field (empty) #unset_field - #path conn -#open 2016-07-13-16-15-38 +#open 2019-04-17-21-00-04 #fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p proto service duration orig_bytes resp_bytes conn_state local_orig local_resp missed_bytes history orig_pkts orig_ip_bytes resp_pkts resp_ip_bytes tunnel_parents #types time string addr port addr port enum string interval count count string bool bool count string count count count count set[string] 1254722767.492060 CHhAvVGS1DHFjwGM9 10.10.1.4 56166 10.10.1.1 53 udp dns 0.034025 34 100 SF - - 0 Dd 1 62 1 128 - 1254722776.690444 C4J4Th3PJpwUYZZ6gc 10.10.1.20 138 10.10.1.255 138 udp - - - - S0 - - 0 D 1 229 0 0 - 1254722767.529046 ClEkJM2Vm5giqnMf4h 10.10.1.4 1470 74.53.140.153 25 tcp - 0.346950 0 0 S1 - - 0 Sh 1 48 1 48 - 1437831776.764391 CtPZjS20MLrsMUOJi2 192.168.133.100 49285 66.196.121.26 5050 tcp - 0.343008 41 0 OTH - - 0 Da 1 93 1 52 - -1437831787.856895 CUM0KZ3MLUfNB0cl11 192.168.133.100 49648 192.168.133.102 25 tcp - 0.048043 162 154 S1 - - 154 ShDA 3 192 1 60 - +1437831787.856895 CUM0KZ3MLUfNB0cl11 192.168.133.100 49648 192.168.133.102 25 tcp - 0.048043 162 154 S1 - - 154 ShDgA 3 192 1 60 - 1437831798.533765 CmES5u32sYpV7JYN 192.168.133.100 49336 74.125.71.189 443 tcp - - - - OTH - - 0 A 1 52 0 0 - -#close 2016-07-13-16-15-38 +#close 2019-04-17-21-00-04 diff --git a/testing/btest/Baseline/scripts.base.protocols.ftp.cwd-navigation/conn.log b/testing/btest/Baseline/scripts.base.protocols.ftp.cwd-navigation/conn.log index 8990518008..2559f88db2 100644 --- a/testing/btest/Baseline/scripts.base.protocols.ftp.cwd-navigation/conn.log +++ b/testing/btest/Baseline/scripts.base.protocols.ftp.cwd-navigation/conn.log @@ -3,8 +3,8 @@ #empty_field (empty) #unset_field - #path conn -#open 2016-07-13-16-16-15 +#open 2019-04-17-21-00-49 #fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p proto service duration orig_bytes resp_bytes conn_state local_orig local_resp missed_bytes history orig_pkts orig_ip_bytes resp_pkts resp_ip_bytes tunnel_parents #types time string addr port addr port enum string interval count count string bool bool count string count count count count set[string] -1464385864.999633 CHhAvVGS1DHFjwGM9 10.3.22.91 58218 10.167.25.101 21 tcp ftp 600.931043 41420 159830 S1 - - 233 ShAdDa 4139 206914 4178 326799 - -#close 2016-07-13-16-16-15 +1464385864.999633 CHhAvVGS1DHFjwGM9 10.3.22.91 58218 10.167.25.101 21 tcp ftp 600.931043 41420 159830 S1 - - 233 ShAdDaGg 4139 206914 4178 326799 - +#close 2019-04-17-21-00-50 diff --git a/testing/btest/Baseline/scripts.base.protocols.ftp.cwd-navigation/ftp.log b/testing/btest/Baseline/scripts.base.protocols.ftp.cwd-navigation/ftp.log index 4516886e52..8a2d00a6c7 100644 --- a/testing/btest/Baseline/scripts.base.protocols.ftp.cwd-navigation/ftp.log +++ b/testing/btest/Baseline/scripts.base.protocols.ftp.cwd-navigation/ftp.log @@ -3,7 +3,7 @@ #empty_field (empty) #unset_field - #path ftp -#open 2016-07-13-16-16-15 +#open 2019-04-17-21-00-48 #fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p user password command arg mime_type file_size reply_code reply_msg data_channel.passive data_channel.orig_h data_channel.resp_h data_channel.resp_p fuid #types time string addr port addr port string string string string string count count string bool addr addr port string 1464385865.669674 CHhAvVGS1DHFjwGM9 10.3.22.91 58218 10.167.25.101 21 anonymous anonymous@ PASV - - - 227 Entering Passive Mode (205,167,25,101,243,251). T 10.3.22.91 205.167.25.101 62459 - @@ -1381,4 +1381,4 @@ 1464386464.737901 CHhAvVGS1DHFjwGM9 10.3.22.91 58218 10.167.25.101 21 anonymous anonymous@ RETR ftp://10.167.25.101/./pub/data/1993/722024-99999-1993.gz - 30171 226 Transfer complete - - - - - 1464386465.294490 CHhAvVGS1DHFjwGM9 10.3.22.91 58218 10.167.25.101 21 anonymous anonymous@ PASV - - - 227 Entering Passive Mode (205,167,25,101,251,88). T 10.3.22.91 205.167.25.101 64344 - 1464386465.471708 CHhAvVGS1DHFjwGM9 10.3.22.91 58218 10.167.25.101 21 anonymous anonymous@ RETR ftp://10.167.25.101/./pub/data/1994/722024-99999-1994.gz - 29736 226 Transfer complete - - - - - -#close 2016-07-13-16-16-15 +#close 2019-04-17-21-00-50