Mirror/zeek
1
0
Fork 0
mirror of https://github.com/zeek/zeek.git synced 2025-10-04 15:48:19 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 2

Merge remote-tracking branch 'origin/topic/vern/content-gap-history'

Browse source 
* origin/topic/vern/content-gap-history:
  Refined state machine update placement to (1) properly deal with gaps capped by clean FIN handshakes, and (1) fix failure to detect split routing.
  added 'g' $history character for content gaps
This commit is contained in:
Jon Siwek 2019-04-22 12:38:06 -07:00
commit d5803d7047
21 changed files with 103 additions and 45 deletions
Download patch file Download diff file Expand all files Collapse all files

8
CHANGES
View file

@ -1,4 +1,12 @@
2.6-241 | 2019-04-22 12:38:06 -0700
* Add 'g' character to conn.log history field to flag content gaps (Vern Paxson, Corelight)
There's also a small change to TCP state machine that distrusts ACKs
appearing at the end of connections (in FIN or RST) such that they won't
count towards revealing a true content gap.
2.6-237 | 2019-04-19 12:00:37 -0700
* GH-236: Add zeek_script_loaded event, deprecate bro_script_loaded (Jon Siwek, Corelight)

9
NEWS
View file

@ -68,6 +68,10 @@ New Functionality
- Added a new event for weirdness found via file analysis: ``file_weird``.
- The conn.log "history" field supports a new character 'G' or 'g'
(capital for originator, lowercase responder) to indicate a content
gap in the TCP stream. These are recorded logarithmically.
Changed Functionality
---------------------
@ -166,6 +170,11 @@ Changed Functionality
- "unknown_gre_version_%d" -> unknown_gre_version
- "unknown_gre_protocol_%u16" -> unknown_gre_protocol
- The "missed_bytes" field of conn.log can be calculated slightly differently
in some cases: ACKs that reveal a content gap, but also come at
the end of a connection (in a FIN or RST) are considered unreliable
and aren't counted as true gaps.
Removed Functionality
---------------------

2
VERSION
View file

@ -1 +1 @@
2.6-237
2.6-241

2
doc

@ -1 +1 @@
Subproject commit 6857222c8c7050c96906757b468cbc1bffb7a807
Subproject commit 8e741019c26015066b1e59c224de3ae6b20ff76f

3
scripts/base/protocols/conn/main.zeek
View file

@ -107,6 +107,7 @@ export {
## f packet with FIN bit set
## r packet with RST bit set
## c packet with a bad checksum (applies to UDP too)
## g a content gap
## t packet with retransmitted payload
## w packet with a zero window advertisement
## i inconsistent packet (e.g. FIN+RST bits set)
@ -122,7 +123,7 @@ export {
## 's' can be recorded multiple times for either direction
## if the associated sequence number differs from the
## last-seen packet of the same flag type.
## 'c', 't' and 'w' are recorded in a logarithmic fashion:
## 'c', 'g', 't' and 'w' are recorded in a logarithmic fashion:
## the second instance represents that the event was seen
## (at least) 10 times; the third instance, 100 times; etc.
history: string &log &optional;

13
src/analyzer/protocol/tcp/TCP.cc
View file

@ -1350,11 +1350,9 @@ void TCP_Analyzer::DeliverPacket(int len, const u_char* data, bool is_orig,
Weird("TCP_ack_underflow_or_misorder");
}
else if ( ! flags.RST() )
// Don't trust ack's in RSt packets.
// Don't trust ack's in RST packets.
update_ack_seq(peer, ack_seq);
}
peer->AckReceived(rel_ack);
}
int32 delta_last = update_last_seq(endpoint, seq_one_past_segment, flags, len);
@ -1365,6 +1363,15 @@ void TCP_Analyzer::DeliverPacket(int len, const u_char* data, bool is_orig,
UpdateStateMachine(current_timestamp, endpoint, peer, base_seq, ack_seq,
len, delta_last, is_orig, flags, do_close, gen_event);
if ( flags.ACK() )
// We wait on doing this until we've updated the state
// machine so that if the ack reveals a content gap,
// we can tell whether it came at the very end of the
// connection (in a FIN or RST). Those gaps aren't
// reliable - especially those for RSTs - and we refrain
// from flagging them in the connection history.
peer->AckReceived(rel_ack);
if ( tcp_packet )
GeneratePacketEvent(rel_seq, rel_ack, data, len, caplen, is_orig,
flags);

12
src/analyzer/protocol/tcp/TCP_Endpoint.cc
View file

@ -32,8 +32,8 @@ TCP_Endpoint::TCP_Endpoint(TCP_Analyzer* arg_analyzer, int arg_is_orig)
tcp_analyzer = arg_analyzer;
is_orig = arg_is_orig;
chk_cnt = rxmt_cnt = win0_cnt = 0;
chk_thresh = rxmt_thresh = win0_thresh = 1;
gap_cnt = chk_cnt = rxmt_cnt = win0_cnt = 0;
gap_thresh = chk_thresh = rxmt_thresh = win0_thresh = 1;
hist_last_SYN = hist_last_FIN = hist_last_RST = 0;
@ -313,3 +313,11 @@ void TCP_Endpoint::ZeroWindow()
Conn()->HistoryThresholdEvent(tcp_multiple_zero_windows,
IsOrig(), t);
}
void TCP_Endpoint::Gap(uint64 seq, uint64 len)
{
uint32 t = gap_thresh;
if ( Conn()->ScaledHistoryEntry(IsOrig() ? 'G' : 'g',
gap_cnt, gap_thresh) )
Conn()->HistoryThresholdEvent(tcp_multiple_gap, IsOrig(), t);
}

4
src/analyzer/protocol/tcp/TCP_Endpoint.h
View file

@ -175,6 +175,9 @@ public:
// Called to inform endpoint that it has offered a zero window.
void ZeroWindow();
// Called to inform endpoint that a gap occurred.
void Gap(uint64 seq, uint64 len);
// Returns true if the data was used (and hence should be recorded
// in the save file), false otherwise.
int DataSent(double t, uint64 seq, int len, int caplen, const u_char* data,
@ -240,6 +243,7 @@ protected:
uint32 chk_cnt, chk_thresh;
uint32 rxmt_cnt, rxmt_thresh;
uint32 win0_cnt, win0_thresh;
uint32 gap_cnt, gap_thresh;
};
#define ENDIAN_UNKNOWN 0

25
src/analyzer/protocol/tcp/TCP_Reassembler.cc
View file

@ -112,27 +112,36 @@ void TCP_Reassembler::SetContentsFile(BroFile* f)
record_contents_file = f;
}
static inline bool established(const TCP_Endpoint* a, const TCP_Endpoint* b)
static inline bool is_clean(const TCP_Endpoint* a)
{
return a->state == TCP_ENDPOINT_ESTABLISHED &&
b->state == TCP_ENDPOINT_ESTABLISHED;
return a->state == TCP_ENDPOINT_ESTABLISHED ||
(a->state == TCP_ENDPOINT_CLOSED &&
a->prev_state == TCP_ENDPOINT_ESTABLISHED);
}
static inline bool established_or_cleanly_closing(const TCP_Endpoint* a,
const TCP_Endpoint* b)
{
return is_clean(a) && is_clean(b);
}
static inline bool report_gap(const TCP_Endpoint* a, const TCP_Endpoint* b)
{
return content_gap &&
( BifConst::report_gaps_for_partial || established(a, b) );
( BifConst::report_gaps_for_partial ||
established_or_cleanly_closing(a, b) );
}
void TCP_Reassembler::Gap(uint64 seq, uint64 len)
{
// Only report on content gaps for connections that
// are in a cleanly established state. In other
// states, these can arise falsely due to things
// are in a cleanly established or closing state. In
// other states, these can arise falsely due to things
// like sequence number mismatches in RSTs, or
// unseen previous packets in partial connections.
// The one opportunity we lose here is on clean FIN
// handshakes, but Oh Well.
if ( established_or_cleanly_closing(endp, endp->peer) )
endp->Gap(seq, len);
if ( report_gap(endp, endp->peer) )
{

18
src/analyzer/protocol/tcp/events.bif
View file

@ -300,7 +300,7 @@ event tcp_rexmit%(c: connection, is_orig: bool, seq: count, len: count, data_in_
## threshold: the threshold that was crossed
##
## .. bro:see:: udp_multiple_checksum_errors
## tcp_multiple_zero_windows tcp_multiple_retransmissions
## tcp_multiple_zero_windows tcp_multiple_retransmissions tcp_multiple_gap
event tcp_multiple_checksum_errors%(c: connection, is_orig: bool, threshold: count%);
## Generated if a TCP flow crosses a zero-window threshold, per
@ -312,7 +312,7 @@ event tcp_multiple_checksum_errors%(c: connection, is_orig: bool, threshold: cou
##
## threshold: the threshold that was crossed
##
## .. bro:see:: tcp_multiple_checksum_errors tcp_multiple_retransmissions
## .. bro:see:: tcp_multiple_checksum_errors tcp_multiple_retransmissions tcp_multiple_gap
event tcp_multiple_zero_windows%(c: connection, is_orig: bool, threshold: count%);
## Generated if a TCP flow crosses a retransmission threshold, per
@ -324,9 +324,21 @@ event tcp_multiple_zero_windows%(c: connection, is_orig: bool, threshold: count%
##
## threshold: the threshold that was crossed
##
## .. bro:see:: tcp_multiple_checksum_errors tcp_multiple_zero_windows
## .. bro:see:: tcp_multiple_checksum_errors tcp_multiple_zero_windows tcp_multiple_gap
event tcp_multiple_retransmissions%(c: connection, is_orig: bool, threshold: count%);
## Generated if a TCP flow crosses a gap threshold, per 'G'/'g' history
## reporting.
##
## c: The connection record for the TCP connection.
##
## is_orig: True if the event is raised for the originator side.
##
## threshold: the threshold that was crossed
##
## .. bro:see:: tcp_multiple_checksum_errors tcp_multiple_zero_windows tcp_multiple_retransmissions
event tcp_multiple_gap%(c: connection, is_orig: bool, threshold: count%);
## Generated when failing to write contents of a TCP stream to a file.
##
## c: The connection whose contents are being recorded.

6
testing/btest/Baseline/core.tcp.large-file-reassembly/conn.log
View file

@ -3,10 +3,10 @@
#empty_field (empty)
#unset_field -
#path conn
#open 2016-07-13-16-13-01
#open 2019-04-19-18-10-57
#fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p proto service duration orig_bytes resp_bytes conn_state local_orig local_resp missed_bytes history orig_pkts orig_ip_bytes resp_pkts resp_ip_bytes tunnel_parents
#types time string addr port addr port enum string interval count count string bool bool count string count count count count set[string]
1395939406.175845 ClEkJM2Vm5giqnMf4h 192.168.56.1 59763 192.168.56.101 63988 tcp ftp-data 0.001676 0 270 SF - - 0 ShAdfFa 5 272 4 486 -
1395939411.361078 C4J4Th3PJpwUYZZ6gc 192.168.56.1 59764 192.168.56.101 37150 tcp ftp-data 150.496065 0 5416666670 SF - - 4675708816 ShAdfFa 13 688 12 24454 -
1395939411.361078 C4J4Th3PJpwUYZZ6gc 192.168.56.1 59764 192.168.56.101 37150 tcp ftp-data 150.496065 0 5416666670 SF - - 5416642848 ShAdgfFa 13 688 12 24454 -
1395939399.984671 CHhAvVGS1DHFjwGM9 192.168.56.1 59762 192.168.56.101 21 tcp ftp 169.634297 104 1041 SF - - 0 ShAdDaFf 31 1728 18 1985 -
#close 2016-07-13-16-13-01
#close 2019-04-19-18-10-57

4
testing/btest/Baseline/core.tcp.large-file-reassembly/files.log
View file

@ -3,9 +3,9 @@
#empty_field (empty)
#unset_field -
#path files
#open 2017-01-25-07-03-11
#open 2019-04-17-20-41-29
#fields ts fuid tx_hosts rx_hosts conn_uids source depth analyzers mime_type filename duration local_orig is_orig seen_bytes total_bytes missing_bytes overflow_bytes timedout parent_fuid md5 sha1 sha256 extracted extracted_cutoff extracted_size
#types time string set[addr] set[addr] set[string] string count set[string] string string interval bool bool count count count count bool string string string string string bool count
1395939406.177079 FAb5m22Dhe2Zi95anf 192.168.56.101 192.168.56.1 ClEkJM2Vm5giqnMf4h FTP_DATA 0 DATA_EVENT text/plain - 0.000000 - F 270 - 0 0 F - - - - - - -
1395939411.364462 FhI0ao2FNTjabdfSBd 192.168.56.101 192.168.56.1 C4J4Th3PJpwUYZZ6gc FTP_DATA 0 DATA_EVENT text/plain - 150.490904 - F 23822 - 5416642848 0 F - - - - - - -
#close 2017-01-25-07-03-11
#close 2019-04-17-20-41-29

6
testing/btest/Baseline/core.tcp.miss-end-data/conn.log
View file

@ -3,8 +3,8 @@
#empty_field (empty)
#unset_field -
#path conn
#open 2016-07-13-16-13-02
#open 2019-04-19-18-11-06
#fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p proto service duration orig_bytes resp_bytes conn_state local_orig local_resp missed_bytes history orig_pkts orig_ip_bytes resp_pkts resp_ip_bytes tunnel_parents
#types time string addr port addr port enum string interval count count string bool bool count string count count count count set[string]
1331764471.664131 CHhAvVGS1DHFjwGM9 192.168.122.230 60648 77.238.160.184 80 tcp http 10.048360 538 2902 SF - - 2902 ShADafF 5 750 4 172 -
#close 2016-07-13-16-13-02
1331764471.664131 CHhAvVGS1DHFjwGM9 192.168.122.230 60648 77.238.160.184 80 tcp http 10.048360 538 2902 SF - - 2902 ShADafgF 5 750 4 172 -
#close 2019-04-19-18-11-07

6
testing/btest/Baseline/core.tcp.rxmit-history/conn-1.log
View file

@ -3,8 +3,8 @@
#empty_field (empty)
#unset_field -
#path conn
#open 2018-01-12-21-43-34
#open 2019-04-17-20-42-43
#fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p proto service duration orig_bytes resp_bytes conn_state local_orig local_resp missed_bytes history orig_pkts orig_ip_bytes resp_pkts resp_ip_bytes tunnel_parents
#types time string addr port addr port enum string interval count count string bool bool count string count count count count set[string]
1285862902.700271 CHhAvVGS1DHFjwGM9 10.0.88.85 50368 192.168.0.27 80 tcp - 60.991770 474 23783 RSTO - - 24257 ShADadtR 17 1250 22 28961 -
#close 2018-01-12-21-43-34
1285862902.700271 CHhAvVGS1DHFjwGM9 10.0.88.85 50368 192.168.0.27 80 tcp - 60.991770 474 23783 RSTO - - 24257 ShADaGdgtR 17 1250 22 28961 -
#close 2019-04-17-20-42-43

4
testing/btest/Baseline/core.tcp.rxmit-history/conn-2.log
View file

@ -3,7 +3,7 @@
#empty_field (empty)
#unset_field -
#path conn
#open 2018-01-12-21-43-35
#open 2019-04-17-20-42-44
#fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p proto service duration orig_bytes resp_bytes conn_state local_orig local_resp missed_bytes history orig_pkts orig_ip_bytes resp_pkts resp_ip_bytes tunnel_parents
#types time string addr port addr port enum string interval count count string bool bool count string count count count count set[string]
1300475167.096535 CHhAvVGS1DHFjwGM9 141.142.220.202 5353 224.0.0.251 5353 udp dns - - - S0 - - 0 D 1 73 0 0 -
@ -40,4 +40,4 @@
1300475168.859163 Ck51lg1bScffFj34Ri 141.142.220.118 49998 208.80.152.3 80 tcp http 0.215893 1130 734 S1 - - 0 ShADad 6 1450 4 950 -
1300475168.892936 CtxTCR2Yer0FR1tIBg 141.142.220.118 50000 208.80.152.3 80 tcp http 0.229603 1148 734 S1 - - 0 ShADad 6 1468 4 950 -
1300475168.895267 CLNN1k2QMum1aexUK7 141.142.220.118 50001 208.80.152.3 80 tcp http 0.227284 1178 734 S1 - - 0 ShADad 6 1498 4 950 -
#close 2018-01-12-21-43-35
#close 2019-04-17-20-42-44

6
testing/btest/Baseline/core.tunnels.gtp.outer_ip_frag/conn.log
View file

@ -3,9 +3,9 @@
#empty_field (empty)
#unset_field -
#path conn
#open 2016-07-13-16-13-10
#open 2019-04-19-18-10-49
#fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p proto service duration orig_bytes resp_bytes conn_state local_orig local_resp missed_bytes history orig_pkts orig_ip_bytes resp_pkts resp_ip_bytes tunnel_parents
#types time string addr port addr port enum string interval count count string bool bool count string count count count count set[string]
1333458850.364667 ClEkJM2Vm5giqnMf4h 10.131.47.185 1923 79.101.110.141 80 tcp http 0.069783 2100 56702 SF - - 0 ShADadfF 27 3204 41 52594 CHhAvVGS1DHFjwGM9
1333458850.364667 ClEkJM2Vm5giqnMf4h 10.131.47.185 1923 79.101.110.141 80 tcp http 0.069783 2100 56702 SF - - 5760 ShADadfgF 27 3204 41 52594 CHhAvVGS1DHFjwGM9
1333458850.364667 CHhAvVGS1DHFjwGM9 239.114.155.111 2152 63.94.149.181 2152 udp gtpv1 0.069813 3420 52922 SF - - 0 Dd 27 4176 41 54070 -
#close 2016-07-13-16-13-10
#close 2019-04-19-18-10-49

6
testing/btest/Baseline/scripts.base.frameworks.netcontrol.packetfilter/conn.log
View file

@ -3,13 +3,13 @@
#empty_field (empty)
#unset_field -
#path conn
#open 2016-07-13-16-15-38
#open 2019-04-17-21-00-04
#fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p proto service duration orig_bytes resp_bytes conn_state local_orig local_resp missed_bytes history orig_pkts orig_ip_bytes resp_pkts resp_ip_bytes tunnel_parents
#types time string addr port addr port enum string interval count count string bool bool count string count count count count set[string]
1254722767.492060 CHhAvVGS1DHFjwGM9 10.10.1.4 56166 10.10.1.1 53 udp dns 0.034025 34 100 SF - - 0 Dd 1 62 1 128 -
1254722776.690444 C4J4Th3PJpwUYZZ6gc 10.10.1.20 138 10.10.1.255 138 udp - - - - S0 - - 0 D 1 229 0 0 -
1254722767.529046 ClEkJM2Vm5giqnMf4h 10.10.1.4 1470 74.53.140.153 25 tcp - 0.346950 0 0 S1 - - 0 Sh 1 48 1 48 -
1437831776.764391 CtPZjS20MLrsMUOJi2 192.168.133.100 49285 66.196.121.26 5050 tcp - 0.343008 41 0 OTH - - 0 Da 1 93 1 52 -
1437831787.856895 CUM0KZ3MLUfNB0cl11 192.168.133.100 49648 192.168.133.102 25 tcp - 0.048043 162 154 S1 - - 154 ShDA 3 192 1 60 -
1437831787.856895 CUM0KZ3MLUfNB0cl11 192.168.133.100 49648 192.168.133.102 25 tcp - 0.048043 162 154 S1 - - 154 ShDgA 3 192 1 60 -
1437831798.533765 CmES5u32sYpV7JYN 192.168.133.100 49336 74.125.71.189 443 tcp - - - - OTH - - 0 A 1 52 0 0 -
#close 2016-07-13-16-15-38
#close 2019-04-17-21-00-04

6
testing/btest/Baseline/scripts.base.protocols.ftp.cwd-navigation/conn.log
View file

@ -3,8 +3,8 @@
#empty_field (empty)
#unset_field -
#path conn
#open 2016-07-13-16-16-15
#open 2019-04-17-21-00-49
#fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p proto service duration orig_bytes resp_bytes conn_state local_orig local_resp missed_bytes history orig_pkts orig_ip_bytes resp_pkts resp_ip_bytes tunnel_parents
#types time string addr port addr port enum string interval count count string bool bool count string count count count count set[string]
1464385864.999633 CHhAvVGS1DHFjwGM9 10.3.22.91 58218 10.167.25.101 21 tcp ftp 600.931043 41420 159830 S1 - - 233 ShAdDa 4139 206914 4178 326799 -
#close 2016-07-13-16-16-15
1464385864.999633 CHhAvVGS1DHFjwGM9 10.3.22.91 58218 10.167.25.101 21 tcp ftp 600.931043 41420 159830 S1 - - 233 ShAdDaGg 4139 206914 4178 326799 -
#close 2019-04-17-21-00-50

4
testing/btest/Baseline/scripts.base.protocols.ftp.cwd-navigation/ftp.log
View file

@ -3,7 +3,7 @@
#empty_field (empty)
#unset_field -
#path ftp
#open 2016-07-13-16-16-15
#open 2019-04-17-21-00-48
#fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p user password command arg mime_type file_size reply_code reply_msg data_channel.passive data_channel.orig_h data_channel.resp_h data_channel.resp_p fuid
#types time string addr port addr port string string string string string count count string bool addr addr port string
1464385865.669674 CHhAvVGS1DHFjwGM9 10.3.22.91 58218 10.167.25.101 21 anonymous anonymous@ PASV - - - 227 Entering Passive Mode (205,167,25,101,243,251). T 10.3.22.91 205.167.25.101 62459 -
@ -1381,4 +1381,4 @@
1464386464.737901 CHhAvVGS1DHFjwGM9 10.3.22.91 58218 10.167.25.101 21 anonymous anonymous@ RETR ftp://10.167.25.101/./pub/data/1993/722024-99999-1993.gz - 30171 226 Transfer complete - - - - -
1464386465.294490 CHhAvVGS1DHFjwGM9 10.3.22.91 58218 10.167.25.101 21 anonymous anonymous@ PASV - - - 227 Entering Passive Mode (205,167,25,101,251,88). T 10.3.22.91 205.167.25.101 64344 -
1464386465.471708 CHhAvVGS1DHFjwGM9 10.3.22.91 58218 10.167.25.101 21 anonymous anonymous@ RETR ftp://10.167.25.101/./pub/data/1994/722024-99999-1994.gz - 29736 226 Transfer complete - - - - -
#close 2016-07-13-16-16-15
#close 2019-04-17-21-00-50

2
testing/external/commit-hash.zeek-testing vendored
View file

@ -1 +1 @@
96f9f7976b98447831fcfa2146007ea9ddb98f74
1ab5538b8cdb0ef78616d665e02343321f269f3d

2
testing/external/commit-hash.zeek-testing-private vendored
View file

@ -1 +1 @@
fb5be2e139ab5c9840eb6b50e691eacc66f62165
f30a47f38dbde8431b5c32f4060da660d162371c