diff --git a/src/analyzer/protocol/tcp/TCP_Reassembler.cc b/src/analyzer/protocol/tcp/TCP_Reassembler.cc index 053e8c8f60..8b4ab0a93e 100644 --- a/src/analyzer/protocol/tcp/TCP_Reassembler.cc +++ b/src/analyzer/protocol/tcp/TCP_Reassembler.cc @@ -117,6 +117,45 @@ void TCP_Reassembler::SetContentsFile(BroFile* f) record_contents_file = f; } +static inline bool established(const TCP_Endpoint* a, const TCP_Endpoint* b) + { + return a->state == TCP_ENDPOINT_ESTABLISHED && + b->state == TCP_ENDPOINT_ESTABLISHED; + } + +static inline bool report_gap(const TCP_Endpoint* a, const TCP_Endpoint* b) + { + return content_gap && + ( BifConst::report_gaps_for_partial || established(a, b) ); + } + +void TCP_Reassembler::Gap(uint64 seq, uint64 len) + { + // Only report on content gaps for connections that + // are in a cleanly established state. In other + // states, these can arise falsely due to things + // like sequence number mismatches in RSTs, or + // unseen previous packets in partial connections. + // The one opportunity we lose here is on clean FIN + // handshakes, but Oh Well. + + if ( report_gap(endp, endp->peer) ) + { + val_list* vl = new val_list; + vl->append(dst_analyzer->BuildConnVal()); + vl->append(new Val(IsOrig(), TYPE_BOOL)); + vl->append(new Val(seq, TYPE_COUNT)); + vl->append(new Val(len, TYPE_COUNT)); + dst_analyzer->ConnectionEvent(content_gap, vl); + } + + if ( type == Direct ) + dst_analyzer->NextUndelivered(seq, len, IsOrig()); + else + dst_analyzer->ForwardUndelivered(seq, len, IsOrig()); + + had_gap = true; + } void TCP_Reassembler::Undelivered(uint64 up_to_seq) { @@ -189,48 +228,33 @@ void TCP_Reassembler::Undelivered(uint64 up_to_seq) if ( ! skip_deliveries ) { - // This can happen because we're processing a trace - // that's been filtered. For example, if it's just - // SYN/FIN data, then there can be data in the FIN - // packet, but it's undelievered because it's out of - // sequence. - - uint64 seq = last_reassem_seq; - uint64 len = up_to_seq - last_reassem_seq; - - // Only report on content gaps for connections that - // are in a cleanly established state. In other - // states, these can arise falsely due to things - // like sequence number mismatches in RSTs, or - // unseen previous packets in partial connections. - // The one opportunity we lose here is on clean FIN - // handshakes, but Oh Well. - - if ( content_gap && - (BifConst::report_gaps_for_partial || - (endpoint->state == TCP_ENDPOINT_ESTABLISHED && - peer->state == TCP_ENDPOINT_ESTABLISHED ) ) ) + DataBlock* b = blocks; + // If we have blocks that begin below up_to_seq, deliver them. + while ( b ) { - val_list* vl = new val_list; - vl->append(dst_analyzer->BuildConnVal()); - vl->append(new Val(IsOrig(), TYPE_BOOL)); - vl->append(new Val(seq, TYPE_COUNT)); - vl->append(new Val(len, TYPE_COUNT)); + if ( b->seq < last_reassem_seq ) + { + // Already delivered this block. + b = b->next; + continue; + } - dst_analyzer->ConnectionEvent(content_gap, vl); + if ( b->seq >= up_to_seq ) + // Block is beyond what we need to process at this point. + break; + + uint64 gap_at_seq = last_reassem_seq; + uint64 gap_len = b->seq - last_reassem_seq; + + Gap(gap_at_seq, gap_len); + last_reassem_seq += gap_len; + BlockInserted(b); + b = b->next; } - if ( type == Direct ) - dst_analyzer->NextUndelivered(last_reassem_seq, - len, IsOrig()); - else - { - dst_analyzer->ForwardUndelivered(last_reassem_seq, - len, IsOrig()); - } + if ( up_to_seq > last_reassem_seq ) + Gap(last_reassem_seq, up_to_seq - last_reassem_seq); } - - had_gap = true; } // We should record and match undelivered even if we are skipping diff --git a/src/analyzer/protocol/tcp/TCP_Reassembler.h b/src/analyzer/protocol/tcp/TCP_Reassembler.h index 3dfe75bf10..5d8badcef1 100644 --- a/src/analyzer/protocol/tcp/TCP_Reassembler.h +++ b/src/analyzer/protocol/tcp/TCP_Reassembler.h @@ -94,6 +94,7 @@ private: DECLARE_SERIAL(TCP_Reassembler); void Undelivered(uint64 up_to_seq); + void Gap(uint64 seq, uint64 len); void RecordToSeq(uint64 start_seq, uint64 stop_seq, BroFile* f); void RecordBlock(DataBlock* b, BroFile* f); diff --git a/testing/btest/Baseline/scripts.base.protocols.http.entity-gap2/entity_data b/testing/btest/Baseline/scripts.base.protocols.http.entity-gap2/entity_data new file mode 100644 index 0000000000..c6e5999e07 --- /dev/null +++ b/testing/btest/Baseline/scripts.base.protocols.http.entity-gap2/entity_data @@ -0,0 +1,4 @@ +^J0.26 | 2012-08-24 15:10:04 -0700^J^J * Fixing update-changes, which could pick the wrong control file. (Robin Sommer)^J^J * Fixing GPG signing script. (Robin Sommer)^J^J0.25 | 2012-08-01 13:55:46 -0500^J^J * Fix configure script to exit with non-zero status on error (Jon Siwek)^J^J0.24 | 2012-07-05 12:50:43 -0700^J^J * Raise minimum required CMake version to 2.6.3 (Jon Siwek)^J^J * Adding script to delete old fully-merged branches. (Robin Sommer)^J^J0.23-2 | 2012-01-25 13:24:01 -0800^J^J * Fix a bro-cut error message. (Daniel Thayer)^J^J0.23 | 2012-01-11 12:16:11 -0800^J^J * Tweaks to release scripts, plus a new one for signing files.^J (Robin Sommer)^J^J0.22 | 2012-01-10 16:45:19 -0800^J^J * Tweaks for OpenBSD support. (Jon Siwek)^J^J * bro-cut extensions and fixes. (Robin Sommer)^J ^J - If no field names are given on the command line, we now pass through^J all fields. Adresses #657.^J^J - Removing some GNUism from awk script. Addresses #653.^J^J - Added option for time output in UTC. Addresses #668.^J^J - Added output field separator option -F. Addresses #649.^J^J - Fixing option -c: only some header lines were passed through^J + rather than all. (Robin Sommer)^J^J * Fix parallel make portability. (Jon Siwek)^J^J0.21-9 | 2011-11-07 05:44:14 -0800^J^J * Fixing compiler warnings. Addresses #388. (Jon Siwek)^J^J0.21-2 | 2011-11-02 18:12:13 -0700^J^J * Fix for misnaming temp file in update-changes script. (Robin Sommer)^J^J0.21-1 | 2011-11-02 18:10:39 -0700^J^J * Little fix for make-release script, which could pick out the wrong^J tag. (Robin Sommer)^J^J0.21 | 2011-10-27 17:40:45 -0700^J^J * Fixing bro-cut's usage message and argument error handling. (Robin Sommer)^J^J * Bugfix in update-changes script. (Robin Sommer)^J^J * update-changes now ignores commits it did itself. (Robin Sommer)^J^J * Fix a bug in the update-changes script. (Robin Sommer)^J^J * bro-cut now always installs to $prefix/bin by `make install`. (Jon Siwek)^J^J * Options to adjust time format for bro-cut. (Robin Sommer)^J^J The default with -d is now ISO format. The new option "-D "^J specifies a custom strftime()-style format string. Alternatively,^J the environment variable BRO_CUT_TIMEFMT can set the format as^J well.^J^J * bro-cut now understands the field separator header. (Robin Sommer)^J^J * Renaming options -h/-H -> -c/-C, and doing some general cleanup.^J^J0.2 | 2011-10-25 19:53:57 -0700^J^J * Adding support for replacing version string in a setup.py. (Robin^J Sommer)^J^J * Change generated root cert DN indices format for RFC2253^J compliance. (Jon Siwek)^J^J * New tool devel-tool +<1448 byte gap> + thread library when necessary (e.g.^J PF_RING's libpcap) (Jon Siwek)^J^J * Install binaries with an RPATH (Jon Siwek)^J^J * Workaround for FreeBSD CMake port missing debug flags (Jon Siwek)^J^J * Rewrite of the update-changes script. (Robin Sommer)^J^J0.1-1 | 2011-06-14 21:12:41 -0700^J^J * Add a script for generating Mozilla's CA list for the SSL analyzer.^J (Seth Hall)^J^J0.1 | 2011-04-01 16:28:22 -0700^J^J * Converting build process to CMake. (Jon Siwek)^J^J * Removing cf/hf/ca-* from distribution. The README has a note where^J to find them now. (Robin Sommer)^J^J * General cleanup. (Robin Sommer)^J^J * Initial import of bro/aux from SVN r7088. (Jon Siwek)^J diff --git a/testing/btest/Baseline/scripts.base.protocols.http.entity-gap2/extract_files.file0 b/testing/btest/Baseline/scripts.base.protocols.http.entity-gap2/extract_files.file0 new file mode 100644 index 0000000000..8eb236cb55 Binary files /dev/null and b/testing/btest/Baseline/scripts.base.protocols.http.entity-gap2/extract_files.file0 differ diff --git a/testing/btest/Baseline/scripts.base.protocols.modbus.exception_handling/modbus.log b/testing/btest/Baseline/scripts.base.protocols.modbus.exception_handling/modbus.log index 407487756d..d51e336b0a 100644 --- a/testing/btest/Baseline/scripts.base.protocols.modbus.exception_handling/modbus.log +++ b/testing/btest/Baseline/scripts.base.protocols.modbus.exception_handling/modbus.log @@ -3,9 +3,13 @@ #empty_field (empty) #unset_field - #path modbus -#open 2013-08-26-19-04-11 +#open 2014-09-11-15-00-05 #fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p func exception #types time string addr port addr port string string 1153491909.414125 CXWv6p3arKYeMETxOg 192.168.66.235 2582 166.161.16.230 502 unknown-156 - +1153491911.997264 CXWv6p3arKYeMETxOg 192.168.66.235 2582 166.161.16.230 502 unknown-160 - 1153491913.013726 CXWv6p3arKYeMETxOg 192.168.66.235 2582 166.161.16.230 502 unknown-162 - -#close 2013-08-26-19-04-11 +1153491923.091742 CXWv6p3arKYeMETxOg 192.168.66.235 2582 166.161.16.230 502 unknown-175 - +1153491923.091742 CXWv6p3arKYeMETxOg 192.168.66.235 2582 166.161.16.230 502 unknown-179 - +1153491923.623460 CXWv6p3arKYeMETxOg 192.168.66.235 2582 166.161.16.230 502 unknown-165 - +#close 2014-09-11-15-00-05 diff --git a/testing/btest/Traces/http/entity_gap2.trace b/testing/btest/Traces/http/entity_gap2.trace new file mode 100644 index 0000000000..25a9302d06 Binary files /dev/null and b/testing/btest/Traces/http/entity_gap2.trace differ diff --git a/testing/btest/scripts/base/protocols/http/entity-gap2.bro b/testing/btest/scripts/base/protocols/http/entity-gap2.bro new file mode 100644 index 0000000000..c9ade93b72 --- /dev/null +++ b/testing/btest/scripts/base/protocols/http/entity-gap2.bro @@ -0,0 +1,24 @@ +# @TEST-EXEC: bro -r $TRACES/http/entity_gap2.trace %INPUT +# @TEST-EXEC: btest-diff entity_data +# @TEST-EXEC: btest-diff extract_files/file0 + +global f = open("entity_data"); +global fn = 0; + +event http_entity_data(c: connection, is_orig: bool, length: count, + data: string) + { + print f, data; + } + +event content_gap(c: connection, is_orig: bool, seq: count, length: count) + { + print f, fmt("<%d byte gap>", length); + } + +event file_new(f: fa_file) + { + Files::add_analyzer(f, Files::ANALYZER_EXTRACT, + [$extract_filename=fmt("file%d", fn)]); + ++fn; + }