diff --git a/CHANGES b/CHANGES index db09fdd340..9872c8bd71 100644 --- a/CHANGES +++ b/CHANGES @@ -1,4 +1,22 @@ +2.3-688 | 2015-04-10 08:10:44 -0700 + + * Update SQLite to 3.8.8.3. + +2.3-687 | 2015-04-10 07:32:52 -0700 + + * Remove stale signature benchmarking code (-L command-line option). + (Jon Siwek) + + * BIT-844: fix UDP payload signatures to match packet-wise. (Jon + Siwek) + +2.3-682 | 2015-04-09 12:07:00 -0700 + + * Fixing input readers' component type. (Robin Sommer) + + * Tiny spelling correction. (Seth Hall) + 2.3-680 | 2015-04-06 16:02:43 -0500 * BIT-1371: remove CMake version check from binary package scripts. diff --git a/NEWS b/NEWS index 7129b293d5..4addcc519c 100644 --- a/NEWS +++ b/NEWS @@ -79,14 +79,17 @@ Changed Functionality - File analysis * Removed ``fa_file`` record's ``mime_type`` and ``mime_types`` - fields. The events ``file_mime_type`` and ``file_mime_types`` - have been added which contain the same information. The - ``mime_type`` field of ``Files::Info`` also still has this info. + fields. The event ``file_metadata_inferred`` has been added + which contain the same information. The ``mime_type`` field of + ``Files::Info`` also still has this info. * The earliest point that new mime type information is available is - in the ``file_mime_type`` event which comes after the ``file_new`` - and ``file_over_new_connection`` events. Scripts which inspected - mime type info within those events will need to be adapted. + in the ``file_metadata_inferred`` event which comes after the + ``file_new`` and ``file_over_new_connection`` events. Scripts + which inspected mime type info within those events will need to be + adapted. (Note: for users that worked w/ versions of Bro from git, + there was also an event called ``file_mime_type`` which is now + replaced be the ``file_metadata_inferred`` event). * Removed ``Files::add_analyzers_for_mime_type`` function. diff --git a/VERSION b/VERSION index 937b335f4d..9a0e965233 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -2.3-680 +2.3-688 diff --git a/aux/binpac b/aux/binpac index ab50e5115b..544330932e 160000 --- a/aux/binpac +++ b/aux/binpac @@ -1 +1 @@ -Subproject commit ab50e5115bc0d217552a63f15382e45ed608f5fc +Subproject commit 544330932e7cd4615d6d19f63907e8aa2acebb9e diff --git a/doc/frameworks/file_analysis_02.bro b/doc/frameworks/file_analysis_02.bro index 141b11fca6..b01a8464a6 100644 --- a/doc/frameworks/file_analysis_02.bro +++ b/doc/frameworks/file_analysis_02.bro @@ -1,7 +1,8 @@ -event file_mime_type(f: fa_file, mime_type: string) +event file_metadata_inferred(f: fa_file, meta: inferred_file_metadata) { + if ( ! meta?$mime_type ) return; print "new file", f$id; - if ( mime_type == "text/plain" ) + if ( meta$mime_type == "text/plain" ) Files::add_analyzer(f, Files::ANALYZER_MD5); } diff --git a/doc/httpmonitor/file_extraction.bro b/doc/httpmonitor/file_extraction.bro index 3860cb361e..b89f87705c 100644 --- a/doc/httpmonitor/file_extraction.bro +++ b/doc/httpmonitor/file_extraction.bro @@ -7,15 +7,18 @@ global mime_to_ext: table[string] of string = { ["text/html"] = "html", }; -event file_mime_type(f: fa_file, mime_type: string) +event file_metadata_inferred(f: fa_file, meta: inferred_file_metadata) { if ( f$source != "HTTP" ) return; - if ( mime_type !in mime_to_ext ) + if ( ! meta?$mime_type ) return; - local fname = fmt("%s-%s.%s", f$source, f$id, mime_to_ext[mime_type]); + if ( meta$mime_type !in mime_to_ext ) + return; + + local fname = fmt("%s-%s.%s", f$source, f$id, mime_to_ext[meta$mime_type]); print fmt("Extracting file %s", fname); Files::add_analyzer(f, Files::ANALYZER_EXTRACT, [$extract_filename=fname]); - } \ No newline at end of file + } diff --git a/scripts/base/frameworks/files/main.bro b/scripts/base/frameworks/files/main.bro index fa4df59cf3..273f45efdb 100644 --- a/scripts/base/frameworks/files/main.bro +++ b/scripts/base/frameworks/files/main.bro @@ -484,16 +484,19 @@ event file_over_new_connection(f: fa_file, c: connection, is_orig: bool) &priori add f$info$rx_hosts[f$is_orig ? cid$resp_h : cid$orig_h]; } -event file_mime_type(f: fa_file, mime_type: string) &priority=10 +event file_metadata_inferred(f: fa_file, meta: inferred_file_metadata) &priority=10 { set_info(f); - f$info$mime_type = mime_type; + if ( ! meta?$mime_type ) + return; + + f$info$mime_type = meta$mime_type; if ( analyze_by_mime_type_automatically && - mime_type in mime_type_to_analyzers ) + meta$mime_type in mime_type_to_analyzers ) { - local analyzers = mime_type_to_analyzers[mime_type]; + local analyzers = mime_type_to_analyzers[meta$mime_type]; for ( a in analyzers ) { add f$info$analyzers[Files::analyzer_name(a)]; diff --git a/scripts/base/init-bare.bro b/scripts/base/init-bare.bro index cfe845eb4f..fb3ccd6698 100644 --- a/scripts/base/init-bare.bro +++ b/scripts/base/init-bare.bro @@ -414,6 +414,14 @@ type fa_file: record { bof_buffer: string &optional; } &redef; +## Metadata that's been inferred about a particular file. +type inferred_file_metadata: record { + ## The strongest matching mime type if one was discovered. + mime_type: string &optional; + ## All matching mime types if any were discovered. + mime_types: mime_matches &optional; +}; + ## Fields of a SYN packet. ## ## .. bro:see:: connection_SYN_packet diff --git a/scripts/base/protocols/ftp/files.bro b/scripts/base/protocols/ftp/files.bro index 617b57348b..8c18d19869 100644 --- a/scripts/base/protocols/ftp/files.bro +++ b/scripts/base/protocols/ftp/files.bro @@ -63,10 +63,13 @@ event file_over_new_connection(f: fa_file, c: connection, is_orig: bool) &priori f$ftp = ftp; } -event file_mime_type(f: fa_file, mime_type: string) &priority=5 +event file_metadata_inferred(f: fa_file, meta: inferred_file_metadata) &priority=5 { if ( ! f?$ftp ) return; - f$ftp$mime_type = mime_type; + if ( ! meta?$mime_type ) + return; + + f$ftp$mime_type = meta$mime_type; } diff --git a/scripts/base/protocols/http/entities.bro b/scripts/base/protocols/http/entities.bro index 597637e63e..f363ec7810 100644 --- a/scripts/base/protocols/http/entities.bro +++ b/scripts/base/protocols/http/entities.bro @@ -93,24 +93,27 @@ event file_over_new_connection(f: fa_file, c: connection, is_orig: bool) &priori } } -event file_mime_type(f: fa_file, mime_type: string) &priority=5 +event file_metadata_inferred(f: fa_file, meta: inferred_file_metadata) &priority=5 { if ( ! f?$http || ! f?$is_orig ) return; + if ( ! meta?$mime_type ) + return; + if ( f$is_orig ) { if ( ! f$http?$orig_mime_types ) - f$http$orig_mime_types = string_vec(mime_type); + f$http$orig_mime_types = string_vec(meta$mime_type); else - f$http$orig_mime_types[|f$http$orig_mime_types|] = mime_type; + f$http$orig_mime_types[|f$http$orig_mime_types|] = meta$mime_type; } else { if ( ! f$http?$resp_mime_types ) - f$http$resp_mime_types = string_vec(mime_type); + f$http$resp_mime_types = string_vec(meta$mime_type); else - f$http$resp_mime_types[|f$http$resp_mime_types|] = mime_type; + f$http$resp_mime_types[|f$http$resp_mime_types|] = meta$mime_type; } } diff --git a/scripts/base/protocols/irc/files.bro b/scripts/base/protocols/irc/files.bro index 518775abb4..ea9bf1bdc2 100644 --- a/scripts/base/protocols/irc/files.bro +++ b/scripts/base/protocols/irc/files.bro @@ -42,8 +42,8 @@ event file_over_new_connection(f: fa_file, c: connection, is_orig: bool) &priori f$irc = irc; } -event file_mime_type(f: fa_file, mime_type: string) &priority=5 +event file_metadata_inferred(f: fa_file, meta: inferred_file_metadata) &priority=5 { - if ( f?$irc ) - f$irc$dcc_mime_type = mime_type; - } \ No newline at end of file + if ( f?$irc && meta?$mime_type ) + f$irc$dcc_mime_type = meta$mime_type; + } diff --git a/src/3rdparty b/src/3rdparty index f2e34d731e..d4c305df93 160000 --- a/src/3rdparty +++ b/src/3rdparty @@ -1 +1 @@ -Subproject commit f2e34d731ed29bb993fbb065846faa342a8c824f +Subproject commit d4c305df93d555ab468dbad4a5b69412bf44a833 diff --git a/src/NetVar.cc b/src/NetVar.cc index 7c66b55bc2..f7f6e12aac 100644 --- a/src/NetVar.cc +++ b/src/NetVar.cc @@ -10,6 +10,7 @@ RecordType* endpoint; RecordType* endpoint_stats; RecordType* connection_type; RecordType* fa_file_type; +RecordType* inferred_file_metadata_type; RecordType* icmp_conn; RecordType* icmp_context; RecordType* SYN_packet; @@ -316,6 +317,7 @@ void init_net_var() endpoint_stats = internal_type("endpoint_stats")->AsRecordType(); connection_type = internal_type("connection")->AsRecordType(); fa_file_type = internal_type("fa_file")->AsRecordType(); + inferred_file_metadata_type = internal_type("inferred_file_metadata")->AsRecordType(); icmp_conn = internal_type("icmp_conn")->AsRecordType(); icmp_context = internal_type("icmp_context")->AsRecordType(); signature_state = internal_type("signature_state")->AsRecordType(); diff --git a/src/NetVar.h b/src/NetVar.h index edd70d1ea6..2c5221f6a7 100644 --- a/src/NetVar.h +++ b/src/NetVar.h @@ -13,6 +13,7 @@ extern RecordType* endpoint; extern RecordType* endpoint_stats; extern RecordType* connection_type; extern RecordType* fa_file_type; +extern RecordType* inferred_file_metadata_type; extern RecordType* icmp_conn; extern RecordType* icmp_context; extern RecordType* signature_state; diff --git a/src/RE.cc b/src/RE.cc index 4855b0e39a..f52eff47eb 100644 --- a/src/RE.cc +++ b/src/RE.cc @@ -20,9 +20,6 @@ int case_insensitive = 0; extern int RE_parse(void); extern void RE_set_input(const char* str); -// If true, the set-wise matching always returns false - for benchmarking. -extern int rule_bench; - Specific_RE_Matcher::Specific_RE_Matcher(match_type arg_mt, int arg_multiline) : equiv_class(NUM_SYM) { @@ -279,9 +276,6 @@ inline void RE_Match_State::AddMatches(const AcceptingSet& as, bool RE_Match_State::Match(const u_char* bv, int n, bool bol, bool eol, bool clear) { - if ( rule_bench > 0 ) - return false; - if ( current_pos == -1 ) { // First call to Match(). diff --git a/src/RuleMatcher.cc b/src/RuleMatcher.cc index ca08388b10..967c4e4e65 100644 --- a/src/RuleMatcher.cc +++ b/src/RuleMatcher.cc @@ -577,9 +577,6 @@ RuleFileMagicState* RuleMatcher::InitFileMagic() const { RuleFileMagicState* state = new RuleFileMagicState(); - if ( rule_bench == 3 ) - return state; - loop_over_list(root->psets[Rule::FILE_MAGIC], i) { RuleHdrTest::PatternSet* set = root->psets[Rule::FILE_MAGIC][i]; @@ -630,9 +627,6 @@ RuleMatcher::MIME_Matches* RuleMatcher::Match(RuleFileMagicState* state, return rval; } - if ( rule_bench >= 2 ) - return rval; - #ifdef DEBUG if ( debug_logger.IsEnabled(DBG_RULES) ) { @@ -712,9 +706,6 @@ RuleEndpointState* RuleMatcher::InitEndpoint(analyzer::Analyzer* analyzer, RuleEndpointState* state = new RuleEndpointState(analyzer, from_orig, opposite, pia); - if ( rule_bench == 3 ) - return state; - rule_hdr_test_list tests; tests.append(root); @@ -837,9 +828,6 @@ void RuleMatcher::Match(RuleEndpointState* state, Rule::PatternType type, // for 'accepted' (that depends on the average number of matching // patterns). - if ( rule_bench >= 2 ) - return; - bool newmatch = false; #ifdef DEBUG @@ -956,9 +944,6 @@ void RuleMatcher::Match(RuleEndpointState* state, Rule::PatternType type, void RuleMatcher::FinishEndpoint(RuleEndpointState* state) { - if ( rule_bench == 3 ) - return; - // Send EOL to payload matchers. Match(state, Rule::PAYLOAD, (const u_char *) "", 0, false, true, false); @@ -1110,15 +1095,9 @@ void RuleMatcher::ExecRule(Rule* rule, RuleEndpointState* state, bool eos) void RuleMatcher::ClearEndpointState(RuleEndpointState* state) { - if ( rule_bench == 3 ) - return; + state->payload_size = -1; ExecPureRules(state, 1); - state->payload_size = -1; - state->matched_by_patterns.clear(); - loop_over_list(state->matched_text, i) - delete state->matched_text[i]; - state->matched_text.clear(); loop_over_list(state->matchers, j) state->matchers[j]->state->Clear(); @@ -1126,9 +1105,6 @@ void RuleMatcher::ClearEndpointState(RuleEndpointState* state) void RuleMatcher::ClearFileMagicState(RuleFileMagicState* state) const { - if ( rule_bench == 3 ) - return; - loop_over_list(state->matchers, j) state->matchers[j]->state->Clear(); } @@ -1496,8 +1472,12 @@ void RuleMatcherState::ClearMatchState(bool orig) if ( ! rule_matcher ) return; - if ( orig_match_state ) - rule_matcher->ClearEndpointState(orig_match_state); - if ( resp_match_state ) + if ( orig ) + { + if ( orig_match_state ) + rule_matcher->ClearEndpointState(orig_match_state); + } + + else if ( resp_match_state ) rule_matcher->ClearEndpointState(resp_match_state); } diff --git a/src/RuleMatcher.h b/src/RuleMatcher.h index da2838cb6d..6ffc971db1 100644 --- a/src/RuleMatcher.h +++ b/src/RuleMatcher.h @@ -22,8 +22,6 @@ //#define MATCHER_PRINT_STATS -extern int rule_bench; - // Parser interface: extern void rules_error(const char* msg); diff --git a/src/analyzer/protocol/pia/PIA.cc b/src/analyzer/protocol/pia/PIA.cc index 69a0c5d312..1adeb54a2d 100644 --- a/src/analyzer/protocol/pia/PIA.cc +++ b/src/analyzer/protocol/pia/PIA.cc @@ -81,7 +81,7 @@ void PIA::PIA_Done() } void PIA::PIA_DeliverPacket(int len, const u_char* data, bool is_orig, uint64 seq, - const IP_Hdr* ip, int caplen) + const IP_Hdr* ip, int caplen, bool clear_state) { if ( pkt_buffer.state == SKIPPING ) return; @@ -108,6 +108,9 @@ void PIA::PIA_DeliverPacket(int len, const u_char* data, bool is_orig, uint64 se // FIXME: I'm not sure why it does not work with eol=true... DoMatch(data, len, is_orig, true, false, false, ip); + if ( clear_state ) + RuleMatcherState::ClearMatchState(is_orig); + pkt_buffer.state = new_state; current_packet.data = 0; diff --git a/src/analyzer/protocol/pia/PIA.h b/src/analyzer/protocol/pia/PIA.h index d6e07f68c3..85683289a9 100644 --- a/src/analyzer/protocol/pia/PIA.h +++ b/src/analyzer/protocol/pia/PIA.h @@ -42,7 +42,7 @@ public: protected: void PIA_Done(); void PIA_DeliverPacket(int len, const u_char* data, bool is_orig, - uint64 seq, const IP_Hdr* ip, int caplen); + uint64 seq, const IP_Hdr* ip, int caplen, bool clear_state); enum State { INIT, BUFFERING, MATCHING_ONLY, SKIPPING } state; @@ -109,7 +109,7 @@ protected: uint64 seq, const IP_Hdr* ip, int caplen) { Analyzer::DeliverPacket(len, data, is_orig, seq, ip, caplen); - PIA_DeliverPacket(len, data, is_orig, seq, ip, caplen); + PIA_DeliverPacket(len, data, is_orig, seq, ip, caplen, true); } virtual void ActivateAnalyzer(analyzer::Tag tag, const Rule* rule); @@ -154,7 +154,7 @@ protected: uint64 seq, const IP_Hdr* ip, int caplen) { Analyzer::DeliverPacket(len, data, is_orig, seq, ip, caplen); - PIA_DeliverPacket(len, data, is_orig, seq, ip, caplen); + PIA_DeliverPacket(len, data, is_orig, seq, ip, caplen, false); } virtual void DeliverStream(int len, const u_char* data, bool is_orig); diff --git a/src/event.bif b/src/event.bif index dd941b6736..871ddd2d25 100644 --- a/src/event.bif +++ b/src/event.bif @@ -905,8 +905,8 @@ event get_file_handle%(tag: Analyzer::Tag, c: connection, is_orig: bool%); ## ## f: The file. ## -## .. bro:see:: file_over_new_connection file_timeout file_gap file_mime_type -## file_state_remove +## .. bro:see:: file_over_new_connection file_timeout file_gap +## file_metadata_inferred file_state_remove event file_new%(f: fa_file%); ## Indicates that a file has been seen being transferred over a connection @@ -918,39 +918,30 @@ event file_new%(f: fa_file%); ## ## is_orig: true if the originator of *c* is the one sending the file. ## -## .. bro:see:: file_new file_timeout file_gap file_mime_type +## .. bro:see:: file_new file_timeout file_gap file_metadata_inferred ## file_state_remove event file_over_new_connection%(f: fa_file, c: connection, is_orig: bool%); -## Provide the most likely matching MIME type for this file. The analysis -## can be augmented at this time via :bro:see:`Files::add_analyzer`. +## Provide all metadata that has been inferred about a particular file +## from inspection of the initial content that been seen at the beginning +## of the file. The analysis can be augmented at this time via +## :bro:see:`Files::add_analyzer`. ## ## f: The file. ## -## mime_type: The mime type that was discovered. +## meta: Metadata that's been discovered about the file. ## -## .. bro:see:: file_over_new_connection file_timeout file_gap file_mime_type -## file_mime_types file_state_remove -event file_mime_type%(f: fa_file, mime_type: string%); - -## Provide all matching MIME types for this file. The analysis can be -## augmented at this time via :bro:see:`Files::add_analyzer`. -## -## f: The file. -## -## mime_types: The mime types that were discovered. -## -## .. bro:see:: file_over_new_connection file_timeout file_gap file_mime_type -## file_mime_types file_state_remove -event file_mime_types%(f: fa_file, mime_types: mime_matches%); +## .. bro:see:: file_over_new_connection file_timeout file_gap +## file_state_remove +event file_metadata_inferred%(f: fa_file, meta: inferred_file_metadata%); ## Indicates that file analysis has timed out because no activity was seen ## for the file in a while. ## ## f: The file. ## -## .. bro:see:: file_new file_over_new_connection file_gap file_mime_type -## file_mime_types file_state_remove default_file_timeout_interval +## .. bro:see:: file_new file_over_new_connection file_gap +## file_metadata_inferred file_state_remove default_file_timeout_interval ## Files::set_timeout_interval event file_timeout%(f: fa_file%); @@ -962,8 +953,8 @@ event file_timeout%(f: fa_file%); ## ## len: The number of missing bytes. ## -## .. bro:see:: file_new file_over_new_connection file_timeout file_mime_type -## file_mime_types file_state_remove file_reassembly_overflow +## .. bro:see:: file_new file_over_new_connection file_timeout +## file_metadata_inferred file_state_remove file_reassembly_overflow event file_gap%(f: fa_file, offset: count, len: count%); ## Indicates that the file had an overflow of the reassembly buffer. @@ -978,10 +969,11 @@ event file_gap%(f: fa_file, offset: count, len: count%); ## file data and get back under the reassembly buffer size limit. ## This value will also be represented as a gap. ## -## .. bro:see:: file_new file_over_new_connection file_timeout file_mime_type -## file_mime_types file_state_remove file_gap Files::enable_reassembler -## Files::reassembly_buffer_size Files::enable_reassembly -## Files::disable_reassembly Files::set_reassembly_buffer_size +## .. bro:see:: file_new file_over_new_connection file_timeout +## file_metadata_inferred file_state_remove file_gap +## Files::enable_reassembler Files::reassembly_buffer_size +## Files::enable_reassembly Files::disable_reassembly +## Files::set_reassembly_buffer_size event file_reassembly_overflow%(f: fa_file, offset: count, skipped: count%); ## This event is generated each time file analysis is ending for a given file. @@ -989,7 +981,7 @@ event file_reassembly_overflow%(f: fa_file, offset: count, skipped: count%); ## f: The file. ## ## .. bro:see:: file_new file_over_new_connection file_timeout file_gap -## file_mime_type file_mime_types +## file_metadata_inferred event file_state_remove%(f: fa_file%); ## Generated when an internal DNS lookup produces the same result as last time. diff --git a/src/file_analysis/File.cc b/src/file_analysis/File.cc index 2b0d76f8d6..28683e9230 100644 --- a/src/file_analysis/File.cc +++ b/src/file_analysis/File.cc @@ -53,31 +53,35 @@ int File::overflow_bytes_idx = -1; int File::timeout_interval_idx = -1; int File::bof_buffer_size_idx = -1; int File::bof_buffer_idx = -1; +int File::meta_mime_type_idx = -1; +int File::meta_mime_types_idx = -1; void File::StaticInit() { if ( id_idx != -1 ) return; - id_idx = Idx("id"); - parent_id_idx = Idx("parent_id"); - source_idx = Idx("source"); - is_orig_idx = Idx("is_orig"); - conns_idx = Idx("conns"); - last_active_idx = Idx("last_active"); - seen_bytes_idx = Idx("seen_bytes"); - total_bytes_idx = Idx("total_bytes"); - missing_bytes_idx = Idx("missing_bytes"); - overflow_bytes_idx = Idx("overflow_bytes"); - timeout_interval_idx = Idx("timeout_interval"); - bof_buffer_size_idx = Idx("bof_buffer_size"); - bof_buffer_idx = Idx("bof_buffer"); + id_idx = Idx("id", fa_file_type); + parent_id_idx = Idx("parent_id", fa_file_type); + source_idx = Idx("source", fa_file_type); + is_orig_idx = Idx("is_orig", fa_file_type); + conns_idx = Idx("conns", fa_file_type); + last_active_idx = Idx("last_active", fa_file_type); + seen_bytes_idx = Idx("seen_bytes", fa_file_type); + total_bytes_idx = Idx("total_bytes", fa_file_type); + missing_bytes_idx = Idx("missing_bytes", fa_file_type); + overflow_bytes_idx = Idx("overflow_bytes", fa_file_type); + timeout_interval_idx = Idx("timeout_interval", fa_file_type); + bof_buffer_size_idx = Idx("bof_buffer_size", fa_file_type); + bof_buffer_idx = Idx("bof_buffer", fa_file_type); + meta_mime_type_idx = Idx("mime_type", inferred_file_metadata_type); + meta_mime_types_idx = Idx("mime_types", inferred_file_metadata_type); } File::File(const string& file_id, const string& source_name, Connection* conn, analyzer::Tag tag, bool is_orig) : id(file_id), val(0), file_reassembler(0), stream_offset(0), - reassembly_max_buffer(0), did_mime_type(false), + reassembly_max_buffer(0), did_metadata_inference(false), reassembly_enabled(false), postpone_timeout(false), done(false), analyzers(this) { @@ -169,11 +173,13 @@ double File::LookupFieldDefaultInterval(int idx) const return rval; } -int File::Idx(const string& field) +int File::Idx(const string& field, const RecordType* type) { - int rval = fa_file_type->FieldOffset(field.c_str()); + int rval = type->FieldOffset(field.c_str()); + if ( rval < 0 ) - reporter->InternalError("Unknown fa_file field: %s", field.c_str()); + reporter->InternalError("Unknown %s field: %s", type->GetName().c_str(), + field.c_str()); return rval; } @@ -281,50 +287,46 @@ void File::SetReassemblyBuffer(uint64 max) reassembly_max_buffer = max; } -bool File::DetectMIME() +void File::InferMetadata() { - did_mime_type = true; + did_metadata_inference = true; Val* bof_buffer_val = val->Lookup(bof_buffer_idx); if ( ! bof_buffer_val ) { if ( bof_buffer.size == 0 ) - return false; + return; BroString* bs = concatenate(bof_buffer.chunks); bof_buffer_val = new StringVal(bs); val->Assign(bof_buffer_idx, bof_buffer_val); } + if ( ! FileEventAvailable(file_metadata_inferred) ) + return; + RuleMatcher::MIME_Matches matches; const u_char* data = bof_buffer_val->AsString()->Bytes(); uint64 len = bof_buffer_val->AsString()->Len(); len = min(len, LookupFieldDefaultCount(bof_buffer_size_idx)); file_mgr->DetectMIME(data, len, &matches); - if ( matches.empty() ) + val_list* vl = new val_list(); + vl->append(val->Ref()); + RecordVal* meta = new RecordVal(inferred_file_metadata_type); + vl->append(meta); + + if ( ! matches.empty() ) { - return false; + meta->Assign(meta_mime_type_idx, + new StringVal(*(matches.begin()->second.begin()))); + meta->Assign(meta_mime_types_idx, + file_analysis::GenMIMEMatchesVal(matches)); } - if ( FileEventAvailable(file_mime_type) ) - { - val_list* vl = new val_list(); - vl->append(val->Ref()); - vl->append(new StringVal(*(matches.begin()->second.begin()))); - FileEvent(file_mime_type, vl); - } - - if ( FileEventAvailable(file_mime_types) ) - { - val_list* vl = new val_list(); - vl->append(val->Ref()); - vl->append(file_analysis::GenMIMEMatchesVal(matches)); - FileEvent(file_mime_types, vl); - } - - return true; + FileEvent(file_metadata_inferred, vl); + return; } bool File::BufferBOF(const u_char* data, uint64 len) @@ -357,9 +359,9 @@ void File::DeliverStream(const u_char* data, uint64 len) // Buffer enough data for the BOF buffer BufferBOF(data, len); - if ( ! did_mime_type && bof_buffer.full && + if ( ! did_metadata_inference && bof_buffer.full && LookupFieldDefaultCount(missing_bytes_idx) == 0 ) - DetectMIME(); + InferMetadata(); DBG_LOG(DBG_FILE_ANALYSIS, "[%s] %" PRIu64 " stream bytes in at offset %" PRIu64 "; %s [%s%s]", @@ -589,7 +591,7 @@ void File::FileEvent(EventHandlerPtr h, val_list* vl) mgr.QueueEvent(h, vl); if ( h == file_new || h == file_over_new_connection || - h == file_mime_type || h == file_mime_types || + h == file_metadata_inferred || h == file_timeout || h == file_extraction_limit ) { // immediate feedback is required for these events. diff --git a/src/file_analysis/File.h b/src/file_analysis/File.h index 645f7d5111..6ad90e986b 100644 --- a/src/file_analysis/File.h +++ b/src/file_analysis/File.h @@ -230,12 +230,11 @@ protected: bool BufferBOF(const u_char* data, uint64 len); /** - * Does mime type detection via file magic signatures and assigns - * strongest matching mime type (if available) to \c mime_type - * field in #val. It uses the data in the BOF buffer. - * @return whether a mime type match was found. + * Does metadata inference (e.g. mime type detection via file + * magic signatures) using data in the BOF (beginning-of-file) buffer + * and raises an event with the metadata. */ - bool DetectMIME(); + void InferMetadata(); /** * Enables reassembly on the file. @@ -266,10 +265,11 @@ protected: /** * Lookup a record field index/offset by name. - * @param field_name the name of the \c fa_file record field. + * @param field_name the name of the record field. + * @param type the record type for which the field will be looked up. * @return the field offset in #val record corresponding to \a field_name. */ - static int Idx(const string& field_name); + static int Idx(const string& field_name, const RecordType* type); /** * Initializes static member. @@ -282,7 +282,7 @@ protected: FileReassembler* file_reassembler; /**< A reassembler for the file if it's needed. */ uint64 stream_offset; /**< The offset of the file which has been forwarded. */ uint64 reassembly_max_buffer; /**< Maximum allowed buffer for reassembly. */ - bool did_mime_type; /**< Whether the mime type ident has already been attempted. */ + bool did_metadata_inference; /**< Whether the metadata inference has already been attempted. */ bool reassembly_enabled; /**< Whether file stream reassembly is needed. */ bool postpone_timeout; /**< Whether postponing timeout is requested. */ bool done; /**< If this object is about to be deleted. */ @@ -313,6 +313,9 @@ protected: static int bof_buffer_idx; static int mime_type_idx; static int mime_types_idx; + + static int meta_mime_type_idx; + static int meta_mime_types_idx; }; } // namespace file_analysis diff --git a/src/input/Component.cc b/src/input/Component.cc index fd70c76216..b7c5d5e30b 100644 --- a/src/input/Component.cc +++ b/src/input/Component.cc @@ -9,7 +9,7 @@ using namespace input; Component::Component(const std::string& name, factory_callback arg_factory) - : plugin::Component(plugin::component::WRITER, name) + : plugin::Component(plugin::component::READER, name) { factory = arg_factory; diff --git a/src/main.cc b/src/main.cc index 24c19c19d9..d7259a7246 100644 --- a/src/main.cc +++ b/src/main.cc @@ -117,7 +117,6 @@ SampleLogger* sample_logger = 0; int signal_val = 0; int optimize = 0; int do_notice_analysis = 0; -int rule_bench = 0; extern char version[]; char* command_line_policy = 0; vector params; @@ -195,7 +194,6 @@ void usage() fprintf(stderr, " -F|--force-dns | force DNS\n"); fprintf(stderr, " -I|--print-id | print out given ID\n"); fprintf(stderr, " -K|--md5-hashkey | set key for MD5-keyed hashing\n"); - fprintf(stderr, " -L|--rule-benchmark | benchmark for rules\n"); fprintf(stderr, " -N|--print-plugins | print available plugins and exit (-NN for verbose)\n"); fprintf(stderr, " -O|--optimize | optimize policy script\n"); fprintf(stderr, " -P|--prime-dns | prime DNS\n"); @@ -503,7 +501,6 @@ int main(int argc, char** argv) {"save-seeds", required_argument, 0, 'H'}, {"set-seed", required_argument, 0, 'J'}, {"md5-hashkey", required_argument, 0, 'K'}, - {"rule-benchmark", no_argument, 0, 'L'}, {"print-plugins", no_argument, 0, 'N'}, {"optimize", no_argument, 0, 'O'}, {"prime-dns", no_argument, 0, 'P'}, @@ -668,10 +665,6 @@ int main(int argc, char** argv) hmac_key_set = 1; break; - case 'L': - ++rule_bench; - break; - case 'N': ++print_plugins; break; diff --git a/testing/btest/Baseline/doc.sphinx.include-doc_frameworks_file_analysis_02_bro/output b/testing/btest/Baseline/doc.sphinx.include-doc_frameworks_file_analysis_02_bro/output index 5e86c8d685..f8ca8e9d1a 100644 --- a/testing/btest/Baseline/doc.sphinx.include-doc_frameworks_file_analysis_02_bro/output +++ b/testing/btest/Baseline/doc.sphinx.include-doc_frameworks_file_analysis_02_bro/output @@ -2,10 +2,11 @@ file_analysis_02.bro -event file_mime_type(f: fa_file, mime_type: string) +event file_metadata_inferred(f: fa_file, meta: inferred_file_metadata) { + if ( ! meta?$mime_type ) return; print "new file", f$id; - if ( mime_type == "text/plain" ) + if ( meta$mime_type == "text/plain" ) Files::add_analyzer(f, Files::ANALYZER_MD5); } diff --git a/testing/btest/Baseline/doc.sphinx.include-doc_httpmonitor_file_extraction_bro/output b/testing/btest/Baseline/doc.sphinx.include-doc_httpmonitor_file_extraction_bro/output index b193e4a530..4a1fe36596 100644 --- a/testing/btest/Baseline/doc.sphinx.include-doc_httpmonitor_file_extraction_bro/output +++ b/testing/btest/Baseline/doc.sphinx.include-doc_httpmonitor_file_extraction_bro/output @@ -11,15 +11,18 @@ global mime_to_ext: table[string] of string = { ["text/html"] = "html", }; -event file_mime_type(f: fa_file, mime_type: string) +event file_metadata_inferred(f: fa_file, meta: inferred_file_metadata) { if ( f$source != "HTTP" ) return; - if ( mime_type !in mime_to_ext ) + if ( ! meta?$mime_type ) return; - local fname = fmt("%s-%s.%s", f$source, f$id, mime_to_ext[mime_type]); + if ( meta$mime_type !in mime_to_ext ) + return; + + local fname = fmt("%s-%s.%s", f$source, f$id, mime_to_ext[meta$mime_type]); print fmt("Extracting file %s", fname); Files::add_analyzer(f, Files::ANALYZER_EXTRACT, [$extract_filename=fname]); - } \ No newline at end of file + } diff --git a/testing/btest/Baseline/plugins.hooks/output b/testing/btest/Baseline/plugins.hooks/output index 2b8df03351..31fdddf184 100644 --- a/testing/btest/Baseline/plugins.hooks/output +++ b/testing/btest/Baseline/plugins.hooks/output @@ -133,7 +133,7 @@ 0.000000 MetaHookPost CallFunction(Files::register_analyzer_add_callback, , (Files::ANALYZER_EXTRACT, FileExtract::on_add{ if (!FileExtract::args?$extract_filename) FileExtract::args$extract_filename = cat(extract-, FileExtract::f$last_active, -, FileExtract::f$source, -, FileExtract::f$id)FileExtract::f$info$extracted = FileExtract::args$extract_filenameFileExtract::args$extract_filename = build_path_compressed(FileExtract::prefix, FileExtract::args$extract_filename)mkdir(FileExtract::prefix)})) -> 0.000000 MetaHookPost CallFunction(Files::register_protocol, , (Analyzer::ANALYZER_DTLS, [get_file_handle=SSL::get_file_handle{ return ()}, describe=SSL::describe_file{ SSL::cid{ if (SSL::f$source != SSL || !SSL::f?$info || !SSL::f$info?$x509 || !SSL::f$info$x509?$certificate) return ()for ([SSL::cid] in SSL::f$conns) { if (SSL::f$conns[SSL::cid]?$ssl) { SSL::c = SSL::f$conns[SSL::cid]return (cat(SSL::c$id$resp_h, :, SSL::c$id$resp_p))}}return (cat(Serial: , SSL::f$info$x509$certificate$serial, Subject: , SSL::f$info$x509$certificate$subject, Issuer: , SSL::f$info$x509$certificate$issuer))}}])) -> 0.000000 MetaHookPost CallFunction(Files::register_protocol, , (Analyzer::ANALYZER_FTP_DATA, [get_file_handle=FTP::get_file_handle{ if (!FTP::c$id$resp_h, FTP::c$id$resp_p in FTP::ftp_data_expected) return ()return (cat(Analyzer::ANALYZER_FTP_DATA, FTP::c$start_time, FTP::c$id, FTP::is_orig))}, describe=FTP::describe_file{ FTP::cid{ if (FTP::f$source != FTP) return ()for ([FTP::cid] in FTP::f$conns) { if (FTP::f$conns[FTP::cid]?$ftp) return (FTP::describe(FTP::f$conns[FTP::cid]$ftp))}return ()}}])) -> -0.000000 MetaHookPost CallFunction(Files::register_protocol, , (Analyzer::ANALYZER_HTTP, [get_file_handle=HTTP::get_file_handle{ if (!HTTP::c?$http) return ()if (HTTP::c$http$range_request && HTTP::c$http?$etag && !HTTP::is_orig) { return (cat(Analyzer::ANALYZER_HTTP, HTTP::is_orig, HTTP::c$id$orig_h, HTTP::build_url(HTTP::c$http), HTTP::c$http$etag))}else{ HTTP::mime_depth = HTTP::is_orig ? HTTP::c$http$orig_mime_depth : HTTP::c$http$resp_mime_depthreturn (cat(Analyzer::ANALYZER_HTTP, HTTP::c$start_time, HTTP::is_orig, HTTP::c$http$trans_depth, HTTP::mime_depth, id_string(HTTP::c$id)))}}, describe=HTTP::describe_file{ HTTP::cid{ if (HTTP::f$source != HTTP) return ()for ([HTTP::cid] in HTTP::f$conns) { if (HTTP::f$conns[HTTP::cid]?$http) return (HTTP::build_url_http(HTTP::f$conns[HTTP::cid]$http))}return ()}}])) -> +0.000000 MetaHookPost CallFunction(Files::register_protocol, , (Analyzer::ANALYZER_HTTP, [get_file_handle=HTTP::get_file_handle{ if (!HTTP::c?$http) return ()if (HTTP::c$http$range_request && !HTTP::is_orig) { return (cat(Analyzer::ANALYZER_HTTP, HTTP::is_orig, HTTP::c$id$orig_h, HTTP::build_url(HTTP::c$http)))}else{ HTTP::mime_depth = HTTP::is_orig ? HTTP::c$http$orig_mime_depth : HTTP::c$http$resp_mime_depthreturn (cat(Analyzer::ANALYZER_HTTP, HTTP::c$start_time, HTTP::is_orig, HTTP::c$http$trans_depth, HTTP::mime_depth, id_string(HTTP::c$id)))}}, describe=HTTP::describe_file{ HTTP::cid{ if (HTTP::f$source != HTTP) return ()for ([HTTP::cid] in HTTP::f$conns) { if (HTTP::f$conns[HTTP::cid]?$http) return (HTTP::build_url_http(HTTP::f$conns[HTTP::cid]$http))}return ()}}])) -> 0.000000 MetaHookPost CallFunction(Files::register_protocol, , (Analyzer::ANALYZER_IRC_DATA, [get_file_handle=IRC::get_file_handle{ return (cat(Analyzer::ANALYZER_IRC_DATA, IRC::c$start_time, IRC::c$id, IRC::is_orig))}, describe=anonymous-function{ return ()}])) -> 0.000000 MetaHookPost CallFunction(Files::register_protocol, , (Analyzer::ANALYZER_SMTP, [get_file_handle=SMTP::get_file_handle{ return (cat(Analyzer::ANALYZER_SMTP, SMTP::c$start_time, SMTP::c$smtp$trans_depth, SMTP::c$smtp_state$mime_depth))}, describe=SMTP::describe_file{ SMTP::cid{ if (SMTP::f$source != SMTP) return ()for ([SMTP::cid] in SMTP::f$conns) { SMTP::c = SMTP::f$conns[SMTP::cid]return (SMTP::describe(SMTP::c$smtp))}return ()}}])) -> 0.000000 MetaHookPost CallFunction(Files::register_protocol, , (Analyzer::ANALYZER_SSL, [get_file_handle=SSL::get_file_handle{ return ()}, describe=SSL::describe_file{ SSL::cid{ if (SSL::f$source != SSL || !SSL::f?$info || !SSL::f$info?$x509 || !SSL::f$info$x509?$certificate) return ()for ([SSL::cid] in SSL::f$conns) { if (SSL::f$conns[SSL::cid]?$ssl) { SSL::c = SSL::f$conns[SSL::cid]return (cat(SSL::c$id$resp_h, :, SSL::c$id$resp_p))}}return (cat(Serial: , SSL::f$info$x509$certificate$serial, Subject: , SSL::f$info$x509$certificate$subject, Issuer: , SSL::f$info$x509$certificate$issuer))}}])) -> @@ -201,7 +201,7 @@ 0.000000 MetaHookPost CallFunction(Log::__create_stream, , (Weird::LOG, [columns=, ev=Weird::log_weird, path=weird])) -> 0.000000 MetaHookPost CallFunction(Log::__create_stream, , (X509::LOG, [columns=, ev=X509::log_x509, path=x509])) -> 0.000000 MetaHookPost CallFunction(Log::__create_stream, , (mysql::LOG, [columns=, ev=MySQL::log_mysql, path=mysql])) -> -0.000000 MetaHookPost CallFunction(Log::__write, , (PacketFilter::LOG, [ts=1429534151.732383, node=bro, filter=ip or not ip, init=T, success=T])) -> +0.000000 MetaHookPost CallFunction(Log::__write, , (PacketFilter::LOG, [ts=1428700698.322438, node=bro, filter=ip or not ip, init=T, success=T])) -> 0.000000 MetaHookPost CallFunction(Log::add_default_filter, , (Cluster::LOG)) -> 0.000000 MetaHookPost CallFunction(Log::add_default_filter, , (Communication::LOG)) -> 0.000000 MetaHookPost CallFunction(Log::add_default_filter, , (Conn::LOG)) -> @@ -298,7 +298,7 @@ 0.000000 MetaHookPost CallFunction(Log::create_stream, , (Weird::LOG, [columns=, ev=Weird::log_weird, path=weird])) -> 0.000000 MetaHookPost CallFunction(Log::create_stream, , (X509::LOG, [columns=, ev=X509::log_x509, path=x509])) -> 0.000000 MetaHookPost CallFunction(Log::create_stream, , (mysql::LOG, [columns=, ev=MySQL::log_mysql, path=mysql])) -> -0.000000 MetaHookPost CallFunction(Log::write, , (PacketFilter::LOG, [ts=1429534151.732383, node=bro, filter=ip or not ip, init=T, success=T])) -> +0.000000 MetaHookPost CallFunction(Log::write, , (PacketFilter::LOG, [ts=1428700698.322438, node=bro, filter=ip or not ip, init=T, success=T])) -> 0.000000 MetaHookPost CallFunction(Notice::want_pp, , ()) -> 0.000000 MetaHookPost CallFunction(PacketFilter::build, , ()) -> 0.000000 MetaHookPost CallFunction(PacketFilter::combine_filters, , (ip or not ip, and, )) -> @@ -754,7 +754,7 @@ 0.000000 MetaHookPre CallFunction(Log::__create_stream, , (Weird::LOG, [columns=, ev=Weird::log_weird, path=weird])) 0.000000 MetaHookPre CallFunction(Log::__create_stream, , (X509::LOG, [columns=, ev=X509::log_x509, path=x509])) 0.000000 MetaHookPre CallFunction(Log::__create_stream, , (mysql::LOG, [columns=, ev=MySQL::log_mysql, path=mysql])) -0.000000 MetaHookPre CallFunction(Log::__write, , (PacketFilter::LOG, [ts=1429534151.732383, node=bro, filter=ip or not ip, init=T, success=T])) +0.000000 MetaHookPre CallFunction(Log::__write, , (PacketFilter::LOG, [ts=1428700698.322438, node=bro, filter=ip or not ip, init=T, success=T])) 0.000000 MetaHookPre CallFunction(Log::add_default_filter, , (Cluster::LOG)) 0.000000 MetaHookPre CallFunction(Log::add_default_filter, , (Communication::LOG)) 0.000000 MetaHookPre CallFunction(Log::add_default_filter, , (Conn::LOG)) @@ -851,7 +851,7 @@ 0.000000 MetaHookPre CallFunction(Log::create_stream, , (Weird::LOG, [columns=, ev=Weird::log_weird, path=weird])) 0.000000 MetaHookPre CallFunction(Log::create_stream, , (X509::LOG, [columns=, ev=X509::log_x509, path=x509])) 0.000000 MetaHookPre CallFunction(Log::create_stream, , (mysql::LOG, [columns=, ev=MySQL::log_mysql, path=mysql])) -0.000000 MetaHookPre CallFunction(Log::write, , (PacketFilter::LOG, [ts=1429534151.732383, node=bro, filter=ip or not ip, init=T, success=T])) +0.000000 MetaHookPre CallFunction(Log::write, , (PacketFilter::LOG, [ts=1428700698.322438, node=bro, filter=ip or not ip, init=T, success=T])) 0.000000 MetaHookPre CallFunction(Notice::want_pp, , ()) 0.000000 MetaHookPre CallFunction(PacketFilter::build, , ()) 0.000000 MetaHookPre CallFunction(PacketFilter::combine_filters, , (ip or not ip, and, )) @@ -1306,7 +1306,7 @@ 0.000000 | HookCallFunction Log::__create_stream(Weird::LOG, [columns=, ev=Weird::log_weird, path=weird]) 0.000000 | HookCallFunction Log::__create_stream(X509::LOG, [columns=, ev=X509::log_x509, path=x509]) 0.000000 | HookCallFunction Log::__create_stream(mysql::LOG, [columns=, ev=MySQL::log_mysql, path=mysql]) -0.000000 | HookCallFunction Log::__write(PacketFilter::LOG, [ts=1429534151.732383, node=bro, filter=ip or not ip, init=T, success=T]) +0.000000 | HookCallFunction Log::__write(PacketFilter::LOG, [ts=1428700698.322438, node=bro, filter=ip or not ip, init=T, success=T]) 0.000000 | HookCallFunction Log::add_default_filter(Cluster::LOG) 0.000000 | HookCallFunction Log::add_default_filter(Communication::LOG) 0.000000 | HookCallFunction Log::add_default_filter(Conn::LOG) @@ -1403,7 +1403,7 @@ 0.000000 | HookCallFunction Log::create_stream(Weird::LOG, [columns=, ev=Weird::log_weird, path=weird]) 0.000000 | HookCallFunction Log::create_stream(X509::LOG, [columns=, ev=X509::log_x509, path=x509]) 0.000000 | HookCallFunction Log::create_stream(mysql::LOG, [columns=, ev=MySQL::log_mysql, path=mysql]) -0.000000 | HookCallFunction Log::write(PacketFilter::LOG, [ts=1429534151.732383, node=bro, filter=ip or not ip, init=T, success=T]) +0.000000 | HookCallFunction Log::write(PacketFilter::LOG, [ts=1428700698.322438, node=bro, filter=ip or not ip, init=T, success=T]) 0.000000 | HookCallFunction Notice::want_pp() 0.000000 | HookCallFunction PacketFilter::build() 0.000000 | HookCallFunction PacketFilter::combine_filters(ip or not ip, and, ) @@ -1773,7 +1773,7 @@ 1362692527.009775 MetaHookPost CallFunction(Log::write, , (Files::LOG, [ts=1362692527.009512, fuid=FakNcS1Jfe01uljb3, tx_hosts={192.150.187.43}, rx_hosts={141.142.228.5}, conn_uids={CXWv6p3arKYeMETxOg}, source=HTTP, depth=0, analyzers={}, mime_type=text/plain, filename=, duration=262.0 usecs, local_orig=, is_orig=F, seen_bytes=4705, total_bytes=4705, missing_bytes=0, overflow_bytes=0, timedout=F, parent_fuid=, md5=, sha1=, sha256=, x509=, extracted=])) -> 1362692527.009775 MetaHookPost CallFunction(Log::write, , (HTTP::LOG, [ts=1362692526.939527, uid=CXWv6p3arKYeMETxOg, id=[orig_h=141.142.228.5, orig_p=59856<...>/plain], current_entity=, orig_mime_depth=1, resp_mime_depth=1])) -> 1362692527.009775 MetaHookPost CallFunction(cat, , (Analyzer::ANALYZER_HTTP, 1362692526.869344, F, 1, 1, 141.142.228.5:59856 > 192.150.187.43:80)) -> -1362692527.009775 MetaHookPost CallFunction(file_mime_type, , ([id=FakNcS1Jfe01uljb3, parent_id=, source=HTTP, is_orig=F, conns={[[orig_h=141.142.228.5, orig_p=59856<...>/plain)) -> +1362692527.009775 MetaHookPost CallFunction(file_metadata_inferred, , ([id=FakNcS1Jfe01uljb3, parent_id=, source=HTTP, is_orig=F, conns={[[orig_h=141.142.228.5, orig_p=59856<...>/plain]]])) -> 1362692527.009775 MetaHookPost CallFunction(file_state_remove, , ([id=FakNcS1Jfe01uljb3, parent_id=, source=HTTP, is_orig=F, conns={[[orig_h=141.142.228.5, orig_p=59856<...>/plain], current_entity=[filename=], orig_mime_depth=1, resp_mime_depth=1], irc=, u2_events=])) -> 1362692527.009775 MetaHookPost CallFunction(fmt, , (%s:%d > %s:%d, 141.142.228.5, 59856<...>/tcp)) -> 1362692527.009775 MetaHookPost CallFunction(get_file_handle, , (Analyzer::ANALYZER_HTTP, [id=[orig_h=141.142.228.5, orig_p=59856<...>/plain], current_entity=, orig_mime_depth=1, resp_mime_depth=1]}, current_request=1, current_response=1, trans_depth=1], irc=, modbus=, mysql=, radius=, rdp=, snmp=, smtp=, smtp_state=, socks=, ssh=, syslog=], F)) -> @@ -1782,7 +1782,7 @@ 1362692527.009775 MetaHookPost CallFunction(id_string, , ([orig_h=141.142.228.5, orig_p=59856<...>/tcp])) -> 1362692527.009775 MetaHookPost CallFunction(set_file_handle, , (Analyzer::ANALYZER_HTTP1362692526.869344F11141.142.228.5:59856 > 192.150.187.43:80)) -> 1362692527.009775 MetaHookPost DrainEvents() -> -1362692527.009775 MetaHookPost QueueEvent(file_mime_type([id=FakNcS1Jfe01uljb3, parent_id=, source=HTTP, is_orig=F, conns={[[orig_h=141.142.228.5, orig_p=59856<...>/plain)) -> false +1362692527.009775 MetaHookPost QueueEvent(file_metadata_inferred([id=FakNcS1Jfe01uljb3, parent_id=, source=HTTP, is_orig=F, conns={[[orig_h=141.142.228.5, orig_p=59856<...>/plain]]])) -> false 1362692527.009775 MetaHookPost QueueEvent(file_state_remove([id=FakNcS1Jfe01uljb3, parent_id=, source=HTTP, is_orig=F, conns={[[orig_h=141.142.228.5, orig_p=59856<...>/plain], current_entity=[filename=], orig_mime_depth=1, resp_mime_depth=1], irc=, u2_events=])) -> false 1362692527.009775 MetaHookPost QueueEvent(get_file_handle(Analyzer::ANALYZER_HTTP, [id=[orig_h=141.142.228.5, orig_p=59856<...>/plain], current_entity=[filename=], orig_mime_depth=1, resp_mime_depth=1]}, current_request=1, current_response=1, trans_depth=1], irc=, modbus=, mysql=, radius=, rdp=, snmp=, smtp=, smtp_state=, socks=, ssh=, syslog=], F)) -> false 1362692527.009775 MetaHookPost QueueEvent(http_end_entity([id=[orig_h=141.142.228.5, orig_p=59856<...>/plain], current_entity=[filename=], orig_mime_depth=1, resp_mime_depth=1]}, current_request=1, current_response=1, trans_depth=1], irc=, modbus=, mysql=, radius=, rdp=, snmp=, smtp=, smtp_state=, socks=, ssh=, syslog=], F)) -> false @@ -1798,7 +1798,7 @@ 1362692527.009775 MetaHookPre CallFunction(Log::write, , (Files::LOG, [ts=1362692527.009512, fuid=FakNcS1Jfe01uljb3, tx_hosts={192.150.187.43}, rx_hosts={141.142.228.5}, conn_uids={CXWv6p3arKYeMETxOg}, source=HTTP, depth=0, analyzers={}, mime_type=text/plain, filename=, duration=262.0 usecs, local_orig=, is_orig=F, seen_bytes=4705, total_bytes=4705, missing_bytes=0, overflow_bytes=0, timedout=F, parent_fuid=, md5=, sha1=, sha256=, x509=, extracted=])) 1362692527.009775 MetaHookPre CallFunction(Log::write, , (HTTP::LOG, [ts=1362692526.939527, uid=CXWv6p3arKYeMETxOg, id=[orig_h=141.142.228.5, orig_p=59856<...>/plain], current_entity=, orig_mime_depth=1, resp_mime_depth=1])) 1362692527.009775 MetaHookPre CallFunction(cat, , (Analyzer::ANALYZER_HTTP, 1362692526.869344, F, 1, 1, 141.142.228.5:59856 > 192.150.187.43:80)) -1362692527.009775 MetaHookPre CallFunction(file_mime_type, , ([id=FakNcS1Jfe01uljb3, parent_id=, source=HTTP, is_orig=F, conns={[[orig_h=141.142.228.5, orig_p=59856<...>/plain)) +1362692527.009775 MetaHookPre CallFunction(file_metadata_inferred, , ([id=FakNcS1Jfe01uljb3, parent_id=, source=HTTP, is_orig=F, conns={[[orig_h=141.142.228.5, orig_p=59856<...>/plain]]])) 1362692527.009775 MetaHookPre CallFunction(file_state_remove, , ([id=FakNcS1Jfe01uljb3, parent_id=, source=HTTP, is_orig=F, conns={[[orig_h=141.142.228.5, orig_p=59856<...>/plain], current_entity=[filename=], orig_mime_depth=1, resp_mime_depth=1], irc=, u2_events=])) 1362692527.009775 MetaHookPre CallFunction(fmt, , (%s:%d > %s:%d, 141.142.228.5, 59856<...>/tcp)) 1362692527.009775 MetaHookPre CallFunction(get_file_handle, , (Analyzer::ANALYZER_HTTP, [id=[orig_h=141.142.228.5, orig_p=59856<...>/plain], current_entity=, orig_mime_depth=1, resp_mime_depth=1]}, current_request=1, current_response=1, trans_depth=1], irc=, modbus=, mysql=, radius=, rdp=, snmp=, smtp=, smtp_state=, socks=, ssh=, syslog=], F)) @@ -1807,7 +1807,7 @@ 1362692527.009775 MetaHookPre CallFunction(id_string, , ([orig_h=141.142.228.5, orig_p=59856<...>/tcp])) 1362692527.009775 MetaHookPre CallFunction(set_file_handle, , (Analyzer::ANALYZER_HTTP1362692526.869344F11141.142.228.5:59856 > 192.150.187.43:80)) 1362692527.009775 MetaHookPre DrainEvents() -1362692527.009775 MetaHookPre QueueEvent(file_mime_type([id=FakNcS1Jfe01uljb3, parent_id=, source=HTTP, is_orig=F, conns={[[orig_h=141.142.228.5, orig_p=59856<...>/plain)) +1362692527.009775 MetaHookPre QueueEvent(file_metadata_inferred([id=FakNcS1Jfe01uljb3, parent_id=, source=HTTP, is_orig=F, conns={[[orig_h=141.142.228.5, orig_p=59856<...>/plain]]])) 1362692527.009775 MetaHookPre QueueEvent(file_state_remove([id=FakNcS1Jfe01uljb3, parent_id=, source=HTTP, is_orig=F, conns={[[orig_h=141.142.228.5, orig_p=59856<...>/plain], current_entity=[filename=], orig_mime_depth=1, resp_mime_depth=1], irc=, u2_events=])) 1362692527.009775 MetaHookPre QueueEvent(get_file_handle(Analyzer::ANALYZER_HTTP, [id=[orig_h=141.142.228.5, orig_p=59856<...>/plain], current_entity=[filename=], orig_mime_depth=1, resp_mime_depth=1]}, current_request=1, current_response=1, trans_depth=1], irc=, modbus=, mysql=, radius=, rdp=, snmp=, smtp=, smtp_state=, socks=, ssh=, syslog=], F)) 1362692527.009775 MetaHookPre QueueEvent(http_end_entity([id=[orig_h=141.142.228.5, orig_p=59856<...>/plain], current_entity=[filename=], orig_mime_depth=1, resp_mime_depth=1]}, current_request=1, current_response=1, trans_depth=1], irc=, modbus=, mysql=, radius=, rdp=, snmp=, smtp=, smtp_state=, socks=, ssh=, syslog=], F)) @@ -1824,7 +1824,7 @@ 1362692527.009775 | HookCallFunction Log::write(Files::LOG, [ts=1362692527.009512, fuid=FakNcS1Jfe01uljb3, tx_hosts={192.150.187.43}, rx_hosts={141.142.228.5}, conn_uids={CXWv6p3arKYeMETxOg}, source=HTTP, depth=0, analyzers={}, mime_type=text/plain, filename=, duration=262.0 usecs, local_orig=, is_orig=F, seen_bytes=4705, total_bytes=4705, missing_bytes=0, overflow_bytes=0, timedout=F, parent_fuid=, md5=, sha1=, sha256=, x509=, extracted=]) 1362692527.009775 | HookCallFunction Log::write(HTTP::LOG, [ts=1362692526.939527, uid=CXWv6p3arKYeMETxOg, id=[orig_h=141.142.228.5, orig_p=59856<...>/plain], current_entity=, orig_mime_depth=1, resp_mime_depth=1]) 1362692527.009775 | HookCallFunction cat(Analyzer::ANALYZER_HTTP, 1362692526.869344, F, 1, 1, 141.142.228.5:59856 > 192.150.187.43:80) -1362692527.009775 | HookCallFunction file_mime_type([id=FakNcS1Jfe01uljb3, parent_id=, source=HTTP, is_orig=F, conns={[[orig_h=141.142.228.5, orig_p=59856<...>/plain) +1362692527.009775 | HookCallFunction file_metadata_inferred([id=FakNcS1Jfe01uljb3, parent_id=, source=HTTP, is_orig=F, conns={[[orig_h=141.142.228.5, orig_p=59856<...>/plain]]]) 1362692527.009775 | HookCallFunction file_state_remove([id=FakNcS1Jfe01uljb3, parent_id=, source=HTTP, is_orig=F, conns={[[orig_h=141.142.228.5, orig_p=59856<...>/plain], current_entity=[filename=], orig_mime_depth=1, resp_mime_depth=1], irc=, u2_events=]) 1362692527.009775 | HookCallFunction fmt(%s:%d > %s:%d, 141.142.228.5, 59856<...>/tcp) 1362692527.009775 | HookCallFunction get_file_handle(Analyzer::ANALYZER_HTTP, [id=[orig_h=141.142.228.5, orig_p=59856<...>/plain], current_entity=, orig_mime_depth=1, resp_mime_depth=1]}, current_request=1, current_response=1, trans_depth=1], irc=, modbus=, mysql=, radius=, rdp=, snmp=, smtp=, smtp_state=, socks=, ssh=, syslog=], F) @@ -1833,7 +1833,7 @@ 1362692527.009775 | HookCallFunction id_string([orig_h=141.142.228.5, orig_p=59856<...>/tcp]) 1362692527.009775 | HookCallFunction set_file_handle(Analyzer::ANALYZER_HTTP1362692526.869344F11141.142.228.5:59856 > 192.150.187.43:80) 1362692527.009775 | HookDrainEvents -1362692527.009775 | HookQueueEvent file_mime_type([id=FakNcS1Jfe01uljb3, parent_id=, source=HTTP, is_orig=F, conns={[[orig_h=141.142.228.5, orig_p=59856<...>/plain) +1362692527.009775 | HookQueueEvent file_metadata_inferred([id=FakNcS1Jfe01uljb3, parent_id=, source=HTTP, is_orig=F, conns={[[orig_h=141.142.228.5, orig_p=59856<...>/plain]]]) 1362692527.009775 | HookQueueEvent file_state_remove([id=FakNcS1Jfe01uljb3, parent_id=, source=HTTP, is_orig=F, conns={[[orig_h=141.142.228.5, orig_p=59856<...>/plain], current_entity=[filename=], orig_mime_depth=1, resp_mime_depth=1], irc=, u2_events=]) 1362692527.009775 | HookQueueEvent get_file_handle(Analyzer::ANALYZER_HTTP, [id=[orig_h=141.142.228.5, orig_p=59856<...>/plain], current_entity=[filename=], orig_mime_depth=1, resp_mime_depth=1]}, current_request=1, current_response=1, trans_depth=1], irc=, modbus=, mysql=, radius=, rdp=, snmp=, smtp=, smtp_state=, socks=, ssh=, syslog=], F) 1362692527.009775 | HookQueueEvent http_end_entity([id=[orig_h=141.142.228.5, orig_p=59856<...>/plain], current_entity=[filename=], orig_mime_depth=1, resp_mime_depth=1]}, current_request=1, current_response=1, trans_depth=1], irc=, modbus=, mysql=, radius=, rdp=, snmp=, smtp=, smtp_state=, socks=, ssh=, syslog=], F) diff --git a/testing/btest/Baseline/plugins.reader/output b/testing/btest/Baseline/plugins.reader/output index fa218d04a5..0f8980d0e7 100644 --- a/testing/btest/Baseline/plugins.reader/output +++ b/testing/btest/Baseline/plugins.reader/output @@ -1,4 +1,4 @@ Demo::Foo - A Foo test input reader (dynamic, version 1.0) - [Writer] Foo (Input::READER_FOO) + [Reader] Foo (Input::READER_FOO) === diff --git a/testing/btest/Baseline/scripts.policy.misc.dump-events/all-events-no-args.log b/testing/btest/Baseline/scripts.policy.misc.dump-events/all-events-no-args.log index f5e53044b9..d5369c07a4 100644 --- a/testing/btest/Baseline/scripts.policy.misc.dump-events/all-events-no-args.log +++ b/testing/btest/Baseline/scripts.policy.misc.dump-events/all-events-no-args.log @@ -59,7 +59,7 @@ 1254722770.692743 file_over_new_connection 1254722770.692743 mime_end_entity 1254722770.692743 get_file_handle -1254722770.692743 file_mime_type +1254722770.692743 file_metadata_inferred 1254722770.692743 file_state_remove 1254722770.692743 get_file_handle 1254722770.692743 mime_begin_entity @@ -70,7 +70,7 @@ 1254722770.692743 file_over_new_connection 1254722770.692804 mime_end_entity 1254722770.692804 get_file_handle -1254722770.692804 file_mime_type +1254722770.692804 file_metadata_inferred 1254722770.692804 file_state_remove 1254722770.692804 get_file_handle 1254722770.692804 mime_end_entity @@ -84,7 +84,7 @@ 1254722770.692804 file_new 1254722770.692804 file_over_new_connection 1254722770.695115 new_connection -1254722771.494181 file_mime_type +1254722771.494181 file_metadata_inferred 1254722771.858334 mime_end_entity 1254722771.858334 get_file_handle 1254722771.858334 file_state_remove diff --git a/testing/btest/Baseline/scripts.policy.misc.dump-events/all-events.log b/testing/btest/Baseline/scripts.policy.misc.dump-events/all-events.log index 397812ae7c..847d5122e2 100644 --- a/testing/btest/Baseline/scripts.policy.misc.dump-events/all-events.log +++ b/testing/btest/Baseline/scripts.policy.misc.dump-events/all-events.log @@ -312,9 +312,9 @@ [1] c: connection = [id=[orig_h=10.10.1.4, orig_p=1470/tcp, resp_h=74.53.140.153, resp_p=25/tcp], orig=[size=1610, state=4, num_pkts=9, num_bytes_ip=518, flow_label=0], resp=[size=462, state=4, num_pkts=10, num_bytes_ip=870, flow_label=0], start_time=1254722767.529046, duration=3.163697, service={^J^ISMTP^J}, addl=, hot=0, history=ShAdDa, uid=CjhGID4nQcgTWjvg4c, tunnel=, dpd=, conn=, extract_orig=F, extract_resp=F, dhcp=, dnp3=, dns=, dns_state=, ftp=, ftp_data_reuse=F, ssl=, http=, http_state=, irc=, modbus=, mysql=, radius=, rdp=, snmp=, smtp=[ts=1254722768.219663, uid=CjhGID4nQcgTWjvg4c, id=[orig_h=10.10.1.4, orig_p=1470/tcp, resp_h=74.53.140.153, resp_p=25/tcp], trans_depth=1, helo=GP, mailfrom=, rcptto={^J^I^J}, date=Mon, 5 Oct 2009 11:36:07 +0530, from="Gurpartap Singh" , to={^J^I^J}, reply_to=, msg_id=<000301ca4581$ef9e57f0$cedb07d0$@in>, in_reply_to=, subject=SMTP, x_originating_ip=, first_received=, second_received=, last_reply=354 Enter message, ending with "." on a line by itself, path=[74.53.140.153, 10.10.1.4], user_agent=Microsoft Office Outlook 12.0, tls=F, process_received_from=T, has_client_activity=T, entity=, fuids=[Fel9gs4OtNEV6gUJZ5]], smtp_state=[helo=GP, messages_transferred=0, pending_messages=, mime_depth=3], socks=, ssh=, syslog=] [2] is_orig: bool = T -1254722770.692743 file_mime_type +1254722770.692743 file_metadata_inferred [0] f: fa_file = [id=Fel9gs4OtNEV6gUJZ5, parent_id=, source=SMTP, is_orig=F, conns={^J^I[[orig_h=10.10.1.4, orig_p=1470/tcp, resp_h=74.53.140.153, resp_p=25/tcp]] = [id=[orig_h=10.10.1.4, orig_p=1470/tcp, resp_h=74.53.140.153, resp_p=25/tcp], orig=[size=1610, state=4, num_pkts=9, num_bytes_ip=518, flow_label=0], resp=[size=462, state=4, num_pkts=10, num_bytes_ip=870, flow_label=0], start_time=1254722767.529046, duration=3.163697, service={^J^I^ISMTP^J^I}, addl=, hot=0, history=ShAdDa, uid=CjhGID4nQcgTWjvg4c, tunnel=, dpd=, conn=, extract_orig=F, extract_resp=F, dhcp=, dnp3=, dns=, dns_state=, ftp=, ftp_data_reuse=F, ssl=, http=, http_state=, irc=, modbus=, mysql=, radius=, rdp=, snmp=, smtp=[ts=1254722768.219663, uid=CjhGID4nQcgTWjvg4c, id=[orig_h=10.10.1.4, orig_p=1470/tcp, resp_h=74.53.140.153, resp_p=25/tcp], trans_depth=1, helo=GP, mailfrom=, rcptto={^J^I^I^J^I}, date=Mon, 5 Oct 2009 11:36:07 +0530, from="Gurpartap Singh" , to={^J^I^I^J^I}, reply_to=, msg_id=<000301ca4581$ef9e57f0$cedb07d0$@in>, in_reply_to=, subject=SMTP, x_originating_ip=, first_received=, second_received=, last_reply=354 Enter message, ending with "." on a line by itself, path=[74.53.140.153, 10.10.1.4], user_agent=Microsoft Office Outlook 12.0, tls=F, process_received_from=T, has_client_activity=T, entity=, fuids=[Fel9gs4OtNEV6gUJZ5]], smtp_state=[helo=GP, messages_transferred=0, pending_messages=, mime_depth=3], socks=, ssh=, syslog=]^J}, last_active=1254722770.692743, seen_bytes=77, total_bytes=, missing_bytes=0, overflow_bytes=0, timeout_interval=2.0 mins, bof_buffer_size=4096, bof_buffer=Hello^M^J^M^J ^M^J^M^JI send u smtp pcap file ^M^J^M^JFind the attachment^M^J^M^J ^M^J^M^JGPS^M^J^M^J, info=[ts=1254722770.692743, fuid=Fel9gs4OtNEV6gUJZ5, tx_hosts={^J^I74.53.140.153^J}, rx_hosts={^J^I10.10.1.4^J}, conn_uids={^J^ICjhGID4nQcgTWjvg4c^J}, source=SMTP, depth=3, analyzers={^J^J}, mime_type=, filename=, duration=0 secs, local_orig=, is_orig=F, seen_bytes=0, total_bytes=, missing_bytes=0, overflow_bytes=0, timedout=F, parent_fuid=, md5=, sha1=, sha256=, x509=, extracted=], ftp=, http=, irc=, u2_events=] - [1] mime_type: string = text/plain + [1] meta: inferred_file_metadata = [mime_type=text/plain, mime_types=[[strength=-20, mime=text/plain]]] 1254722770.692743 file_state_remove [0] f: fa_file = [id=Fel9gs4OtNEV6gUJZ5, parent_id=, source=SMTP, is_orig=F, conns={^J^I[[orig_h=10.10.1.4, orig_p=1470/tcp, resp_h=74.53.140.153, resp_p=25/tcp]] = [id=[orig_h=10.10.1.4, orig_p=1470/tcp, resp_h=74.53.140.153, resp_p=25/tcp], orig=[size=1610, state=4, num_pkts=9, num_bytes_ip=518, flow_label=0], resp=[size=462, state=4, num_pkts=10, num_bytes_ip=870, flow_label=0], start_time=1254722767.529046, duration=3.163697, service={^J^I^ISMTP^J^I}, addl=, hot=0, history=ShAdDa, uid=CjhGID4nQcgTWjvg4c, tunnel=, dpd=, conn=, extract_orig=F, extract_resp=F, dhcp=, dnp3=, dns=, dns_state=, ftp=, ftp_data_reuse=F, ssl=, http=, http_state=, irc=, modbus=, mysql=, radius=, rdp=, snmp=, smtp=[ts=1254722768.219663, uid=CjhGID4nQcgTWjvg4c, id=[orig_h=10.10.1.4, orig_p=1470/tcp, resp_h=74.53.140.153, resp_p=25/tcp], trans_depth=1, helo=GP, mailfrom=, rcptto={^J^I^I^J^I}, date=Mon, 5 Oct 2009 11:36:07 +0530, from="Gurpartap Singh" , to={^J^I^I^J^I}, reply_to=, msg_id=<000301ca4581$ef9e57f0$cedb07d0$@in>, in_reply_to=, subject=SMTP, x_originating_ip=, first_received=, second_received=, last_reply=354 Enter message, ending with "." on a line by itself, path=[74.53.140.153, 10.10.1.4], user_agent=Microsoft Office Outlook 12.0, tls=F, process_received_from=T, has_client_activity=T, entity=, fuids=[Fel9gs4OtNEV6gUJZ5]], smtp_state=[helo=GP, messages_transferred=0, pending_messages=, mime_depth=3], socks=, ssh=, syslog=]^J}, last_active=1254722770.692743, seen_bytes=77, total_bytes=, missing_bytes=0, overflow_bytes=0, timeout_interval=2.0 mins, bof_buffer_size=4096, bof_buffer=Hello^M^J^M^J ^M^J^M^JI send u smtp pcap file ^M^J^M^JFind the attachment^M^J^M^J ^M^J^M^JGPS^M^J^M^J, info=[ts=1254722770.692743, fuid=Fel9gs4OtNEV6gUJZ5, tx_hosts={^J^I74.53.140.153^J}, rx_hosts={^J^I10.10.1.4^J}, conn_uids={^J^ICjhGID4nQcgTWjvg4c^J}, source=SMTP, depth=3, analyzers={^J^J}, mime_type=text/plain, filename=, duration=0 secs, local_orig=, is_orig=F, seen_bytes=77, total_bytes=, missing_bytes=0, overflow_bytes=0, timedout=F, parent_fuid=, md5=, sha1=, sha256=, x509=, extracted=], ftp=, http=, irc=, u2_events=] @@ -356,9 +356,9 @@ [1] c: connection = [id=[orig_h=10.10.1.4, orig_p=1470/tcp, resp_h=74.53.140.153, resp_p=25/tcp], orig=[size=4530, state=4, num_pkts=11, num_bytes_ip=3518, flow_label=0], resp=[size=462, state=4, num_pkts=10, num_bytes_ip=870, flow_label=0], start_time=1254722767.529046, duration=3.163758, service={^J^ISMTP^J}, addl=, hot=0, history=ShAdDa, uid=CjhGID4nQcgTWjvg4c, tunnel=, dpd=, conn=, extract_orig=F, extract_resp=F, dhcp=, dnp3=, dns=, dns_state=, ftp=, ftp_data_reuse=F, ssl=, http=, http_state=, irc=, modbus=, mysql=, radius=, rdp=, snmp=, smtp=[ts=1254722768.219663, uid=CjhGID4nQcgTWjvg4c, id=[orig_h=10.10.1.4, orig_p=1470/tcp, resp_h=74.53.140.153, resp_p=25/tcp], trans_depth=1, helo=GP, mailfrom=, rcptto={^J^I^J}, date=Mon, 5 Oct 2009 11:36:07 +0530, from="Gurpartap Singh" , to={^J^I^J}, reply_to=, msg_id=<000301ca4581$ef9e57f0$cedb07d0$@in>, in_reply_to=, subject=SMTP, x_originating_ip=, first_received=, second_received=, last_reply=354 Enter message, ending with "." on a line by itself, path=[74.53.140.153, 10.10.1.4], user_agent=Microsoft Office Outlook 12.0, tls=F, process_received_from=T, has_client_activity=T, entity=, fuids=[Fel9gs4OtNEV6gUJZ5, Ft4M3f2yMvLlmwtbq9]], smtp_state=[helo=GP, messages_transferred=0, pending_messages=, mime_depth=4], socks=, ssh=, syslog=] [2] is_orig: bool = T -1254722770.692804 file_mime_type +1254722770.692804 file_metadata_inferred [0] f: fa_file = [id=Ft4M3f2yMvLlmwtbq9, parent_id=, source=SMTP, is_orig=F, conns={^J^I[[orig_h=10.10.1.4, orig_p=1470/tcp, resp_h=74.53.140.153, resp_p=25/tcp]] = [id=[orig_h=10.10.1.4, orig_p=1470/tcp, resp_h=74.53.140.153, resp_p=25/tcp], orig=[size=4530, state=4, num_pkts=11, num_bytes_ip=3518, flow_label=0], resp=[size=462, state=4, num_pkts=10, num_bytes_ip=870, flow_label=0], start_time=1254722767.529046, duration=3.163758, service={^J^I^ISMTP^J^I}, addl=, hot=0, history=ShAdDa, uid=CjhGID4nQcgTWjvg4c, tunnel=, dpd=, conn=, extract_orig=F, extract_resp=F, dhcp=, dnp3=, dns=, dns_state=, ftp=, ftp_data_reuse=F, ssl=, http=, http_state=, irc=, modbus=, mysql=, radius=, rdp=, snmp=, smtp=[ts=1254722768.219663, uid=CjhGID4nQcgTWjvg4c, id=[orig_h=10.10.1.4, orig_p=1470/tcp, resp_h=74.53.140.153, resp_p=25/tcp], trans_depth=1, helo=GP, mailfrom=, rcptto={^J^I^I^J^I}, date=Mon, 5 Oct 2009 11:36:07 +0530, from="Gurpartap Singh" , to={^J^I^I^J^I}, reply_to=, msg_id=<000301ca4581$ef9e57f0$cedb07d0$@in>, in_reply_to=, subject=SMTP, x_originating_ip=, first_received=, second_received=, last_reply=354 Enter message, ending with "." on a line by itself, path=[74.53.140.153, 10.10.1.4], user_agent=Microsoft Office Outlook 12.0, tls=F, process_received_from=T, has_client_activity=T, entity=, fuids=[Fel9gs4OtNEV6gUJZ5, Ft4M3f2yMvLlmwtbq9]], smtp_state=[helo=GP, messages_transferred=0, pending_messages=, mime_depth=4], socks=, ssh=, syslog=]^J}, last_active=1254722770.692804, seen_bytes=1868, total_bytes=, missing_bytes=0, overflow_bytes=0, timeout_interval=2.0 mins, bof_buffer_size=4096, bof_buffer=^M^J^M^J^M^J^M^J^M^J^M^J^M^J^M^J^M^J^M^J^M^J
^M^J^M^J

Hello

^M^J^M^J

 

^M^J^M^J

I send u smtp pcap file

^M^J^M^J

Find the attachment

^M^J^M^J

 

^M^J^M^J

GPS

^M^J^M^J
^M^J^M^J^M^J^M^J^M^J^M^J, info=[ts=1254722770.692743, fuid=Ft4M3f2yMvLlmwtbq9, tx_hosts={^J^I74.53.140.153^J}, rx_hosts={^J^I10.10.1.4^J}, conn_uids={^J^ICjhGID4nQcgTWjvg4c^J}, source=SMTP, depth=4, analyzers={^J^J}, mime_type=, filename=, duration=0 secs, local_orig=, is_orig=F, seen_bytes=0, total_bytes=, missing_bytes=0, overflow_bytes=0, timedout=F, parent_fuid=, md5=, sha1=, sha256=, x509=, extracted=], ftp=, http=, irc=, u2_events=] - [1] mime_type: string = text/html + [1] meta: inferred_file_metadata = [mime_type=text/html, mime_types=[[strength=45, mime=text/html], [strength=41, mime=text/html], [strength=-20, mime=text/plain]]] 1254722770.692804 file_state_remove [0] f: fa_file = [id=Ft4M3f2yMvLlmwtbq9, parent_id=, source=SMTP, is_orig=F, conns={^J^I[[orig_h=10.10.1.4, orig_p=1470/tcp, resp_h=74.53.140.153, resp_p=25/tcp]] = [id=[orig_h=10.10.1.4, orig_p=1470/tcp, resp_h=74.53.140.153, resp_p=25/tcp], orig=[size=4530, state=4, num_pkts=11, num_bytes_ip=3518, flow_label=0], resp=[size=462, state=4, num_pkts=10, num_bytes_ip=870, flow_label=0], start_time=1254722767.529046, duration=3.163758, service={^J^I^ISMTP^J^I}, addl=, hot=0, history=ShAdDa, uid=CjhGID4nQcgTWjvg4c, tunnel=, dpd=, conn=, extract_orig=F, extract_resp=F, dhcp=, dnp3=, dns=, dns_state=, ftp=, ftp_data_reuse=F, ssl=, http=, http_state=, irc=, modbus=, mysql=, radius=, rdp=, snmp=, smtp=[ts=1254722768.219663, uid=CjhGID4nQcgTWjvg4c, id=[orig_h=10.10.1.4, orig_p=1470/tcp, resp_h=74.53.140.153, resp_p=25/tcp], trans_depth=1, helo=GP, mailfrom=, rcptto={^J^I^I^J^I}, date=Mon, 5 Oct 2009 11:36:07 +0530, from="Gurpartap Singh" , to={^J^I^I^J^I}, reply_to=, msg_id=<000301ca4581$ef9e57f0$cedb07d0$@in>, in_reply_to=, subject=SMTP, x_originating_ip=, first_received=, second_received=, last_reply=354 Enter message, ending with "." on a line by itself, path=[74.53.140.153, 10.10.1.4], user_agent=Microsoft Office Outlook 12.0, tls=F, process_received_from=T, has_client_activity=T, entity=, fuids=[Fel9gs4OtNEV6gUJZ5, Ft4M3f2yMvLlmwtbq9]], smtp_state=[helo=GP, messages_transferred=0, pending_messages=, mime_depth=4], socks=, ssh=, syslog=]^J}, last_active=1254722770.692804, seen_bytes=1868, total_bytes=, missing_bytes=0, overflow_bytes=0, timeout_interval=2.0 mins, bof_buffer_size=4096, bof_buffer=^M^J^M^J^M^J^M^J^M^J^M^J^M^J^M^J^M^J^M^J^M^J
^M^J^M^J

Hello

^M^J^M^J

 

^M^J^M^J

I send u smtp pcap file

^M^J^M^J

Find the attachment

^M^J^M^J

 

^M^J^M^J

GPS

^M^J^M^J
^M^J^M^J^M^J^M^J^M^J^M^J, info=[ts=1254722770.692743, fuid=Ft4M3f2yMvLlmwtbq9, tx_hosts={^J^I74.53.140.153^J}, rx_hosts={^J^I10.10.1.4^J}, conn_uids={^J^ICjhGID4nQcgTWjvg4c^J}, source=SMTP, depth=4, analyzers={^J^J}, mime_type=text/html, filename=, duration=61.0 usecs, local_orig=, is_orig=F, seen_bytes=1868, total_bytes=, missing_bytes=0, overflow_bytes=0, timedout=F, parent_fuid=, md5=, sha1=, sha256=, x509=, extracted=], ftp=, http=, irc=, u2_events=] @@ -412,9 +412,9 @@ 1254722770.695115 new_connection [0] c: connection = [id=[orig_h=192.168.1.1, orig_p=3/icmp, resp_h=10.10.1.4, resp_p=4/icmp], orig=[size=0, state=0, num_pkts=0, num_bytes_ip=0, flow_label=0], resp=[size=0, state=0, num_pkts=0, num_bytes_ip=0, flow_label=0], start_time=1254722770.695115, duration=0.0, service={^J^J}, addl=, hot=0, history=, uid=CCvvfg3TEfuqmmG4bh, tunnel=, dpd=, conn=, extract_orig=F, extract_resp=F, dhcp=, dnp3=, dns=, dns_state=, ftp=, ftp_data_reuse=F, ssl=, http=, http_state=, irc=, modbus=, mysql=, radius=, rdp=, snmp=, smtp=, smtp_state=, socks=, ssh=, syslog=] -1254722771.494181 file_mime_type +1254722771.494181 file_metadata_inferred [0] f: fa_file = [id=FL9Y0d45OI4LpS6fmh, parent_id=, source=SMTP, is_orig=F, conns={^J^I[[orig_h=10.10.1.4, orig_p=1470/tcp, resp_h=74.53.140.153, resp_p=25/tcp]] = [id=[orig_h=10.10.1.4, orig_p=1470/tcp, resp_h=74.53.140.153, resp_p=25/tcp], orig=[size=4530, state=4, num_pkts=11, num_bytes_ip=3518, flow_label=0], resp=[size=462, state=4, num_pkts=10, num_bytes_ip=870, flow_label=0], start_time=1254722767.529046, duration=3.163758, service={^J^I^ISMTP^J^I}, addl=, hot=0, history=ShAdDa, uid=CjhGID4nQcgTWjvg4c, tunnel=, dpd=, conn=, extract_orig=F, extract_resp=F, dhcp=, dnp3=, dns=, dns_state=, ftp=, ftp_data_reuse=F, ssl=, http=, http_state=, irc=, modbus=, mysql=, radius=, rdp=, snmp=, smtp=[ts=1254722768.219663, uid=CjhGID4nQcgTWjvg4c, id=[orig_h=10.10.1.4, orig_p=1470/tcp, resp_h=74.53.140.153, resp_p=25/tcp], trans_depth=1, helo=GP, mailfrom=, rcptto={^J^I^I^J^I}, date=Mon, 5 Oct 2009 11:36:07 +0530, from="Gurpartap Singh" , to={^J^I^I^J^I}, reply_to=, msg_id=<000301ca4581$ef9e57f0$cedb07d0$@in>, in_reply_to=, subject=SMTP, x_originating_ip=, first_received=, second_received=, last_reply=354 Enter message, ending with "." on a line by itself, path=[74.53.140.153, 10.10.1.4], user_agent=Microsoft Office Outlook 12.0, tls=F, process_received_from=T, has_client_activity=T, entity=[filename=NEWS.txt], fuids=[Fel9gs4OtNEV6gUJZ5, Ft4M3f2yMvLlmwtbq9, FL9Y0d45OI4LpS6fmh]], smtp_state=[helo=GP, messages_transferred=0, pending_messages=, mime_depth=5], socks=, ssh=, syslog=]^J}, last_active=1254722771.494181, seen_bytes=4027, total_bytes=, missing_bytes=0, overflow_bytes=0, timeout_interval=2.0 mins, bof_buffer_size=4096, bof_buffer=Version 4.9.9.1^M^J* Many bug fixes^M^J* Improved editor^M^J^M^JVersion 4.9.9.0^M^J* Support for latest Mingw compiler system builds^M^J* Bug fixes^M^J^M^JVersion 4.9.8.9^M^J* New code tooltip display^M^J* Improved Indent/Unindent and Remove Comment^M^J* Improved automatic indent^M^J* Added support for the "interface" keyword^M^J* WebUpdate should now report installation problems from PackMan^M^J* New splash screen and association icons^M^J* Improved installer^M^J* Many bug fixes^M^J^M^JVersion 4.9.8.7^M^J* Added support for GCC > 3.2^M^J* Debug variables are now resent during next debug session^M^J* Watched Variables not in correct context are now kept and updated when it is needed^M^J* Added new compiler/linker options: ^M^J - Strip executable^M^J - Generate instructions for a specific machine (i386, i486, i586, i686, pentium, pentium-mmx, pentiumpro, pentium2, pentium3, pentium4, ^M^J k6, k6-2, k6-3, athlon, athlon-tbird, athlon-4, athlon-xp, athlon-mp, winchip-c6, winchip2, k8, c3 and c3-2)^M^J - Enable use of processor specific built-in functions (mmmx, sse, sse2, pni, 3dnow)^M^J* "Default" button in Compiler Options is back^M^J* Error messages parsing improved^M^J* Bug fixes^M^J^M^JVersion 4.9.8.5^M^J* Added the possibility to modify the value of a variable during debugging (right click on a watch variable and select "Modify value")^M^J* During Dev-C++ First Time COnfiguration window, users can now choose between using or not class browser and code completion features.^M^J* Many bug fixes^M^J^M^JVersion 4.9.8.4^M^J* Added the possibility to specify an include directory for the code completion cache to be created at Dev-C++ first startup^M^J* Improved code completion cache^M^J* WebUpdate will now backup downloaded DevPaks in Dev-C++\Packages directory, and Dev-C++ executable in devcpp.exe.BACKUP^M^J* Big speed up in function parameters listing while editing^M^J* Bug fixes^M^J^M^JVersion 4.9.8.3^M^J* On Dev-C++ first time configuration dialog, a code completion cache of all the standard ^M^J include files can now be generated.^M^J* Improved WebUpdate module^M^J* Many bug fixes^M^J^M^JVersion 4.9.8.2^M^J* New debug feature for DLLs: attach to a running process^M^J* New project option: Use custom Makefile. ^M^J* New WebUpdater module.^M^J* Allow user to specify an alternate configuration file in Environment Options ^M^J (still can be overriden by using "-c" command line parameter).^M^J* Lots of bug fixes.^M^J^M^JVersion 4.9.8.1^M^J* When creating a DLL, the created static lib respects now the project-defined output directory^M^J^M^JVersion 4.9.8.0^M^J* Changed position of compiler/linker parameters in Project Options.^M^J* Improved help file^M^J* Bug fixes^M^J^M^JVersion 4.9.7.9^M^J* Resource errors are now reported in the Resource sheet^M^J* Many bug fixes^M^J^M^JVersion 4.9.7.8^M^J* Made whole bottom report control floating instead of only debug output.^M^J* Many bug fixes^M^J^M^JVersion 4.9.7.7^M^J* Printing settings are now saved^M^J* New environment options : "watch variable under mouse" and "Report watch errors"^M^J* Bug fixes^M^J^M^JVersion 4.9.7.6^M^J* Debug variable browser^M^J* Added possibility to include in a Template the Project's directories (include, libs and ressources)^M^J* Changed tint of Class browser pictures colors to match the New Look style^M^J* Bug fixes^M^J^M^JVersion 4.9.7.5^M^J* Bug fixes^M^J^M^JVersion 4.9.7.4^M^J* When compiling with debugging symbols, an extra definition is passed to the^M^J compiler: -D__DEBUG__^M^J* Each project creates a _private.h file containing version^M^J information definitions^M^J* When compiling the current file only, no dependency checks are performed^M^J* ~300% Speed-up in class parser^M^J* Added "External programs" in Tools/Environment Options (for units "Open with")^M^J* Added "Open with" in project units context menu^M^J* Added "Classes" toolbar^M^J* Fixed pre-compilation dependency checks to work correctly^M^J* Added new file menu entry: Save Project As^M^J* Bug-fix for double quotes in devcpp.cfg file read by vUpdate^M^J* Other bug fixes^M^J^M^JVersion 4.9.7.3^M^J* When adding debugging symbols on request, remove "-s" option from linker^M^J* Compiling progress window^M^J* Environment options : "Show progress window" and "Auto-close progress , info=[ts=1254722770.692804, fuid=FL9Y0d45OI4LpS6fmh, tx_hosts={^J^I74.53.140.153^J}, rx_hosts={^J^I10.10.1.4^J}, conn_uids={^J^ICjhGID4nQcgTWjvg4c^J}, source=SMTP, depth=5, analyzers={^J^J}, mime_type=, filename=NEWS.txt, duration=0 secs, local_orig=, is_orig=F, seen_bytes=0, total_bytes=, missing_bytes=0, overflow_bytes=0, timedout=F, parent_fuid=, md5=, sha1=, sha256=, x509=, extracted=], ftp=, http=, irc=, u2_events=] - [1] mime_type: string = text/plain + [1] meta: inferred_file_metadata = [mime_type=text/plain, mime_types=[[strength=-20, mime=text/plain]]] 1254722771.858334 mime_end_entity [0] c: connection = [id=[orig_h=10.10.1.4, orig_p=1470/tcp, resp_h=74.53.140.153, resp_p=25/tcp], orig=[size=14699, state=4, num_pkts=23, num_bytes_ip=21438, flow_label=0], resp=[size=462, state=4, num_pkts=15, num_bytes_ip=1070, flow_label=0], start_time=1254722767.529046, duration=4.329288, service={^J^ISMTP^J}, addl=, hot=0, history=ShAdDa, uid=CjhGID4nQcgTWjvg4c, tunnel=, dpd=, conn=, extract_orig=F, extract_resp=F, dhcp=, dnp3=, dns=, dns_state=, ftp=, ftp_data_reuse=F, ssl=, http=, http_state=, irc=, modbus=, mysql=, radius=, rdp=, snmp=, smtp=[ts=1254722768.219663, uid=CjhGID4nQcgTWjvg4c, id=[orig_h=10.10.1.4, orig_p=1470/tcp, resp_h=74.53.140.153, resp_p=25/tcp], trans_depth=1, helo=GP, mailfrom=, rcptto={^J^I^J}, date=Mon, 5 Oct 2009 11:36:07 +0530, from="Gurpartap Singh" , to={^J^I^J}, reply_to=, msg_id=<000301ca4581$ef9e57f0$cedb07d0$@in>, in_reply_to=, subject=SMTP, x_originating_ip=, first_received=, second_received=, last_reply=354 Enter message, ending with "." on a line by itself, path=[74.53.140.153, 10.10.1.4], user_agent=Microsoft Office Outlook 12.0, tls=F, process_received_from=T, has_client_activity=T, entity=[filename=NEWS.txt], fuids=[Fel9gs4OtNEV6gUJZ5, Ft4M3f2yMvLlmwtbq9, FL9Y0d45OI4LpS6fmh]], smtp_state=[helo=GP, messages_transferred=0, pending_messages=, mime_depth=5], socks=, ssh=, syslog=] diff --git a/testing/btest/Baseline/signatures.udp-packetwise-match/out b/testing/btest/Baseline/signatures.udp-packetwise-match/out new file mode 100644 index 0000000000..5b5066d638 --- /dev/null +++ b/testing/btest/Baseline/signatures.udp-packetwise-match/out @@ -0,0 +1,6 @@ +signature match, Found .*XXXX, XXXX +signature match, Found .*YYYY, YYYY +signature match, Found XXXX, XXXX +signature match, Found YYYY, YYYY +signature match, Found ^XXXX, XXXX +signature match, Found ^YYYY, YYYY diff --git a/testing/btest/Traces/udp-signature-test.pcap b/testing/btest/Traces/udp-signature-test.pcap new file mode 100644 index 0000000000..01a880fae1 Binary files /dev/null and b/testing/btest/Traces/udp-signature-test.pcap differ diff --git a/testing/btest/doc/sphinx/include-doc_frameworks_file_analysis_02_bro.btest b/testing/btest/doc/sphinx/include-doc_frameworks_file_analysis_02_bro.btest index 5e86c8d685..f8ca8e9d1a 100644 --- a/testing/btest/doc/sphinx/include-doc_frameworks_file_analysis_02_bro.btest +++ b/testing/btest/doc/sphinx/include-doc_frameworks_file_analysis_02_bro.btest @@ -2,10 +2,11 @@ file_analysis_02.bro -event file_mime_type(f: fa_file, mime_type: string) +event file_metadata_inferred(f: fa_file, meta: inferred_file_metadata) { + if ( ! meta?$mime_type ) return; print "new file", f$id; - if ( mime_type == "text/plain" ) + if ( meta$mime_type == "text/plain" ) Files::add_analyzer(f, Files::ANALYZER_MD5); } diff --git a/testing/btest/doc/sphinx/include-doc_httpmonitor_file_extraction_bro.btest b/testing/btest/doc/sphinx/include-doc_httpmonitor_file_extraction_bro.btest index b193e4a530..4a1fe36596 100644 --- a/testing/btest/doc/sphinx/include-doc_httpmonitor_file_extraction_bro.btest +++ b/testing/btest/doc/sphinx/include-doc_httpmonitor_file_extraction_bro.btest @@ -11,15 +11,18 @@ global mime_to_ext: table[string] of string = { ["text/html"] = "html", }; -event file_mime_type(f: fa_file, mime_type: string) +event file_metadata_inferred(f: fa_file, meta: inferred_file_metadata) { if ( f$source != "HTTP" ) return; - if ( mime_type !in mime_to_ext ) + if ( ! meta?$mime_type ) return; - local fname = fmt("%s-%s.%s", f$source, f$id, mime_to_ext[mime_type]); + if ( meta$mime_type !in mime_to_ext ) + return; + + local fname = fmt("%s-%s.%s", f$source, f$id, mime_to_ext[meta$mime_type]); print fmt("Extracting file %s", fname); Files::add_analyzer(f, Files::ANALYZER_EXTRACT, [$extract_filename=fname]); - } \ No newline at end of file + } diff --git a/testing/btest/signatures/udp-packetwise-match.bro b/testing/btest/signatures/udp-packetwise-match.bro new file mode 100644 index 0000000000..706b632dd7 --- /dev/null +++ b/testing/btest/signatures/udp-packetwise-match.bro @@ -0,0 +1,53 @@ +# @TEST-EXEC: bro -r $TRACES/udp-signature-test.pcap %INPUT | sort >out +# @TEST-EXEC: btest-diff out + +@load-sigs test.sig + +@TEST-START-FILE test.sig +signature xxxx { + ip-proto = udp + payload /XXXX/ + event "Found XXXX" +} + +signature axxxx { + ip-proto = udp + payload /^XXXX/ + event "Found ^XXXX" +} + +signature sxxxx { + ip-proto = udp + payload /.*XXXX/ + event "Found .*XXXX" +} + +signature yyyy { + ip-proto = udp + payload /YYYY/ + event "Found YYYY" +} + +signature ayyyy { + ip-proto = udp + payload /^YYYY/ + event "Found ^YYYY" +} + +signature syyyy { + ip-proto = udp + payload /.*YYYY/ + event "Found .*YYYY" +} + +signature nope { + ip-proto = udp + payload /.*nope/ + event "Found .*nope" +} +@TEST-END-FILE + +event signature_match(state: signature_state, msg: string, data: string) + { + print "signature match", msg, data; + }