diff --git a/scripts/base/frameworks/file-analysis/main.bro b/scripts/base/frameworks/file-analysis/main.bro index 88a8663739..3b21ae556f 100644 --- a/scripts/base/frameworks/file-analysis/main.bro +++ b/scripts/base/frameworks/file-analysis/main.bro @@ -54,12 +54,11 @@ export { ## path which was read, or some other input source. source: string &log &optional; - ## The set of connections over which the file was transferred, - ## indicated by UID strings. - conn_uids: set[string] &log &optional; - ## The set of connections over which the file was transferred, - ## indicated by 5-tuples. - conn_ids: set[conn_id] &optional; + ## The set of connections over which the file was transferred. + conns: table[conn_id] of connection &optional; + + ## The time at which the last activity for the file was seen. + last_active: time &log; ## Number of bytes provided to the file analysis engine for the file. seen_bytes: count &log &default=0; @@ -123,6 +122,7 @@ event bro_init() &priority=5 } redef record FileAnalysis::Info += { + conn_uids: set[string] &log &optional; actions_taken: set[Action] &log &optional; extracted_files: set[string] &log &optional; md5: string &log &optional; @@ -136,6 +136,11 @@ hook FileAnalysis::policy(trig: FileAnalysis::Trigger, info: FileAnalysis::Info) if ( trig != FileAnalysis::TRIGGER_EOF && trig != FileAnalysis::TRIGGER_DONE ) return; + info$conn_uids = set(); + if ( info?$conns ) + for ( cid in info$conns ) + add info$conn_uids[info$conns[cid]$uid]; + info$actions_taken = set(); info$extracted_files = set(); diff --git a/scripts/base/protocols/http/file-extract.bro b/scripts/base/protocols/http/file-extract.bro index 466d18c3b4..079636fb6a 100644 --- a/scripts/base/protocols/http/file-extract.bro +++ b/scripts/base/protocols/http/file-extract.bro @@ -2,8 +2,7 @@ ##! the message body from the server can be extracted with this script. @load ./main -@load ./file-ident -@load base/utils/files +@load ./file-analysis module HTTP; @@ -16,45 +15,77 @@ export { redef record Info += { ## On-disk file where the response body was extracted to. - extraction_file: file &log &optional; + extraction_file: string &log &optional; ## Indicates if the response body is to be extracted or not. Must be - ## set before or by the first :bro:id:`http_entity_data` event for the - ## content. + ## set before or by the first :bro:enum:`FileAnalysis::TRIGGER_NEW` + ## for the file content. extract_file: bool &default=F; }; } -event http_entity_data(c: connection, is_orig: bool, length: count, data: string) &priority=-5 +global extract_count: count = 0; + +hook FileAnalysis::policy(trig: FileAnalysis::Trigger, info: FileAnalysis::Info) + &priority=5 { - # Client body extraction is not currently supported in this script. - if ( is_orig ) - return; - - if ( c$http$first_chunk ) + if ( trig != FileAnalysis::TRIGGER_TYPE ) return; + if ( ! info?$mime_type ) return; + if ( ! info?$source ) return; + if ( info$source != "HTTP" ) return; + if ( extract_file_types !in info$mime_type ) return; + + for ( act in info$actions ) + if ( act$act == FileAnalysis::ACTION_EXTRACT ) return; + + local fname: string = fmt("%s-%s-%d.dat", extraction_prefix, info$file_id, + extract_count); + ++extract_count; + FileAnalysis::add_action(info$file_id, [$act=FileAnalysis::ACTION_EXTRACT, + $extract_filename=fname]); + + if ( ! info?$conns ) return; + + for ( cid in info$conns ) { - if ( c$http?$mime_type && - extract_file_types in c$http$mime_type ) - { - c$http$extract_file = T; - } - + local c: connection = info$conns[cid]; + + if ( ! c?$http ) next; + + c$http$extraction_file = fname; + } + } + +hook FileAnalysis::policy(trig: FileAnalysis::Trigger, info: FileAnalysis::Info) + &priority=5 + { + if ( trig != FileAnalysis::TRIGGER_NEW ) return; + if ( ! info?$source ) return; + if ( info$source != "HTTP" ) return; + if ( ! info?$conns ) return; + + local fname: string = fmt("%s-%s-%d.dat", extraction_prefix, info$file_id, + extract_count); + local extracting: bool = F; + + for ( cid in info$conns ) + { + local c: connection = info$conns[cid]; + + if ( ! c?$http ) next; + if ( c$http$extract_file ) { - local suffix = fmt("%s_%d.dat", is_orig ? "orig" : "resp", c$http_state$current_response); - local fname = generate_extraction_filename(extraction_prefix, c, suffix); - - c$http$extraction_file = open(fname); - enable_raw_output(c$http$extraction_file); + if ( ! extracting ) + { + FileAnalysis::add_action(info$file_id, + [$act=FileAnalysis::ACTION_EXTRACT, + $extract_filename=fname]); + extracting = T; + ++extract_count; + } + + c$http$extraction_file = fname; } } - - if ( c$http?$extraction_file ) - print c$http$extraction_file, data; - } - -event http_end_entity(c: connection, is_orig: bool) - { - if ( c$http?$extraction_file ) - close(c$http$extraction_file); } diff --git a/scripts/base/protocols/http/file-hash.bro b/scripts/base/protocols/http/file-hash.bro index 2545cbf817..7659ba2b97 100644 --- a/scripts/base/protocols/http/file-hash.bro +++ b/scripts/base/protocols/http/file-hash.bro @@ -1,15 +1,11 @@ ##! Calculate hashes for HTTP body transfers. -@load ./file-ident +@load ./main +@load ./file-analysis module HTTP; export { - redef enum Notice::Type += { - ## Indicates that an MD5 sum was calculated for an HTTP response body. - MD5, - }; - redef record Info += { ## MD5 sum for a file transferred over HTTP calculated from the ## response body. @@ -19,10 +15,6 @@ export { ## if a file should have an MD5 sum generated. It must be ## set to T at the time of or before the first chunk of body data. calc_md5: bool &default=F; - - ## Indicates if an MD5 sum is being calculated for the current - ## request/response pair. - md5_handle: opaque of md5 &optional; }; ## Generate MD5 sums for these filetypes. @@ -31,62 +23,67 @@ export { &redef; } -## Initialize and calculate the hash. -event http_entity_data(c: connection, is_orig: bool, length: count, data: string) &priority=5 +hook FileAnalysis::policy(trig: FileAnalysis::Trigger, info: FileAnalysis::Info) + &priority=5 { - if ( is_orig || ! c?$http ) return; - - if ( c$http$first_chunk ) + if ( trig != FileAnalysis::TRIGGER_TYPE ) return; + if ( ! info?$mime_type ) return; + if ( ! info?$source ) return; + if ( info$source != "HTTP" ) return; + + if ( generate_md5 in info$mime_type ) + FileAnalysis::add_action(info$file_id, [$act=FileAnalysis::ACTION_MD5]); + else if ( info?$conns ) { - if ( c$http$calc_md5 || - (c$http?$mime_type && generate_md5 in c$http$mime_type) ) + for ( cid in info$conns ) { - c$http$md5_handle = md5_hash_init(); + local c: connection = info$conns[cid]; + + if ( ! c?$http ) next; + + if ( c$http$calc_md5 ) + { + FileAnalysis::add_action(info$file_id, + [$act=FileAnalysis::ACTION_MD5]); + return; + } } } - - if ( c$http?$md5_handle ) - md5_hash_update(c$http$md5_handle, data); - } - -## In the event of a content gap during a file transfer, detect the state for -## the MD5 sum calculation and stop calculating the MD5 since it would be -## incorrect anyway. -event content_gap(c: connection, is_orig: bool, seq: count, length: count) &priority=5 - { - if ( is_orig || ! c?$http || ! c$http?$md5_handle ) return; - - set_state(c, F, is_orig); - md5_hash_finish(c$http$md5_handle); # Ignore return value. - delete c$http$md5_handle; } -## When the file finishes downloading, finish the hash and generate a notice. -event http_message_done(c: connection, is_orig: bool, stat: http_message_stat) &priority=-3 +hook FileAnalysis::policy(trig: FileAnalysis::Trigger, info: FileAnalysis::Info) + &priority=5 { - if ( is_orig || ! c?$http ) return; - - if ( c$http?$md5_handle ) + if ( trig != FileAnalysis::TRIGGER_DONE && + trig != FileAnalysis::TRIGGER_EOF ) return; + if ( ! info?$source ) return; + if ( info$source != "HTTP" ) return; + if ( ! info?$conns ) return; + + local act: FileAnalysis::ActionArgs = [$act=FileAnalysis::ACTION_MD5]; + + if ( act !in info$actions ) return; + + local result = info$actions[act]; + + if ( ! result?$md5 ) return; + + for ( cid in info$conns ) { - local url = build_url_http(c$http); - c$http$md5 = md5_hash_finish(c$http$md5_handle); - delete c$http$md5_handle; - - NOTICE([$note=MD5, $msg=fmt("%s %s %s", c$id$orig_h, c$http$md5, url), - $sub=c$http$md5, $conn=c]); + local c: connection = info$conns[cid]; + + if ( ! c?$http ) next; + + c$http$md5 = result$md5; } } -event connection_state_remove(c: connection) &priority=-5 +hook FileAnalysis::policy(trig: FileAnalysis::Trigger, info: FileAnalysis::Info) + &priority=5 { - if ( c?$http_state && - c$http_state$current_response in c$http_state$pending && - c$http_state$pending[c$http_state$current_response]?$md5_handle ) - { - # The MD5 sum isn't going to be saved anywhere since the entire - # body wouldn't have been seen anyway and we'd just be giving an - # incorrect MD5 sum. - md5_hash_finish(c$http$md5_handle); - delete c$http$md5_handle; - } + if ( trig != FileAnalysis::TRIGGER_GAP ) return; + if ( ! info?$source ) return; + if ( info$source != "HTTP" ) return; + + FileAnalysis::add_action(info$file_id, [$act=FileAnalysis::ACTION_MD5]); } diff --git a/scripts/base/protocols/http/file-ident.bro b/scripts/base/protocols/http/file-ident.bro index 706ea58558..71802c3bde 100644 --- a/scripts/base/protocols/http/file-ident.bro +++ b/scripts/base/protocols/http/file-ident.bro @@ -1,15 +1,9 @@ ##! Identification of file types in HTTP response bodies with file content sniffing. -@load base/frameworks/signatures @load base/frameworks/notice @load ./main @load ./utils - -# Add the magic number signatures to the core signature set. -@load-sigs ./file-ident.sig - -# Ignore the signatures used to match files -redef Signatures::ignored_ids += /^matchfile-/; +@load ./file-analysis module HTTP; @@ -22,11 +16,6 @@ export { redef record Info += { ## Mime type of response body identified by content sniffing. mime_type: string &log &optional; - - ## Indicates that no data of the current file transfer has been - ## seen yet. After the first :bro:id:`http_entity_data` event, it - ## will be set to F. - first_chunk: bool &default=T; }; ## Mapping between mime types and regular expressions for URLs @@ -43,43 +32,34 @@ export { const ignored_incorrect_file_type_urls = /^$/ &redef; } -event signature_match(state: signature_state, msg: string, data: string) &priority=5 +hook FileAnalysis::policy(trig: FileAnalysis::Trigger, info: FileAnalysis::Info) + &priority=5 { - # Only signatures matching file types are dealt with here. - if ( /^matchfile-/ !in state$sig_id ) return; + if ( trig != FileAnalysis::TRIGGER_TYPE ) return; + if ( ! info?$mime_type ) return; + if ( ! info?$source ) return; + if ( info$source != "HTTP" ) return; + if ( ! info?$conns ) return; - local c = state$conn; - set_state(c, F, F); - - # Not much point in any of this if we don't know about the HTTP session. - if ( ! c?$http ) return; - - # Set the mime type that was detected. - c$http$mime_type = msg; - - if ( msg in mime_types_extensions && - c$http?$uri && mime_types_extensions[msg] !in c$http$uri ) + for ( cid in info$conns ) { + local c: connection = info$conns[cid]; + + if ( ! c?$http ) next; + + c$http$mime_type = info$mime_type; + + if ( info$mime_type !in mime_types_extensions ) next; + if ( ! c$http?$uri ) next; + if ( mime_types_extensions[info$mime_type] in c$http$uri ) next; + local url = build_url_http(c$http); - - if ( url == ignored_incorrect_file_type_urls ) - return; - - local message = fmt("%s %s %s", msg, c$http$method, url); + + if ( url == ignored_incorrect_file_type_urls ) next; + + local message = fmt("%s %s %s", info$mime_type, c$http$method, url); NOTICE([$note=Incorrect_File_Type, $msg=message, $conn=c]); } } - -event http_entity_data(c: connection, is_orig: bool, length: count, data: string) &priority=5 - { - if ( c$http$first_chunk && ! c$http?$mime_type ) - c$http$mime_type = split1(identify_data(data, T), /;/)[1]; - } - -event http_entity_data(c: connection, is_orig: bool, length: count, data: string) &priority=-10 - { - if ( c$http$first_chunk ) - c$http$first_chunk=F; - } diff --git a/scripts/base/protocols/http/file-ident.sig b/scripts/base/protocols/http/file-ident.sig deleted file mode 100644 index 971a32bbfc..0000000000 --- a/scripts/base/protocols/http/file-ident.sig +++ /dev/null @@ -1,144 +0,0 @@ -# These signatures are used as a replacement for libmagic. The signature -# name needs to start with "matchfile" and the "event" directive takes -# the mime type of the file matched by the http-reply-body pattern. -# -# Signatures from: http://www.garykessler.net/library/file_sigs.html - -signature matchfile-exe { - http-reply-body /\x4D\x5A/ - event "application/x-dosexec" -} - -signature matchfile-elf { - http-reply-body /\x7F\x45\x4C\x46/ - event "application/x-executable" -} - -signature matchfile-script { - # This is meant to match the interpreter declaration at the top of many - # interpreted scripts. - http-reply-body /\#\![[:blank:]]?\// - event "application/x-script" -} - -signature matchfile-wmv { - http-reply-body /\x30\x26\xB2\x75\x8E\x66\xCF\x11\xA6\xD9\x00\xAA\x00\x62\xCE\x6C/ - event "video/x-ms-wmv" -} - -signature matchfile-flv { - http-reply-body /\x46\x4C\x56\x01/ - event "video/x-flv" -} - -signature matchfile-swf { - http-reply-body /[\x46\x43]\x57\x53/ - event "application/x-shockwave-flash" -} - -signature matchfile-jar { - http-reply-body /\x5F\x27\xA8\x89/ - event "application/java-archive" -} - -signature matchfile-class { - http-reply-body /\xCA\xFE\xBA\xBE/ - event "application/java-byte-code" -} - -signature matchfile-msoffice-2007 { - # MS Office 2007 XML documents - http-reply-body /\x50\x4B\x03\x04\x14\x00\x06\x00/ - event "application/msoffice" -} - -signature matchfile-msoffice { - # Older MS Office files - http-reply-body /\xD0\xCF\x11\xE0\xA1\xB1\x1A\xE1/ - event "application/msoffice" -} - -signature matchfile-rtf { - http-reply-body /\x7B\x5C\x72\x74\x66\x31/ - event "application/rtf" -} - -signature matchfile-lnk { - http-reply-body /\x4C\x00\x00\x00\x01\x14\x02\x00\x00\x00\x00\x00\xC0\x00\x00\x00\x00\x00\x00\x46/ - event "application/x-ms-shortcut" -} - -signature matchfile-torrent { - http-reply-body /\x64\x38\x3A\x61\x6E\x6E\x6F\x75\x6E\x63\x65/ - event "application/x-bittorrent" -} - -signature matchfile-pdf { - http-reply-body /\x25\x50\x44\x46/ - event "application/pdf" -} - -signature matchfile-html { - http-reply-body /<[hH][tT][mM][lL]/ - event "text/html" -} - -signature matchfile-html2 { - http-reply-body /DataIn(data, len, Conn(), orig); + file_mgr->DataIn(data, len, GetTag(), Conn(), orig); } void IRC_Data::Undelivered(int seq, int len, bool orig) { File_Analyzer::Undelivered(seq, len, orig); - file_mgr->Gap(seq, len, Conn(), orig); + file_mgr->Gap(seq, len, GetTag(), Conn(), orig); } FTP_Data::FTP_Data(Connection* conn) @@ -103,11 +103,11 @@ void FTP_Data::Done() void FTP_Data::DeliverStream(int len, const u_char* data, bool orig) { File_Analyzer::DeliverStream(len, data, orig); - file_mgr->DataIn(data, len, Conn(), orig); + file_mgr->DataIn(data, len, GetTag(), Conn(), orig); } void FTP_Data::Undelivered(int seq, int len, bool orig) { File_Analyzer::Undelivered(seq, len, orig); - file_mgr->Gap(seq, len, Conn(), orig); + file_mgr->Gap(seq, len, GetTag(), Conn(), orig); } diff --git a/src/HTTP.cc b/src/HTTP.cc index 7110e2d1fa..5ce2dfa114 100644 --- a/src/HTTP.cc +++ b/src/HTTP.cc @@ -42,7 +42,7 @@ HTTP_Entity::HTTP_Entity(HTTP_Message *arg_message, MIME_Entity* parent_entity, expect_data_length = 0; body_length = 0; header_length = 0; - deliver_body = (http_entity_data != 0); + deliver_body = true; encoding = IDENTITY; zip = 0; is_partial_content = false; @@ -238,6 +238,11 @@ int HTTP_Entity::Undelivered(int64_t len) if ( end_of_data && in_header ) return 0; + file_mgr->Gap(body_length, len, + http_message->MyHTTP_Analyzer()->GetTag(), + http_message->MyHTTP_Analyzer()->Conn(), + http_message->IsOrig()); + if ( chunked_transfer_state != NON_CHUNKED_TRANSFER ) { if ( chunked_transfer_state == EXPECT_CHUNK_DATA && @@ -291,9 +296,11 @@ void HTTP_Entity::SubmitData(int len, const char* buf) { if ( send_size && instance_length > 0 ) file_mgr->SetSize(instance_length, + http_message->MyHTTP_Analyzer()->GetTag(), http_message->MyHTTP_Analyzer()->Conn(), http_message->IsOrig()); file_mgr->DataIn(reinterpret_cast(buf), len, offset, + http_message->MyHTTP_Analyzer()->GetTag(), http_message->MyHTTP_Analyzer()->Conn(), http_message->IsOrig()); offset += len; @@ -302,9 +309,11 @@ void HTTP_Entity::SubmitData(int len, const char* buf) { if ( send_size && content_length > 0 ) file_mgr->SetSize(content_length, + http_message->MyHTTP_Analyzer()->GetTag(), http_message->MyHTTP_Analyzer()->Conn(), http_message->IsOrig()); file_mgr->DataIn(reinterpret_cast(buf), len, + http_message->MyHTTP_Analyzer()->GetTag(), http_message->MyHTTP_Analyzer()->Conn(), http_message->IsOrig()); } @@ -554,6 +563,10 @@ void HTTP_Message::Done(const int interrupted, const char* detail) // DEBUG_MSG("%.6f HTTP message done.\n", network_time); top_level->EndOfData(); + if ( is_orig || MyHTTP_Analyzer()->HTTP_ReplyCode() != 206 ) + // multipart/byteranges may span multiple connections + file_mgr->EndOfFile(MyHTTP_Analyzer()->Conn(), is_orig); + if ( http_message_done ) { val_list* vl = new val_list; @@ -563,10 +576,6 @@ void HTTP_Message::Done(const int interrupted, const char* detail) GetAnalyzer()->ConnectionEvent(http_message_done, vl); } - if ( is_orig || MyHTTP_Analyzer()->HTTP_ReplyCode() != 206 ) - // multipart/byteranges may span multiple connections - file_mgr->EndOfFile(MyHTTP_Analyzer()->Conn(), is_orig); - MyHTTP_Analyzer()->HTTP_MessageDone(is_orig, this); delete_strings(buffers); @@ -689,9 +698,6 @@ void HTTP_Message::SubmitData(int len, const char* buf) int HTTP_Message::RequestBuffer(int* plen, char** pbuf) { - if ( ! http_entity_data ) - return 0; - if ( ! data_buffer ) if ( ! InitBuffer(mime_segment_length) ) return 0; diff --git a/src/MIME.cc b/src/MIME.cc index d5610c2bcb..b1d52a3970 100644 --- a/src/MIME.cc +++ b/src/MIME.cc @@ -1127,8 +1127,9 @@ void MIME_Mail::SubmitData(int len, const char* buf) analyzer->ConnectionEvent(mime_segment_data, vl); } + // is_orig param not available, doesn't matter as long as it's consistent file_mgr->DataIn(reinterpret_cast(buf), len, - analyzer->Conn(), false); // is_orig param N/A + analyzer->GetTag(), analyzer->Conn(), false); buffer_start = (buf + len) - (char*)data_buffer->Bytes(); } diff --git a/src/file_analysis/Info.cc b/src/file_analysis/Info.cc index 0eda8d6104..8e18795ba6 100644 --- a/src/file_analysis/Info.cc +++ b/src/file_analysis/Info.cc @@ -8,22 +8,15 @@ #include "Reporter.h" #include "Val.h" #include "Type.h" +#include "Analyzer.h" using namespace file_analysis; -static TableVal* empty_conn_id_set() +static TableVal* empty_connection_table() { - TypeList* set_index = new TypeList(conn_id); - set_index->Append(conn_id->Ref()); - return new TableVal(new SetType(set_index, 0)); - } - -static StringVal* get_conn_uid_val(Connection* conn) - { - char tmp[20]; - if ( ! conn->GetUID() ) - conn->SetUID(calculate_unique_id()); - return new StringVal(uitoa_n(conn->GetUID(), tmp, sizeof(tmp), 62)); + TypeList* tbl_index = new TypeList(conn_id); + tbl_index->Append(conn_id->Ref()); + return new TableVal(new TableType(tbl_index, connection_type->Ref())); } static RecordVal* get_conn_id_val(const Connection* conn) @@ -39,8 +32,8 @@ static RecordVal* get_conn_id_val(const Connection* conn) int Info::file_id_idx = -1; int Info::parent_file_id_idx = -1; int Info::source_idx = -1; -int Info::conn_uids_idx = -1; -int Info::conn_ids_idx = -1; +int Info::conns_idx = -1; +int Info::last_active_idx = -1; int Info::seen_bytes_idx = -1; int Info::total_bytes_idx = -1; int Info::missing_bytes_idx = -1; @@ -64,8 +57,8 @@ void Info::StaticInit() file_id_idx = Idx("file_id"); parent_file_id_idx = Idx("parent_file_id"); source_idx = Idx("source"); - conn_uids_idx = Idx("conn_uids"); - conn_ids_idx = Idx("conn_ids"); + conns_idx = Idx("conns"); + last_active_idx = Idx("last_active"); seen_bytes_idx = Idx("seen_bytes"); total_bytes_idx = Idx("total_bytes"); missing_bytes_idx = Idx("missing_bytes"); @@ -83,10 +76,9 @@ void Info::StaticInit() salt = BifConst::FileAnalysis::salt->CheckString(); } -Info::Info(const string& unique, Connection* conn) - : file_id(unique), unique(unique), val(0), last_activity_time(network_time), - postpone_timeout(false), need_reassembly(false), done(false), - actions(this) +Info::Info(const string& unique, Connection* conn, AnalyzerTag::Tag tag) + : file_id(unique), unique(unique), val(0), postpone_timeout(false), + need_reassembly(false), done(false), actions(this) { StaticInit(); @@ -106,29 +98,15 @@ Info::Info(const string& unique, Connection* conn) if ( conn ) { - // update source and connection fields - RecordVal* cval = conn->BuildConnVal(); - ListVal* services = cval->Lookup(5)->AsTableVal()->ConvertToPureList(); - Unref(cval); - string source; - - for ( int i = 0; i < services->Length(); ++i ) - { - if ( i > 0 ) - source += ", "; - source += services->Index(i)->AsStringVal()->CheckString(); - } - - Unref(services); - - if ( ! source.empty() ) - val->Assign(source_idx, new StringVal(source.c_str())); - + // add source and connection fields + val->Assign(source_idx, new StringVal(Analyzer::GetTagName(tag))); UpdateConnectionFields(conn); } else // use the unique file handle as source val->Assign(source_idx, new StringVal(unique.c_str())); + + UpdateLastActivityTime(); } Info::~Info() @@ -137,19 +115,28 @@ Info::~Info() Unref(val); } +void Info::UpdateLastActivityTime() + { + val->Assign(last_active_idx, new Val(network_time, TYPE_TIME)); + } + +double Info::GetLastActivityTime() const + { + return val->Lookup(last_active_idx)->AsTime(); + } + void Info::UpdateConnectionFields(Connection* conn) { if ( ! conn ) return; - Val* conn_uids = val->Lookup(conn_uids_idx); - Val* conn_ids = val->Lookup(conn_ids_idx); - if ( ! conn_uids ) - val->Assign(conn_uids_idx, conn_uids = new TableVal(string_set)); - if ( ! conn_ids ) - val->Assign(conn_ids_idx, conn_ids = empty_conn_id_set()); + Val* conns = val->Lookup(conns_idx); - conn_uids->AsTableVal()->Assign(get_conn_uid_val(conn), 0); - conn_ids->AsTableVal()->Assign(get_conn_id_val(conn), 0); + if ( ! conns ) + val->Assign(conns_idx, conns = empty_connection_table()); + + Val* idx = get_conn_id_val(conn); + conns->AsTableVal()->Assign(idx, conn->BuildConnVal()); + Unref(idx); } uint64 Info::LookupFieldDefaultCount(int idx) const diff --git a/src/file_analysis/Info.h b/src/file_analysis/Info.h index 2ec9efef6e..21589a54c5 100644 --- a/src/file_analysis/Info.h +++ b/src/file_analysis/Info.h @@ -5,6 +5,7 @@ #include #include +#include "AnalyzerTags.h" #include "Conn.h" #include "Val.h" #include "ActionSet.h" @@ -49,14 +50,14 @@ public: string GetUnique() const { return unique; } /** - * @return #last_activity_time + * @return value of "last_active" field in #val record; */ - double GetLastActivityTime() const { return last_activity_time; } + double GetLastActivityTime() const; /** - * Refreshes #last_activity_time with current network time. + * Refreshes "last_active" field of #val record with current network time. */ - void UpdateLastActivityTime() { last_activity_time = network_time; } + void UpdateLastActivityTime(); /** * Set "total_bytes" field of #val record to \a size. @@ -73,7 +74,7 @@ public: /** * Create a timer to be dispatched after the amount of time indicated by * the "timeout_interval" field of the #val record in order to check if - * #last_activity_time is old enough to timeout analysis of the file. + * "last_active" field is old enough to timeout analysis of the file. */ void ScheduleInactivityTimer() const; @@ -117,7 +118,8 @@ protected: /** * Constructor; only file_analysis::Manager should be creating these. */ - Info(const string& unique, Connection* conn = 0); + Info(const string& unique, Connection* conn = 0, + AnalyzerTag::Tag tag = AnalyzerTag::Error); /** * Updates the "conn_ids" and "conn_uids" fields in #val record with the @@ -156,7 +158,6 @@ protected: FileID file_id; /**< A pretty hash that likely identifies file*/ string unique; /**< A string that uniquely identifies file */ RecordVal* val; /**< \c FileAnalysis::Info from script layer. */ - double last_activity_time; /**< Time of last activity. */ bool postpone_timeout; /**< Whether postponing timeout is requested. */ bool need_reassembly; /**< Whether file stream reassembly is needed. */ bool done; /**< If this object is about to be deleted. */ @@ -192,8 +193,8 @@ public: static int file_id_idx; static int parent_file_id_idx; static int source_idx; - static int conn_uids_idx; - static int conn_ids_idx; + static int conns_idx; + static int last_active_idx; static int seen_bytes_idx; static int total_bytes_idx; static int missing_bytes_idx; diff --git a/src/file_analysis/Manager.cc b/src/file_analysis/Manager.cc index a6d2dfa3fc..3f354dd148 100644 --- a/src/file_analysis/Manager.cc +++ b/src/file_analysis/Manager.cc @@ -108,7 +108,7 @@ void Manager::Terminate() } bool Manager::DataIn(const u_char* data, uint64 len, uint64 offset, - Connection* conn, bool is_orig) + AnalyzerTag::Tag tag, Connection* conn, bool is_orig) { DrainPending(); @@ -116,12 +116,12 @@ bool Manager::DataIn(const u_char* data, uint64 len, uint64 offset, if ( ! unique.empty() ) { - DataIn(data, len, offset, GetInfo(unique, conn)); + DataIn(data, len, offset, GetInfo(unique, conn, tag)); return true; } if ( ! is_draining ) - pending.push_back(new PendingDataInChunk(data, len, offset, conn, + pending.push_back(new PendingDataInChunk(data, len, offset, tag, conn, is_orig)); return false; @@ -146,8 +146,8 @@ void Manager::DataIn(const u_char* data, uint64 len, uint64 offset, RemoveFile(info->GetUnique()); } -bool Manager::DataIn(const u_char* data, uint64 len, Connection* conn, - bool is_orig) +bool Manager::DataIn(const u_char* data, uint64 len, AnalyzerTag::Tag tag, + Connection* conn, bool is_orig) { DrainPending(); @@ -155,12 +155,13 @@ bool Manager::DataIn(const u_char* data, uint64 len, Connection* conn, if ( ! unique.empty() ) { - DataIn(data, len, GetInfo(unique, conn)); + DataIn(data, len, GetInfo(unique, conn, tag)); return true; } if ( ! is_draining ) - pending.push_back(new PendingDataInStream(data, len, conn, is_orig)); + pending.push_back(new PendingDataInStream(data, len, tag, conn, + is_orig)); return false; } @@ -212,7 +213,8 @@ void Manager::EndOfFile(const string& unique) RemoveFile(unique); } -bool Manager::Gap(uint64 offset, uint64 len, Connection* conn, bool is_orig) +bool Manager::Gap(uint64 offset, uint64 len, AnalyzerTag::Tag tag, + Connection* conn, bool is_orig) { DrainPending(); @@ -220,12 +222,12 @@ bool Manager::Gap(uint64 offset, uint64 len, Connection* conn, bool is_orig) if ( ! unique.empty() ) { - Gap(offset, len, GetInfo(unique, conn)); + Gap(offset, len, GetInfo(unique, conn, tag)); return true; } if ( ! is_draining ) - pending.push_back(new PendingGap(offset, len, conn, is_orig)); + pending.push_back(new PendingGap(offset, len, tag, conn, is_orig)); return false; } @@ -244,7 +246,8 @@ void Manager::Gap(uint64 offset, uint64 len, Info* info) info->Gap(offset, len); } -bool Manager::SetSize(uint64 size, Connection* conn, bool is_orig) +bool Manager::SetSize(uint64 size, AnalyzerTag::Tag tag, Connection* conn, + bool is_orig) { DrainPending(); @@ -252,12 +255,12 @@ bool Manager::SetSize(uint64 size, Connection* conn, bool is_orig) if ( ! unique.empty() ) { - SetSize(size, GetInfo(unique, conn)); + SetSize(size, GetInfo(unique, conn, tag)); return true; } if ( ! is_draining ) - pending.push_back(new PendingSize(size, conn, is_orig)); + pending.push_back(new PendingSize(size, tag, conn, is_orig)); return false; } @@ -326,7 +329,8 @@ bool Manager::RemoveAction(const FileID& file_id, const RecordVal* args) const return info->RemoveAction(args); } -Info* Manager::GetInfo(const string& unique, Connection* conn) +Info* Manager::GetInfo(const string& unique, Connection* conn, + AnalyzerTag::Tag tag) { if ( IsIgnored(unique) ) return 0; @@ -334,7 +338,7 @@ Info* Manager::GetInfo(const string& unique, Connection* conn) if ( ! rval ) { - rval = str_map[unique] = new Info(unique, conn); + rval = str_map[unique] = new Info(unique, conn, tag); FileID id = rval->GetFileID(); if ( id_map[id] ) diff --git a/src/file_analysis/Manager.h b/src/file_analysis/Manager.h index 6f92553c3b..5c4c4ce1d7 100644 --- a/src/file_analysis/Manager.h +++ b/src/file_analysis/Manager.h @@ -7,6 +7,7 @@ #include #include "Net.h" +#include "AnalyzerTags.h" #include "Conn.h" #include "Val.h" #include "Analyzer.h" @@ -47,7 +48,7 @@ public: * Pass in non-sequential file data. */ bool DataIn(const u_char* data, uint64 len, uint64 offset, - Connection* conn, bool is_orig); + AnalyzerTag::Tag tag, Connection* conn, bool is_orig); void DataIn(const u_char* data, uint64 len, uint64 offset, const string& unique); void DataIn(const u_char* data, uint64 len, uint64 offset, @@ -56,7 +57,8 @@ public: /** * Pass in sequential file data. */ - bool DataIn(const u_char* data, uint64 len, Connection* conn, bool is_orig); + bool DataIn(const u_char* data, uint64 len, AnalyzerTag::Tag tag, + Connection* conn, bool is_orig); void DataIn(const u_char* data, uint64 len, const string& unique); void DataIn(const u_char* data, uint64 len, Info* info); @@ -70,14 +72,16 @@ public: /** * Signal a gap in the file data stream. */ - bool Gap(uint64 offset, uint64 len, Connection* conn, bool is_orig); + bool Gap(uint64 offset, uint64 len, AnalyzerTag::Tag tag, Connection* conn, + bool is_orig); void Gap(uint64 offset, uint64 len, const string& unique); void Gap(uint64 offset, uint64 len, Info* info); /** * Provide the expected number of bytes that comprise a file. */ - bool SetSize(uint64 size, Connection* conn, bool is_orig); + bool SetSize(uint64 size, AnalyzerTag::Tag tag, Connection* conn, + bool is_orig); void SetSize(uint64 size, const string& unique); void SetSize(uint64 size, Info* info); @@ -131,7 +135,8 @@ protected: * activity time is refreshed along with any connection-related * fields. */ - Info* GetInfo(const string& unique, Connection* conn = 0); + Info* GetInfo(const string& unique, Connection* conn = 0, + AnalyzerTag::Tag tag = AnalyzerTag::Error); /** * @return a string which can uniquely identify the file being transported diff --git a/src/file_analysis/PendingFile.cc b/src/file_analysis/PendingFile.cc index 5d94c441a2..d148953294 100644 --- a/src/file_analysis/PendingFile.cc +++ b/src/file_analysis/PendingFile.cc @@ -20,8 +20,10 @@ static string conn_str(Connection* c) return rval; } -PendingFile::PendingFile(Connection* arg_conn, bool arg_is_orig) - : conn(arg_conn), is_orig(arg_is_orig), creation_time(network_time) +PendingFile::PendingFile(Connection* arg_conn, bool arg_is_orig, + AnalyzerTag::Tag arg_tag) + : conn(arg_conn), is_orig(arg_is_orig), creation_time(network_time), + tag(arg_tag) { Ref(conn); DBG_LOG(DBG_FILE_ANALYSIS, "New pending file: %s", conn_str(conn).c_str()); @@ -47,16 +49,18 @@ bool PendingFile::IsStale() const } PendingDataInChunk::PendingDataInChunk(const u_char* arg_data, uint64 arg_len, - uint64 arg_offset, Connection* arg_conn, - bool arg_is_orig) - : PendingFile(arg_conn, arg_is_orig), len(arg_len), offset(arg_offset) + uint64 arg_offset, + AnalyzerTag::Tag arg_tag, + Connection* arg_conn, bool arg_is_orig) + : PendingFile(arg_conn, arg_is_orig, arg_tag), len(arg_len), + offset(arg_offset) { copy_data(&data, arg_data, len); } bool PendingDataInChunk::Retry() const { - return file_mgr->DataIn(data, len, offset, conn, is_orig); + return file_mgr->DataIn(data, len, offset, tag, conn, is_orig); } PendingDataInChunk::~PendingDataInChunk() @@ -65,15 +69,16 @@ PendingDataInChunk::~PendingDataInChunk() } PendingDataInStream::PendingDataInStream(const u_char* arg_data, uint64 arg_len, - Connection* arg_conn, bool arg_is_orig) - : PendingFile(arg_conn, arg_is_orig), len(arg_len) + AnalyzerTag::Tag arg_tag, + Connection* arg_conn, bool arg_is_orig) + : PendingFile(arg_conn, arg_is_orig, arg_tag), len(arg_len) { copy_data(&data, arg_data, len); } bool PendingDataInStream::Retry() const { - return file_mgr->DataIn(data, len, conn, is_orig); + return file_mgr->DataIn(data, len, tag, conn, is_orig); } PendingDataInStream::~PendingDataInStream() @@ -81,15 +86,17 @@ PendingDataInStream::~PendingDataInStream() delete [] data; } -PendingGap::PendingGap(uint64 arg_offset, uint64 arg_len, Connection* arg_conn, +PendingGap::PendingGap(uint64 arg_offset, uint64 arg_len, + AnalyzerTag::Tag arg_tag, Connection* arg_conn, bool arg_is_orig) - : PendingFile(arg_conn, arg_is_orig), offset(arg_offset), len(arg_len) + : PendingFile(arg_conn, arg_is_orig, arg_tag), offset(arg_offset), + len(arg_len) { } bool PendingGap::Retry() const { - return file_mgr->Gap(offset, len, conn, is_orig); + return file_mgr->Gap(offset, len, tag, conn, is_orig); } PendingEOF::PendingEOF(Connection* arg_conn, bool arg_is_orig) @@ -102,13 +109,13 @@ bool PendingEOF::Retry() const return file_mgr->EndOfFile(conn, is_orig); } -PendingSize::PendingSize(uint64 arg_size, Connection* arg_conn, - bool arg_is_orig) - : PendingFile(arg_conn, arg_is_orig), size(arg_size) +PendingSize::PendingSize(uint64 arg_size, AnalyzerTag::Tag arg_tag, + Connection* arg_conn, bool arg_is_orig) + : PendingFile(arg_conn, arg_is_orig, arg_tag), size(arg_size) { } bool PendingSize::Retry() const { - return file_mgr->SetSize(size, conn, is_orig); + return file_mgr->SetSize(size, tag, conn, is_orig); } diff --git a/src/file_analysis/PendingFile.h b/src/file_analysis/PendingFile.h index 53e760e998..34d46fe7e9 100644 --- a/src/file_analysis/PendingFile.h +++ b/src/file_analysis/PendingFile.h @@ -1,6 +1,7 @@ #ifndef FILE_ANALYSIS_PENDINGFILE_H #define FILE_ANALYSIS_PENDINGFILE_H +#include "AnalyzerTags.h" #include "Conn.h" namespace file_analysis { @@ -16,19 +17,21 @@ public: protected: - PendingFile(Connection* arg_conn, bool arg_is_orig); + PendingFile(Connection* arg_conn, bool arg_is_orig, + AnalyzerTag::Tag arg_tag = AnalyzerTag::Error); Connection* conn; bool is_orig; double creation_time; + AnalyzerTag::Tag tag; }; class PendingDataInChunk : public PendingFile { public: PendingDataInChunk(const u_char* arg_data, uint64 arg_len, - uint64 arg_offset, Connection* arg_conn, - bool arg_is_orig); + uint64 arg_offset, AnalyzerTag::Tag tag, + Connection* arg_conn, bool arg_is_orig); virtual ~PendingDataInChunk(); @@ -45,7 +48,8 @@ class PendingDataInStream : public PendingFile { public: PendingDataInStream(const u_char* arg_data, uint64 arg_len, - Connection* arg_conn, bool arg_is_orig); + AnalyzerTag::Tag tag, Connection* arg_conn, + bool arg_is_orig); virtual ~PendingDataInStream(); @@ -60,8 +64,8 @@ protected: class PendingGap : public PendingFile { public: - PendingGap(uint64 arg_offset, uint64 arg_len, Connection* arg_conn, - bool arg_is_orig); + PendingGap(uint64 arg_offset, uint64 arg_len, AnalyzerTag::Tag tag, + Connection* arg_conn, bool arg_is_orig); virtual bool Retry() const; @@ -82,7 +86,8 @@ public: class PendingSize : public PendingFile { public: - PendingSize(uint64 arg_size, Connection* arg_conn, bool arg_is_orig); + PendingSize(uint64 arg_size, AnalyzerTag::Tag tag, Connection* arg_conn, + bool arg_is_orig); virtual bool Retry() const; diff --git a/testing/btest/Baseline/core.tunnels.ayiya/http.log b/testing/btest/Baseline/core.tunnels.ayiya/http.log index 2a97fd9b69..902d0e0fb9 100644 --- a/testing/btest/Baseline/core.tunnels.ayiya/http.log +++ b/testing/btest/Baseline/core.tunnels.ayiya/http.log @@ -3,10 +3,10 @@ #empty_field (empty) #unset_field - #path http -#open 2009-11-08-04-41-41 +#open 2013-03-22-14-38-11 #fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p trans_depth method host uri referrer user_agent request_body_len response_body_len status_code status_msg info_code info_msg filename tags username password proxied mime_type md5 extraction_file -#types time string addr port addr port count string string string string string count count count string count string string table[enum] string string table[string] string string file -1257655301.652206 5OKnoww6xl4 2001:4978:f:4c::2 53382 2001:4860:b002::68 80 1 GET ipv6.google.com / - Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10.5; en; rv:1.9.0.15pre) Gecko/2009091516 Camino/2.0b4 (like Firefox/3.0.15pre) 0 10102 200 OK - - - (empty) - - - text/html - - +#types time string addr port addr port count string string string string string count count count string count string string table[enum] string string table[string] string string string +1257655301.652206 5OKnoww6xl4 2001:4978:f:4c::2 53382 2001:4860:b002::68 80 1 GET ipv6.google.com / - Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10.5; en; rv:1.9.0.15pre) Gecko/2009091516 Camino/2.0b4 (like Firefox/3.0.15pre) 0 10102 200 OK - - - (empty) - - - text/html; charset=us-ascii - - 1257655302.514424 5OKnoww6xl4 2001:4978:f:4c::2 53382 2001:4860:b002::68 80 2 GET ipv6.google.com /csi?v=3&s=webhp&action=&tran=undefined&e=17259,19771,21517,21766,21887,22212&ei=BUz2Su7PMJTglQfz3NzCAw&rt=prt.77,xjs.565,ol.645 http://ipv6.google.com/ Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10.5; en; rv:1.9.0.15pre) Gecko/2009091516 Camino/2.0b4 (like Firefox/3.0.15pre) 0 0 204 No Content - - - (empty) - - - - - - 1257655303.603569 5OKnoww6xl4 2001:4978:f:4c::2 53382 2001:4860:b002::68 80 3 GET ipv6.google.com /gen_204?atyp=i&ct=fade&cad=1254&ei=BUz2Su7PMJTglQfz3NzCAw&zx=1257655303600 http://ipv6.google.com/ Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10.5; en; rv:1.9.0.15pre) Gecko/2009091516 Camino/2.0b4 (like Firefox/3.0.15pre) 0 0 204 No Content - - - (empty) - - - - - - -#close 2009-11-08-04-41-57 +#close 2013-03-22-14-38-11 diff --git a/testing/btest/Baseline/core.tunnels.gtp.different_dl_and_ul/http.log b/testing/btest/Baseline/core.tunnels.gtp.different_dl_and_ul/http.log index 66b17e1200..02c5cf6e63 100644 --- a/testing/btest/Baseline/core.tunnels.gtp.different_dl_and_ul/http.log +++ b/testing/btest/Baseline/core.tunnels.gtp.different_dl_and_ul/http.log @@ -3,9 +3,9 @@ #empty_field (empty) #unset_field - #path http -#open 2012-10-19-17-03-55 +#open 2013-03-22-14-37-45 #fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p trans_depth method host uri referrer user_agent request_body_len response_body_len status_code status_msg info_code info_msg filename tags username password proxied mime_type md5 extraction_file -#types time string addr port addr port count string string string string string count count count string count string string table[enum] string string table[string] string string file -1333458850.340368 arKYeMETxOg 10.131.17.170 51803 173.199.115.168 80 1 GET cdn.epicgameads.com /ads/flash/728x90_nx8com.swf?clickTAG=http://www.epicgameads.com/ads/bannerclickPage.php?id=e3ubwU6IF&pd=1&adid=0&icpc=1&axid=0&uctt=1&channel=4&cac=1&t=728x90&cb=1333458879 http://www.epicgameads.com/ads/banneriframe.php?id=e3ubwU6IF&t=728x90&channel=4&cb=1333458905296 Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; Trident/5.0) 0 31461 200 OK - - - (empty) - - - application/x-shockwave-flash - - -1333458850.399501 arKYeMETxOg 10.131.17.170 51803 173.199.115.168 80 2 GET cdn.epicgameads.com /ads/flash/728x90_nx8com.swf?clickTAG=http://www.epicgameads.com/ads/bannerclickPage.php?id=e3ubwU6IF&pd=1&adid=0&icpc=1&axid=0&uctt=1&channel=0&cac=1&t=728x90&cb=1333458881 http://www.epicgameads.com/ads/banneriframe.php?id=e3ubwU6IF&t=728x90&cb=1333458920207 Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; Trident/5.0) 0 31461 200 OK - - - (empty) - - - application/x-shockwave-flash - - -#close 2012-10-19-17-03-55 +#types time string addr port addr port count string string string string string count count count string count string string table[enum] string string table[string] string string string +1333458850.340368 arKYeMETxOg 10.131.17.170 51803 173.199.115.168 80 1 GET cdn.epicgameads.com /ads/flash/728x90_nx8com.swf?clickTAG=http://www.epicgameads.com/ads/bannerclickPage.php?id=e3ubwU6IF&pd=1&adid=0&icpc=1&axid=0&uctt=1&channel=4&cac=1&t=728x90&cb=1333458879 http://www.epicgameads.com/ads/banneriframe.php?id=e3ubwU6IF&t=728x90&channel=4&cb=1333458905296 Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; Trident/5.0) 0 31461 200 OK - - - (empty) - - - application/x-shockwave-flash; charset=binary - - +1333458850.399501 arKYeMETxOg 10.131.17.170 51803 173.199.115.168 80 2 GET cdn.epicgameads.com /ads/flash/728x90_nx8com.swf?clickTAG=http://www.epicgameads.com/ads/bannerclickPage.php?id=e3ubwU6IF&pd=1&adid=0&icpc=1&axid=0&uctt=1&channel=0&cac=1&t=728x90&cb=1333458881 http://www.epicgameads.com/ads/banneriframe.php?id=e3ubwU6IF&t=728x90&cb=1333458920207 Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; Trident/5.0) 0 31461 200 OK - - - (empty) - - - application/x-shockwave-flash; charset=binary - - +#close 2013-03-22-14-37-45 diff --git a/testing/btest/Baseline/core.tunnels.gtp.outer_ip_frag/http.log b/testing/btest/Baseline/core.tunnels.gtp.outer_ip_frag/http.log index 8a994d56af..cbe52f5252 100644 --- a/testing/btest/Baseline/core.tunnels.gtp.outer_ip_frag/http.log +++ b/testing/btest/Baseline/core.tunnels.gtp.outer_ip_frag/http.log @@ -3,8 +3,8 @@ #empty_field (empty) #unset_field - #path http -#open 2012-10-19-16-44-02 +#open 2013-03-22-14-37-46 #fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p trans_depth method host uri referrer user_agent request_body_len response_body_len status_code status_msg info_code info_msg filename tags username password proxied mime_type md5 extraction_file -#types time string addr port addr port count string string string string string count count count string count string string table[enum] string string table[string] string string file -1333458850.375568 arKYeMETxOg 10.131.47.185 1923 79.101.110.141 80 1 GET o-o.preferred.telekomrs-beg1.v2.lscache8.c.youtube.com /videoplayback?upn=MTU2MDY5NzQ5OTM0NTI3NDY4NDc&sparams=algorithm,burst,cp,factor,id,ip,ipbits,itag,source,upn,expire&fexp=912300,907210&algorithm=throttle-factor&itag=34&ip=212.0.0.0&burst=40&sver=3&signature=832FB1042E20780CFCA77A4DB5EA64AC593E8627.D1166C7E8365732E52DAFD68076DAE0146E0AE01&source=youtube&expire=1333484980&key=yt1&ipbits=8&factor=1.25&cp=U0hSSFRTUl9NSkNOMl9MTVZKOjh5eEN2SG8tZF84&id=ebf1e932d4bd1286&cm2=1 http://s.ytimg.com/yt/swfbin/watch_as3-vflqrJwOA.swf Mozilla/5.0 (Windows NT 5.1) AppleWebKit/535.11 (KHTML, like Gecko; X-SBLSP) Chrome/17.0.963.83 Safari/535.11 0 56320 206 Partial Content - - - (empty) - - - application/octet-stream - - -#close 2012-10-19-16-44-02 +#types time string addr port addr port count string string string string string count count count string count string string table[enum] string string table[string] string string string +1333458850.375568 arKYeMETxOg 10.131.47.185 1923 79.101.110.141 80 1 GET o-o.preferred.telekomrs-beg1.v2.lscache8.c.youtube.com /videoplayback?upn=MTU2MDY5NzQ5OTM0NTI3NDY4NDc&sparams=algorithm,burst,cp,factor,id,ip,ipbits,itag,source,upn,expire&fexp=912300,907210&algorithm=throttle-factor&itag=34&ip=212.0.0.0&burst=40&sver=3&signature=832FB1042E20780CFCA77A4DB5EA64AC593E8627.D1166C7E8365732E52DAFD68076DAE0146E0AE01&source=youtube&expire=1333484980&key=yt1&ipbits=8&factor=1.25&cp=U0hSSFRTUl9NSkNOMl9MTVZKOjh5eEN2SG8tZF84&id=ebf1e932d4bd1286&cm2=1 http://s.ytimg.com/yt/swfbin/watch_as3-vflqrJwOA.swf Mozilla/5.0 (Windows NT 5.1) AppleWebKit/535.11 (KHTML, like Gecko; X-SBLSP) Chrome/17.0.963.83 Safari/535.11 0 56320 206 Partial Content - - - (empty) - - - - - - +#close 2013-03-22-14-37-46 diff --git a/testing/btest/Baseline/core.tunnels.teredo/http.log b/testing/btest/Baseline/core.tunnels.teredo/http.log index c77297c58d..fe015a130d 100644 --- a/testing/btest/Baseline/core.tunnels.teredo/http.log +++ b/testing/btest/Baseline/core.tunnels.teredo/http.log @@ -3,11 +3,11 @@ #empty_field (empty) #unset_field - #path http -#open 2008-05-16-15-50-58 +#open 2013-03-22-14-37-44 #fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p trans_depth method host uri referrer user_agent request_body_len response_body_len status_code status_msg info_code info_msg filename tags username password proxied mime_type md5 extraction_file -#types time string addr port addr port count string string string string string count count count string count string string table[enum] string string table[string] string string file -1210953057.917183 3PKsZ2Uye21 192.168.2.16 1578 75.126.203.78 80 1 POST download913.avast.com /cgi-bin/iavs4stats.cgi - Syncer/4.80 (av_pro-1169;f) 589 0 204 - - - (empty) - - - text/plain - - -1210953061.585996 70MGiRM1Qf4 2001:0:4137:9e50:8000:f12a:b9c8:2815 1286 2001:4860:0:2001::68 80 1 GET ipv6.google.com / - Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.9b5) Gecko/2008032620 Firefox/3.0b5 0 6640 200 OK - - - (empty) - - - text/html - - -1210953073.381474 70MGiRM1Qf4 2001:0:4137:9e50:8000:f12a:b9c8:2815 1286 2001:4860:0:2001::68 80 2 GET ipv6.google.com /search?hl=en&q=Wireshark+!&btnG=Google+Search http://ipv6.google.com/ Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.9b5) Gecko/2008032620 Firefox/3.0b5 0 25119 200 OK - - - (empty) - - - text/html - - -1210953074.674817 c4Zw9TmAE05 192.168.2.16 1580 67.228.110.120 80 1 GET www.wireshark.org / http://ipv6.google.com/search?hl=en&q=Wireshark+%21&btnG=Google+Search Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.9b5) Gecko/2008032620 Firefox/3.0b5 0 11845 200 OK - - - (empty) - - - text/xml - - -#close 2008-05-16-15-51-16 +#types time string addr port addr port count string string string string string count count count string count string string table[enum] string string table[string] string string string +1210953057.917183 3PKsZ2Uye21 192.168.2.16 1578 75.126.203.78 80 1 POST download913.avast.com /cgi-bin/iavs4stats.cgi - Syncer/4.80 (av_pro-1169;f) 589 0 204 - - - (empty) - - - text/plain; charset=us-ascii - - +1210953061.585996 70MGiRM1Qf4 2001:0:4137:9e50:8000:f12a:b9c8:2815 1286 2001:4860:0:2001::68 80 1 GET ipv6.google.com / - Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.9b5) Gecko/2008032620 Firefox/3.0b5 0 6640 200 OK - - - (empty) - - - text/html; charset=us-ascii - - +1210953073.381474 70MGiRM1Qf4 2001:0:4137:9e50:8000:f12a:b9c8:2815 1286 2001:4860:0:2001::68 80 2 GET ipv6.google.com /search?hl=en&q=Wireshark+!&btnG=Google+Search http://ipv6.google.com/ Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.9b5) Gecko/2008032620 Firefox/3.0b5 0 25119 200 OK - - - (empty) - - - text/html; charset=us-ascii - - +1210953074.674817 c4Zw9TmAE05 192.168.2.16 1580 67.228.110.120 80 1 GET www.wireshark.org / http://ipv6.google.com/search?hl=en&q=Wireshark+%21&btnG=Google+Search Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.9b5) Gecko/2008032620 Firefox/3.0b5 0 11845 200 OK - - - (empty) - - - application/xml; charset=us-ascii - - +#close 2013-03-22-14-37-44 diff --git a/testing/btest/Baseline/core.tunnels.teredo_bubble_with_payload/http.log b/testing/btest/Baseline/core.tunnels.teredo_bubble_with_payload/http.log index e0b223d114..6d6f00a151 100644 --- a/testing/btest/Baseline/core.tunnels.teredo_bubble_with_payload/http.log +++ b/testing/btest/Baseline/core.tunnels.teredo_bubble_with_payload/http.log @@ -3,9 +3,9 @@ #empty_field (empty) #unset_field - #path http -#open 2012-06-19-17-39-37 +#open 2013-03-22-14-37-44 #fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p trans_depth method host uri referrer user_agent request_body_len response_body_len status_code status_msg info_code info_msg filename tags username password proxied mime_type md5 extraction_file -#types time string addr port addr port count string string string string string count count count string count string string table[enum] string string table[string] string string file -1340127577.361683 FrJExwHcSal 2001:0:4137:9e50:8000:f12a:b9c8:2815 1286 2001:4860:0:2001::68 80 1 GET ipv6.google.com / - Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.9b5) Gecko/2008032620 Firefox/3.0b5 0 6640 200 OK - - - (empty) - - - text/html - - -1340127577.379360 FrJExwHcSal 2001:0:4137:9e50:8000:f12a:b9c8:2815 1286 2001:4860:0:2001::68 80 2 GET ipv6.google.com /search?hl=en&q=Wireshark+!&btnG=Google+Search http://ipv6.google.com/ Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.9b5) Gecko/2008032620 Firefox/3.0b5 0 25119 200 OK - - - (empty) - - - text/html - - -#close 2012-06-19-17-39-37 +#types time string addr port addr port count string string string string string count count count string count string string table[enum] string string table[string] string string string +1340127577.361683 FrJExwHcSal 2001:0:4137:9e50:8000:f12a:b9c8:2815 1286 2001:4860:0:2001::68 80 1 GET ipv6.google.com / - Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.9b5) Gecko/2008032620 Firefox/3.0b5 0 6640 200 OK - - - (empty) - - - text/html; charset=us-ascii - - +1340127577.379360 FrJExwHcSal 2001:0:4137:9e50:8000:f12a:b9c8:2815 1286 2001:4860:0:2001::68 80 2 GET ipv6.google.com /search?hl=en&q=Wireshark+!&btnG=Google+Search http://ipv6.google.com/ Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.9b5) Gecko/2008032620 Firefox/3.0b5 0 25119 200 OK - - - (empty) - - - text/html; charset=us-ascii - - +#close 2013-03-22-14-37-44 diff --git a/testing/btest/Baseline/istate.events-ssl/receiver.http.log b/testing/btest/Baseline/istate.events-ssl/receiver.http.log index 3fc7f1b66f..aa69373171 100644 --- a/testing/btest/Baseline/istate.events-ssl/receiver.http.log +++ b/testing/btest/Baseline/istate.events-ssl/receiver.http.log @@ -3,8 +3,8 @@ #empty_field (empty) #unset_field - #path http -#open 2012-07-20-01-53-03 +#open 2013-03-22-21-05-55 #fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p trans_depth method host uri referrer user_agent request_body_len response_body_len status_code status_msg info_code info_msg filename tags username password proxied mime_type md5 extraction_file -#types time string addr port addr port count string string string string string count count count string count string string table[enum] string string table[string] string string file -1342749182.906082 arKYeMETxOg 141.42.64.125 56730 125.190.109.199 80 1 GET www.icir.org / - Wget/1.10 0 9130 200 OK - - - (empty) - - - text/html - - -#close 2012-07-20-01-53-04 +#types time string addr port addr port count string string string string string count count count string count string string table[enum] string string table[string] string string string +1363986354.505533 arKYeMETxOg 141.42.64.125 56730 125.190.109.199 80 1 GET www.icir.org / - Wget/1.10 0 9130 200 OK - - - (empty) - - - - - - +#close 2013-03-22-21-05-56 diff --git a/testing/btest/Baseline/istate.events-ssl/sender.http.log b/testing/btest/Baseline/istate.events-ssl/sender.http.log index 3fc7f1b66f..aa69373171 100644 --- a/testing/btest/Baseline/istate.events-ssl/sender.http.log +++ b/testing/btest/Baseline/istate.events-ssl/sender.http.log @@ -3,8 +3,8 @@ #empty_field (empty) #unset_field - #path http -#open 2012-07-20-01-53-03 +#open 2013-03-22-21-05-55 #fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p trans_depth method host uri referrer user_agent request_body_len response_body_len status_code status_msg info_code info_msg filename tags username password proxied mime_type md5 extraction_file -#types time string addr port addr port count string string string string string count count count string count string string table[enum] string string table[string] string string file -1342749182.906082 arKYeMETxOg 141.42.64.125 56730 125.190.109.199 80 1 GET www.icir.org / - Wget/1.10 0 9130 200 OK - - - (empty) - - - text/html - - -#close 2012-07-20-01-53-04 +#types time string addr port addr port count string string string string string count count count string count string string table[enum] string string table[string] string string string +1363986354.505533 arKYeMETxOg 141.42.64.125 56730 125.190.109.199 80 1 GET www.icir.org / - Wget/1.10 0 9130 200 OK - - - (empty) - - - - - - +#close 2013-03-22-21-05-56 diff --git a/testing/btest/Baseline/istate.events/receiver.http.log b/testing/btest/Baseline/istate.events/receiver.http.log index 6862c08b98..2531eb4bc0 100644 --- a/testing/btest/Baseline/istate.events/receiver.http.log +++ b/testing/btest/Baseline/istate.events/receiver.http.log @@ -3,8 +3,8 @@ #empty_field (empty) #unset_field - #path http -#open 2012-07-20-01-53-12 +#open 2013-03-22-21-03-17 #fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p trans_depth method host uri referrer user_agent request_body_len response_body_len status_code status_msg info_code info_msg filename tags username password proxied mime_type md5 extraction_file -#types time string addr port addr port count string string string string string count count count string count string string table[enum] string string table[string] string string file -1342749191.765740 arKYeMETxOg 141.42.64.125 56730 125.190.109.199 80 1 GET www.icir.org / - Wget/1.10 0 9130 200 OK - - - (empty) - - - text/html - - -#close 2012-07-20-01-53-13 +#types time string addr port addr port count string string string string string count count count string count string string table[enum] string string table[string] string string string +1363986197.076696 arKYeMETxOg 141.42.64.125 56730 125.190.109.199 80 1 GET www.icir.org / - Wget/1.10 0 9130 200 OK - - - (empty) - - - - - - +#close 2013-03-22-21-03-18 diff --git a/testing/btest/Baseline/istate.events/sender.http.log b/testing/btest/Baseline/istate.events/sender.http.log index 6862c08b98..2531eb4bc0 100644 --- a/testing/btest/Baseline/istate.events/sender.http.log +++ b/testing/btest/Baseline/istate.events/sender.http.log @@ -3,8 +3,8 @@ #empty_field (empty) #unset_field - #path http -#open 2012-07-20-01-53-12 +#open 2013-03-22-21-03-17 #fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p trans_depth method host uri referrer user_agent request_body_len response_body_len status_code status_msg info_code info_msg filename tags username password proxied mime_type md5 extraction_file -#types time string addr port addr port count string string string string string count count count string count string string table[enum] string string table[string] string string file -1342749191.765740 arKYeMETxOg 141.42.64.125 56730 125.190.109.199 80 1 GET www.icir.org / - Wget/1.10 0 9130 200 OK - - - (empty) - - - text/html - - -#close 2012-07-20-01-53-13 +#types time string addr port addr port count string string string string string count count count string count string string table[enum] string string table[string] string string string +1363986197.076696 arKYeMETxOg 141.42.64.125 56730 125.190.109.199 80 1 GET www.icir.org / - Wget/1.10 0 9130 200 OK - - - (empty) - - - - - - +#close 2013-03-22-21-03-18 diff --git a/testing/btest/Baseline/scripts.base.frameworks.file-analysis.actions.data_event/out b/testing/btest/Baseline/scripts.base.frameworks.file-analysis.actions.data_event/out index 934c22f049..6311f768f2 100644 --- a/testing/btest/Baseline/scripts.base.frameworks.file-analysis.actions.data_event/out +++ b/testing/btest/Baseline/scripts.base.frameworks.file-analysis.actions.data_event/out @@ -17,12 +17,7 @@ file_stream, Cx92a0ym5R8, 1024, copied source (Jon Siwek)^J^J * Small tweak to file_chunk, Cx92a0ym5R8, 1024, 3000, copied source (Jon Siwek)^J^J * Small tweak to make-release for forced git-clean. (Jon Siwek)^J^J * Fix to not let updates scripts loose their executable permissions.^J (Robin Sommer)^J^J * devel-tools/update-changes now looks for a 'release' tag to^J idenfify the stable version, and 'beta' for the beta versions.^J (Robin Sommer).^J^J * Distribution cleanup. (Robin Sommer)^J^J * New script devel-tools/make-release to create source tar balls.^J (Robin Sommer)^J^J * Removing bdcat. With the new log format, this isn't very useful^J anymore. (Robin Sommer)^J^J * Adding script that shows all pending git fastpath commits. (Robin^J Sommer)^J^J * Script to measure CPU time by loading an increasing set of^J scripts. (Robin Sommer)^J^J * extract-conn script now deals wit *.gz files. (Robin Sommer)^J^J * Tiny update to output a valid CA list file for SSL cert^J validation. (Seth Hall)^J^J * Adding "install-aux" target. Addresses #622. (Jon Siwek)^J^J * Distribution cleanup. (Jon Siwek and Robin Sommer)^J^J * FindPCAP FileAnalysis::TRIGGER_DONE Cx92a0ym5R8, 4705, 0 -{ -UWkUyAuUGXf -} -{ [orig_h=141.142.228.5, orig_p=59856/tcp, resp_h=192.150.187.43, resp_p=80/tcp] -} total bytes: 4705 source: HTTP file_stream, Cx92a0ym5R8, 476, now links against thread library when necessary (e.g.^J PF_RING's libpcap) (Jon Siwek)^J^J * Install binaries with an RPATH (Jon Siwek)^J^J * Workaround for FreeBSD CMake port missing debug flags (Jon Siwek)^J^J * Rewrite of the update-changes script. (Robin Sommer)^J^J0.1-1 | 2011-06-14 21:12:41 -0700^J^J * Add a script for generating Mozilla's CA list for the SSL analyzer.^J (Seth Hall)^J^J0.1 | 2011-04-01 16:28:22 -0700^J^J * Converting build process to CMake. (Jon Siwek)^J diff --git a/testing/btest/Baseline/scripts.base.frameworks.file-analysis.bifs.postpone_timeout/bro..stdout b/testing/btest/Baseline/scripts.base.frameworks.file-analysis.bifs.postpone_timeout/bro..stdout index 2ac9682b76..52f774b8c1 100644 --- a/testing/btest/Baseline/scripts.base.frameworks.file-analysis.bifs.postpone_timeout/bro..stdout +++ b/testing/btest/Baseline/scripts.base.frameworks.file-analysis.bifs.postpone_timeout/bro..stdout @@ -2,12 +2,7 @@ FileAnalysis::TRIGGER_NEW oDwT1BbzjM1, 0, 0 FileAnalysis::TRIGGER_DONE oDwT1BbzjM1, 1022920, 0 -{ -UWkUyAuUGXf -} -{ [orig_h=192.168.72.14, orig_p=3254/tcp, resp_h=65.54.95.206, resp_p=80/tcp] -} total bytes: 1022920 source: HTTP FileAnalysis::TRIGGER_NEW @@ -16,11 +11,6 @@ FileAnalysis::TRIGGER_TIMEOUT FileAnalysis::TRIGGER_TIMEOUT FileAnalysis::TRIGGER_EOF oDwT1BbzjM1, 206024, 0 -{ -arKYeMETxOg -} -{ [orig_h=192.168.72.14, orig_p=3257/tcp, resp_h=65.54.95.14, resp_p=80/tcp] -} total bytes: 1022920 source: HTTP diff --git a/testing/btest/Baseline/scripts.base.frameworks.file-analysis.bifs.remove_action/get.out b/testing/btest/Baseline/scripts.base.frameworks.file-analysis.bifs.remove_action/get.out index 56ce58a1f8..d0ad118ac1 100644 --- a/testing/btest/Baseline/scripts.base.frameworks.file-analysis.bifs.remove_action/get.out +++ b/testing/btest/Baseline/scripts.base.frameworks.file-analysis.bifs.remove_action/get.out @@ -8,11 +8,6 @@ file type is set mime type is set FileAnalysis::TRIGGER_DONE Cx92a0ym5R8, 4705, 0 -{ -UWkUyAuUGXf -} -{ [orig_h=141.142.228.5, orig_p=59856/tcp, resp_h=192.150.187.43, resp_p=80/tcp] -} total bytes: 4705 source: HTTP diff --git a/testing/btest/Baseline/scripts.base.frameworks.file-analysis.ftp/out b/testing/btest/Baseline/scripts.base.frameworks.file-analysis.ftp/out index 3a71ea1241..c0fb36c86e 100644 --- a/testing/btest/Baseline/scripts.base.frameworks.file-analysis.ftp/out +++ b/testing/btest/Baseline/scripts.base.frameworks.file-analysis.ftp/out @@ -8,13 +8,8 @@ file type is set mime type is set FileAnalysis::TRIGGER_EOF sidhzrR4IT8, 16557, 0 -{ -arKYeMETxOg -} -{ [orig_h=141.142.228.5, orig_p=50737/tcp, resp_h=141.142.192.162, resp_p=38141/tcp] -} -source: ftp-data +source: FTP_DATA SHA1: 44586aed07cfe19cad25076af98f535585cd5797 MD5: 7192a8075196267203adb3dfaa5c908d SHA256: 202674eba48e832690a4475113acf8b16a3f6c82c04c94b36bb2c7ce457ac8d2 diff --git a/testing/btest/Baseline/scripts.base.frameworks.file-analysis.http.get/get-gzip.out b/testing/btest/Baseline/scripts.base.frameworks.file-analysis.http.get/get-gzip.out index 8aa5329cfe..bc0eee737c 100644 --- a/testing/btest/Baseline/scripts.base.frameworks.file-analysis.http.get/get-gzip.out +++ b/testing/btest/Baseline/scripts.base.frameworks.file-analysis.http.get/get-gzip.out @@ -8,12 +8,7 @@ file type is set mime type is set FileAnalysis::TRIGGER_EOF kg59rqyYxN, 197, 0 -{ -UWkUyAuUGXf -} -{ [orig_h=141.142.228.5, orig_p=50153/tcp, resp_h=54.243.118.187, resp_p=80/tcp] -} source: HTTP SHA1: e351b8c693c3353716787c02e2923f4d12ebbb31 MD5: 5baba7eea57bc8a42a92c817ed566d72 diff --git a/testing/btest/Baseline/scripts.base.frameworks.file-analysis.http.get/get.out b/testing/btest/Baseline/scripts.base.frameworks.file-analysis.http.get/get.out index d8d895385d..34d7e942a2 100644 --- a/testing/btest/Baseline/scripts.base.frameworks.file-analysis.http.get/get.out +++ b/testing/btest/Baseline/scripts.base.frameworks.file-analysis.http.get/get.out @@ -8,12 +8,7 @@ file type is set mime type is set FileAnalysis::TRIGGER_DONE Cx92a0ym5R8, 4705, 0 -{ -UWkUyAuUGXf -} -{ [orig_h=141.142.228.5, orig_p=59856/tcp, resp_h=192.150.187.43, resp_p=80/tcp] -} total bytes: 4705 source: HTTP SHA1: 1dd7ac0398df6cbc0696445a91ec681facf4dc47 diff --git a/testing/btest/Baseline/scripts.base.frameworks.file-analysis.http.partial-content/a.out b/testing/btest/Baseline/scripts.base.frameworks.file-analysis.http.partial-content/a.out index 0ce1272201..addb0c0b4a 100644 --- a/testing/btest/Baseline/scripts.base.frameworks.file-analysis.http.partial-content/a.out +++ b/testing/btest/Baseline/scripts.base.frameworks.file-analysis.http.partial-content/a.out @@ -2,13 +2,7 @@ FileAnalysis::TRIGGER_NEW 7gZBKVUgy4l, 0, 0 FileAnalysis::TRIGGER_DONE 7gZBKVUgy4l, 555523, 0 -{ -UWkUyAuUGXf, -arKYeMETxOg -} -{ -[orig_h=10.101.84.70, orig_p=10978/tcp, resp_h=129.174.93.161, resp_p=80/tcp], +[orig_h=10.101.84.70, orig_p=10978/tcp, resp_h=129.174.93.161, resp_p=80/tcp] [orig_h=10.101.84.70, orig_p=10977/tcp, resp_h=129.174.93.161, resp_p=80/tcp] -} total bytes: 555523 source: HTTP diff --git a/testing/btest/Baseline/scripts.base.frameworks.file-analysis.http.partial-content/b.out b/testing/btest/Baseline/scripts.base.frameworks.file-analysis.http.partial-content/b.out index f88ccd0d50..c9f234c5e8 100644 --- a/testing/btest/Baseline/scripts.base.frameworks.file-analysis.http.partial-content/b.out +++ b/testing/btest/Baseline/scripts.base.frameworks.file-analysis.http.partial-content/b.out @@ -2,12 +2,7 @@ FileAnalysis::TRIGGER_NEW oDwT1BbzjM1, 0, 0 FileAnalysis::TRIGGER_DONE oDwT1BbzjM1, 1022920, 0 -{ -UWkUyAuUGXf -} -{ [orig_h=192.168.72.14, orig_p=3254/tcp, resp_h=65.54.95.206, resp_p=80/tcp] -} total bytes: 1022920 source: HTTP FileAnalysis::TRIGGER_NEW @@ -15,11 +10,6 @@ oDwT1BbzjM1, 0, 0 FileAnalysis::TRIGGER_TIMEOUT FileAnalysis::TRIGGER_EOF oDwT1BbzjM1, 206024, 0 -{ -arKYeMETxOg -} -{ [orig_h=192.168.72.14, orig_p=3257/tcp, resp_h=65.54.95.14, resp_p=80/tcp] -} total bytes: 1022920 source: HTTP diff --git a/testing/btest/Baseline/scripts.base.frameworks.file-analysis.http.partial-content/c.out b/testing/btest/Baseline/scripts.base.frameworks.file-analysis.http.partial-content/c.out index 5f4e6db178..547aa0b568 100644 --- a/testing/btest/Baseline/scripts.base.frameworks.file-analysis.http.partial-content/c.out +++ b/testing/btest/Baseline/scripts.base.frameworks.file-analysis.http.partial-content/c.out @@ -2,13 +2,7 @@ FileAnalysis::TRIGGER_NEW uHS14uhRKGe, 0, 0 FileAnalysis::TRIGGER_DONE uHS14uhRKGe, 498702, 0 -{ -UWkUyAuUGXf, -arKYeMETxOg -} -{ -[orig_h=10.45.179.94, orig_p=19950/tcp, resp_h=129.174.93.170, resp_p=80/tcp], +[orig_h=10.45.179.94, orig_p=19950/tcp, resp_h=129.174.93.170, resp_p=80/tcp] [orig_h=10.45.179.94, orig_p=19953/tcp, resp_h=129.174.93.170, resp_p=80/tcp] -} total bytes: 498668 source: HTTP diff --git a/testing/btest/Baseline/scripts.base.frameworks.file-analysis.http.pipeline/out b/testing/btest/Baseline/scripts.base.frameworks.file-analysis.http.pipeline/out index 9755b08cc1..77ca415398 100644 --- a/testing/btest/Baseline/scripts.base.frameworks.file-analysis.http.pipeline/out +++ b/testing/btest/Baseline/scripts.base.frameworks.file-analysis.http.pipeline/out @@ -8,12 +8,7 @@ file type is set mime type is set FileAnalysis::TRIGGER_EOF aFQKI8SPOL2, 2675, 0 -{ -UWkUyAuUGXf -} -{ [orig_h=192.168.1.104, orig_p=1673/tcp, resp_h=63.245.209.11, resp_p=80/tcp] -} source: HTTP SHA1: 0e42ae17eea9b074981bd3a34535ad3a22d02706 MD5: b932c3310ce47e158d1a5a42e0b01279 @@ -28,12 +23,7 @@ file type is set mime type is set FileAnalysis::TRIGGER_EOF CCU3vUEr06l, 21421, 0 -{ -UWkUyAuUGXf -} -{ [orig_h=192.168.1.104, orig_p=1673/tcp, resp_h=63.245.209.11, resp_p=80/tcp] -} source: HTTP SHA1: 8f241117afaa8ca5f41dc059e66d75c283dcc983 MD5: e732f7bf1d7cb4eedcb1661697d7bc8c @@ -48,12 +38,7 @@ file type is set mime type is set FileAnalysis::TRIGGER_DONE HCzA0dVwDPj, 94, 0 -{ -UWkUyAuUGXf -} -{ [orig_h=192.168.1.104, orig_p=1673/tcp, resp_h=63.245.209.11, resp_p=80/tcp] -} total bytes: 94 source: HTTP SHA1: 81f5f056ce5e97d940854bb0c48017b45dd9f15e @@ -69,12 +54,7 @@ file type is set mime type is set FileAnalysis::TRIGGER_DONE a1Zu1fteVEf, 2349, 0 -{ -UWkUyAuUGXf -} -{ [orig_h=192.168.1.104, orig_p=1673/tcp, resp_h=63.245.209.11, resp_p=80/tcp] -} total bytes: 2349 source: HTTP SHA1: 560eab5a0177246827a94042dd103916d8765ac7 @@ -90,12 +70,7 @@ file type is set mime type is set FileAnalysis::TRIGGER_DONE xXlF7wFdsR, 27579, 0 -{ -UWkUyAuUGXf -} -{ [orig_h=192.168.1.104, orig_p=1673/tcp, resp_h=63.245.209.11, resp_p=80/tcp] -} total bytes: 27579 source: HTTP SHA1: ee2b41bdef85de14ef332da14fc392f110b84249 diff --git a/testing/btest/Baseline/scripts.base.frameworks.file-analysis.http.post/out b/testing/btest/Baseline/scripts.base.frameworks.file-analysis.http.post/out index bc15e528a8..e6d417be47 100644 --- a/testing/btest/Baseline/scripts.base.frameworks.file-analysis.http.post/out +++ b/testing/btest/Baseline/scripts.base.frameworks.file-analysis.http.post/out @@ -8,12 +8,7 @@ file type is set mime type is set FileAnalysis::TRIGGER_DONE v5HLI7MxPQh, 11, 0 -{ -UWkUyAuUGXf -} -{ [orig_h=141.142.228.5, orig_p=53595/tcp, resp_h=54.243.55.129, resp_p=80/tcp] -} total bytes: 11 source: HTTP SHA1: 2aae6c35c94fcfb415dbe95f408b9ce91ee846ed @@ -29,12 +24,7 @@ file type is set mime type is set FileAnalysis::TRIGGER_DONE PZS1XGHkIf1, 366, 0 -{ -UWkUyAuUGXf -} -{ [orig_h=141.142.228.5, orig_p=53595/tcp, resp_h=54.243.55.129, resp_p=80/tcp] -} total bytes: 366 source: HTTP SHA1: 6a1582672c203210c6d18d700322060b676365e7 diff --git a/testing/btest/Baseline/scripts.base.frameworks.file-analysis.irc/out b/testing/btest/Baseline/scripts.base.frameworks.file-analysis.irc/out index fd515304b1..739ed1fbcc 100644 --- a/testing/btest/Baseline/scripts.base.frameworks.file-analysis.irc/out +++ b/testing/btest/Baseline/scripts.base.frameworks.file-analysis.irc/out @@ -8,13 +8,8 @@ file type is set mime type is set FileAnalysis::TRIGGER_EOF wqKMAamJVSb, 42208, 0 -{ -arKYeMETxOg -} -{ [orig_h=192.168.1.77, orig_p=57655/tcp, resp_h=209.197.168.151, resp_p=1024/tcp] -} -source: irc-dcc-data +source: IRC_DATA SHA1: 8abe0239263fd7326eb803d4465cf494f8bea218 MD5: 8c0803242f549c2780cb88b9a9215c65 SHA256: e4f0b0b9d7580e7a22dc1093c8db4df7d0115a4f3b03cc2875cc69705f0d0204 diff --git a/testing/btest/Baseline/scripts.base.frameworks.file-analysis.logging/file_analysis.log b/testing/btest/Baseline/scripts.base.frameworks.file-analysis.logging/file_analysis.log index 1d78b0dee6..f52d73e1a8 100644 --- a/testing/btest/Baseline/scripts.base.frameworks.file-analysis.logging/file_analysis.log +++ b/testing/btest/Baseline/scripts.base.frameworks.file-analysis.logging/file_analysis.log @@ -3,8 +3,8 @@ #empty_field (empty) #unset_field - #path file_analysis -#open 2013-03-20-18-29-14 -#fields file_id parent_file_id source conn_uids seen_bytes total_bytes missing_bytes overflow_bytes timeout_interval bof_buffer_size file_type mime_type actions_taken extracted_files md5 sha1 sha256 -#types string string string table[string] count count count count interval count string string table[enum] table[string] string string string -Cx92a0ym5R8 - HTTP UWkUyAuUGXf 4705 4705 0 0 120.000000 1024 set set FileAnalysis::ACTION_SHA1,FileAnalysis::ACTION_EXTRACT,FileAnalysis::ACTION_DATA_EVENT,FileAnalysis::ACTION_MD5,FileAnalysis::ACTION_SHA256 Cx92a0ym5R8-file 397168fd09991a0e712254df7bc639ac 1dd7ac0398df6cbc0696445a91ec681facf4dc47 4e7c7ef0984119447e743e3ec77e1de52713e345cde03fe7df753a35849bed18 -#close 2013-03-20-18-29-14 +#open 2013-03-22-20-24-04 +#fields file_id parent_file_id source last_active seen_bytes total_bytes missing_bytes overflow_bytes timeout_interval bof_buffer_size file_type mime_type conn_uids actions_taken extracted_files md5 sha1 sha256 +#types string string string time count count count count interval count string string table[string] table[enum] table[string] string string string +Cx92a0ym5R8 - HTTP 1362692527.009775 4705 4705 0 0 120.000000 1024 set set UWkUyAuUGXf FileAnalysis::ACTION_SHA1,FileAnalysis::ACTION_EXTRACT,FileAnalysis::ACTION_DATA_EVENT,FileAnalysis::ACTION_MD5,FileAnalysis::ACTION_SHA256 Cx92a0ym5R8-file 397168fd09991a0e712254df7bc639ac 1dd7ac0398df6cbc0696445a91ec681facf4dc47 4e7c7ef0984119447e743e3ec77e1de52713e345cde03fe7df753a35849bed18 +#close 2013-03-22-20-24-04 diff --git a/testing/btest/Baseline/scripts.base.frameworks.file-analysis.smtp/out b/testing/btest/Baseline/scripts.base.frameworks.file-analysis.smtp/out index 91093aa577..fa08f6dc18 100644 --- a/testing/btest/Baseline/scripts.base.frameworks.file-analysis.smtp/out +++ b/testing/btest/Baseline/scripts.base.frameworks.file-analysis.smtp/out @@ -8,12 +8,7 @@ file type is set mime type is set FileAnalysis::TRIGGER_EOF 9VCisPgrqVj, 79, 0 -{ -arKYeMETxOg -} -{ [orig_h=10.10.1.4, orig_p=1470/tcp, resp_h=74.53.140.153, resp_p=25/tcp] -} source: SMTP SHA1: b7e497be8a9f5e2c4b6980fceb015360f98f4a13 MD5: 92bca2e6cdcde73647125da7dccbdd07 @@ -28,12 +23,7 @@ file type is set mime type is set FileAnalysis::TRIGGER_EOF ZAOEQmRyxv1, 1918, 0 -{ -arKYeMETxOg -} -{ [orig_h=10.10.1.4, orig_p=1470/tcp, resp_h=74.53.140.153, resp_p=25/tcp] -} source: SMTP SHA1: e54af6c6616525611364b80bd6557a7ea21dae94 MD5: d194c6359c85bb88b54caee18b1e9b44 @@ -48,12 +38,7 @@ file type is set mime type is set FileAnalysis::TRIGGER_EOF Ltd7QO7jEv3, 10823, 0 -{ -arKYeMETxOg -} -{ [orig_h=10.10.1.4, orig_p=1470/tcp, resp_h=74.53.140.153, resp_p=25/tcp] -} source: SMTP SHA1: 43bf1cea1cd4b7d15243e15611859aa49d515665 MD5: a968bb0f9f9d95835b2e74c845877e87 diff --git a/testing/btest/Baseline/scripts.base.frameworks.logging.ascii-escape-odd-url/http.log b/testing/btest/Baseline/scripts.base.frameworks.logging.ascii-escape-odd-url/http.log index f1ff4db3b8..6a58e612c8 100644 --- a/testing/btest/Baseline/scripts.base.frameworks.logging.ascii-escape-odd-url/http.log +++ b/testing/btest/Baseline/scripts.base.frameworks.logging.ascii-escape-odd-url/http.log @@ -3,8 +3,8 @@ #empty_field (empty) #unset_field - #path http -#open 2011-09-12-03-57-36 +#open 2013-03-22-14-38-21 #fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p trans_depth method host uri referrer user_agent request_body_len response_body_len status_code status_msg info_code info_msg filename tags username password proxied mime_type md5 extraction_file -#types time string addr port addr port count string string string string string count count count string count string string table[enum] string string table[string] string string file -1315799856.264750 UWkUyAuUGXf 10.0.1.104 64216 193.40.5.162 80 1 GET lepo.it.da.ut.ee /~cect/teoreetilised seminarid_2010/arheoloogia_uurimisr\xfchma_seminar/Joyce et al - The Languages of Archaeology ~ Dialogue, Narrative and Writing.pdf - Wget/1.12 (darwin10.8.0) 0 346 404 Not Found - - - (empty) - - - text/html - - -#close 2011-09-12-03-57-37 +#types time string addr port addr port count string string string string string count count count string count string string table[enum] string string table[string] string string string +1315799856.264750 UWkUyAuUGXf 10.0.1.104 64216 193.40.5.162 80 1 GET lepo.it.da.ut.ee /~cect/teoreetilised seminarid_2010/arheoloogia_uurimisr\xfchma_seminar/Joyce et al - The Languages of Archaeology ~ Dialogue, Narrative and Writing.pdf - Wget/1.12 (darwin10.8.0) 0 346 404 Not Found - - - (empty) - - - text/html; charset=iso-8859-1 - - +#close 2013-03-22-14-38-21 diff --git a/testing/btest/Baseline/scripts.base.frameworks.logging.writer-path-conflict/http.log b/testing/btest/Baseline/scripts.base.frameworks.logging.writer-path-conflict/http.log index 97273995bc..5d707d5cb8 100644 --- a/testing/btest/Baseline/scripts.base.frameworks.logging.writer-path-conflict/http.log +++ b/testing/btest/Baseline/scripts.base.frameworks.logging.writer-path-conflict/http.log @@ -3,9 +3,9 @@ #empty_field (empty) #unset_field - #path http -#open 2011-03-18-19-06-08 +#open 2013-03-22-14-38-24 #fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p trans_depth method host uri referrer user_agent request_body_len response_body_len status_code status_msg info_code info_msg filename tags username password proxied mime_type md5 extraction_file -#types time string addr port addr port count string string string string string count count count string count string string table[enum] string string table[string] string string file +#types time string addr port addr port count string string string string string count count count string count string string table[enum] string string table[string] string string string 1300475168.784020 j4u32Pc5bif 141.142.220.118 48649 208.80.152.118 80 1 GET bits.wikimedia.org /skins-1.5/monobook/main.css http://www.wikipedia.org/ Mozilla/5.0 (X11; U; Linux x86_64; en-US; rv:1.9.2.15) Gecko/20110303 Ubuntu/10.04 (lucid) Firefox/3.6.15 0 0 304 Not Modified - - - (empty) - - - - - - 1300475168.916018 VW0XPVINV8a 141.142.220.118 49997 208.80.152.3 80 1 GET upload.wikimedia.org /wikipedia/commons/6/63/Wikipedia-logo.png http://www.wikipedia.org/ Mozilla/5.0 (X11; U; Linux x86_64; en-US; rv:1.9.2.15) Gecko/20110303 Ubuntu/10.04 (lucid) Firefox/3.6.15 0 0 304 Not Modified - - - (empty) - - - - - - 1300475168.916183 3PKsZ2Uye21 141.142.220.118 49996 208.80.152.3 80 1 GET upload.wikimedia.org /wikipedia/commons/thumb/b/bb/Wikipedia_wordmark.svg/174px-Wikipedia_wordmark.svg.png http://www.wikipedia.org/ Mozilla/5.0 (X11; U; Linux x86_64; en-US; rv:1.9.2.15) Gecko/20110303 Ubuntu/10.04 (lucid) Firefox/3.6.15 0 0 304 Not Modified - - - (empty) - - - - - - @@ -20,4 +20,4 @@ 1300475169.014619 Tw8jXtpTGu6 141.142.220.118 50000 208.80.152.3 80 2 GET upload.wikimedia.org /wikipedia/commons/thumb/4/4a/Commons-logo.svg/35px-Commons-logo.svg.png http://www.wikipedia.org/ Mozilla/5.0 (X11; U; Linux x86_64; en-US; rv:1.9.2.15) Gecko/20110303 Ubuntu/10.04 (lucid) Firefox/3.6.15 0 0 304 Not Modified - - - (empty) - - - - - - 1300475169.014593 P654jzLoe3a 141.142.220.118 49999 208.80.152.3 80 2 GET upload.wikimedia.org /wikipedia/commons/thumb/9/91/Wikiversity-logo.svg/35px-Wikiversity-logo.svg.png http://www.wikipedia.org/ Mozilla/5.0 (X11; U; Linux x86_64; en-US; rv:1.9.2.15) Gecko/20110303 Ubuntu/10.04 (lucid) Firefox/3.6.15 0 0 304 Not Modified - - - (empty) - - - - - - 1300475169.014927 0Q4FH8sESw5 141.142.220.118 50001 208.80.152.3 80 2 GET upload.wikimedia.org /wikipedia/commons/thumb/7/75/Wikimedia_Community_Logo.svg/35px-Wikimedia_Community_Logo.svg.png http://www.wikipedia.org/ Mozilla/5.0 (X11; U; Linux x86_64; en-US; rv:1.9.2.15) Gecko/20110303 Ubuntu/10.04 (lucid) Firefox/3.6.15 0 0 304 Not Modified - - - (empty) - - - - - - -#close 2011-03-18-19-06-13 +#close 2013-03-22-14-38-24 diff --git a/testing/btest/Baseline/scripts.base.protocols.http.100-continue/http.log b/testing/btest/Baseline/scripts.base.protocols.http.100-continue/http.log index 13c8b12502..f6920ac6b3 100644 --- a/testing/btest/Baseline/scripts.base.protocols.http.100-continue/http.log +++ b/testing/btest/Baseline/scripts.base.protocols.http.100-continue/http.log @@ -3,8 +3,8 @@ #empty_field (empty) #unset_field - #path http -#open 2009-03-19-05-21-36 +#open 2013-03-22-14-38-28 #fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p trans_depth method host uri referrer user_agent request_body_len response_body_len status_code status_msg info_code info_msg filename tags username password proxied mime_type md5 extraction_file -#types time string addr port addr port count string string string string string count count count string count string string table[enum] string string table[string] string string file -1237440095.634312 UWkUyAuUGXf 192.168.3.103 54102 128.146.216.51 80 1 POST www.osu.edu / - curl/7.17.1 (i386-apple-darwin8.11.1) libcurl/7.17.1 zlib/1.2.3 2001 60731 200 OK 100 Continue - (empty) - - - text/html - - -#close 2009-03-19-05-21-36 +#types time string addr port addr port count string string string string string count count count string count string string table[enum] string string table[string] string string string +1237440095.634312 UWkUyAuUGXf 192.168.3.103 54102 128.146.216.51 80 1 POST www.osu.edu / - curl/7.17.1 (i386-apple-darwin8.11.1) libcurl/7.17.1 zlib/1.2.3 2001 60731 200 OK 100 Continue - (empty) - - - text/html; charset=us-ascii - - +#close 2013-03-22-14-38-28 diff --git a/testing/btest/Baseline/scripts.base.protocols.http.http-extract-files/http-item_141.42.64.125:56730-125.190.109.199:80_resp_1.dat b/testing/btest/Baseline/scripts.base.protocols.http.http-extract-files/http-item-BFymS6bFgT3-0.dat similarity index 100% rename from testing/btest/Baseline/scripts.base.protocols.http.http-extract-files/http-item_141.42.64.125:56730-125.190.109.199:80_resp_1.dat rename to testing/btest/Baseline/scripts.base.protocols.http.http-extract-files/http-item-BFymS6bFgT3-0.dat diff --git a/testing/btest/Baseline/scripts.base.protocols.http.http-extract-files/http.log b/testing/btest/Baseline/scripts.base.protocols.http.http-extract-files/http.log index 0d61a6c8b3..f42a66f796 100644 --- a/testing/btest/Baseline/scripts.base.protocols.http.http-extract-files/http.log +++ b/testing/btest/Baseline/scripts.base.protocols.http.http-extract-files/http.log @@ -3,8 +3,8 @@ #empty_field (empty) #unset_field - #path http -#open 2005-10-07-23-23-56 +#open 2013-03-22-14-38-28 #fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p trans_depth method host uri referrer user_agent request_body_len response_body_len status_code status_msg info_code info_msg filename tags username password proxied mime_type md5 extraction_file -#types time string addr port addr port count string string string string string count count count string count string string table[enum] string string table[string] string string file -1128727435.634189 arKYeMETxOg 141.42.64.125 56730 125.190.109.199 80 1 GET www.icir.org / - Wget/1.10 0 9130 200 OK - - - (empty) - - - text/html - http-item_141.42.64.125:56730-125.190.109.199:80_resp_1.dat -#close 2005-10-07-23-23-57 +#types time string addr port addr port count string string string string string count count count string count string string table[enum] string string table[string] string string string +1128727435.634189 arKYeMETxOg 141.42.64.125 56730 125.190.109.199 80 1 GET www.icir.org / - Wget/1.10 0 9130 200 OK - - - (empty) - - - text/html; charset=us-ascii - http-item-BFymS6bFgT3-0.dat +#close 2013-03-22-14-38-28 diff --git a/testing/btest/Baseline/scripts.base.protocols.http.http-mime-and-md5/http.log b/testing/btest/Baseline/scripts.base.protocols.http.http-mime-and-md5/http.log index 409d8fc812..61b1e16a2f 100644 --- a/testing/btest/Baseline/scripts.base.protocols.http.http-mime-and-md5/http.log +++ b/testing/btest/Baseline/scripts.base.protocols.http.http-mime-and-md5/http.log @@ -3,12 +3,12 @@ #empty_field (empty) #unset_field - #path http -#open 2009-11-18-20-58-04 +#open 2013-03-22-16-25-59 #fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p trans_depth method host uri referrer user_agent request_body_len response_body_len status_code status_msg info_code info_msg filename tags username password proxied mime_type md5 extraction_file -#types time string addr port addr port count string string string string string count count count string count string string table[enum] string string table[string] string string file -1258577884.844956 UWkUyAuUGXf 192.168.1.104 1673 63.245.209.11 80 1 GET www.mozilla.org /style/enhanced.css http://www.mozilla.org/projects/calendar/ Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.9.1.5) Gecko/20091102 Firefox/3.5.5 0 2675 200 OK - - - (empty) - - - FAKE_MIME - - -1258577884.960135 UWkUyAuUGXf 192.168.1.104 1673 63.245.209.11 80 2 GET www.mozilla.org /script/urchin.js http://www.mozilla.org/projects/calendar/ Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.9.1.5) Gecko/20091102 Firefox/3.5.5 0 21421 200 OK - - - (empty) - - - FAKE_MIME - - -1258577885.317160 UWkUyAuUGXf 192.168.1.104 1673 63.245.209.11 80 3 GET www.mozilla.org /images/template/screen/bullet_utility.png http://www.mozilla.org/style/screen.css Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.9.1.5) Gecko/20091102 Firefox/3.5.5 0 94 200 OK - - - (empty) - - - FAKE_MIME - - -1258577885.349639 UWkUyAuUGXf 192.168.1.104 1673 63.245.209.11 80 4 GET www.mozilla.org /images/template/screen/key-point-top.png http://www.mozilla.org/style/screen.css Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.9.1.5) Gecko/20091102 Firefox/3.5.5 0 2349 200 OK - - - (empty) - - - image/png e0029eea80812e9a8e57b8d05d52938a - -1258577885.394612 UWkUyAuUGXf 192.168.1.104 1673 63.245.209.11 80 5 GET www.mozilla.org /projects/calendar/images/header-sunbird.png http://www.mozilla.org/projects/calendar/calendar.css Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.9.1.5) Gecko/20091102 Firefox/3.5.5 0 27579 200 OK - - - (empty) - - - image/png 30aa926344f58019d047e85ba049ca1e - -#close 2009-11-18-20-58-32 +#types time string addr port addr port count string string string string string count count count string count string string table[enum] string string table[string] string string string +1258577884.844956 UWkUyAuUGXf 192.168.1.104 1673 63.245.209.11 80 1 GET www.mozilla.org /style/enhanced.css http://www.mozilla.org/projects/calendar/ Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.9.1.5) Gecko/20091102 Firefox/3.5.5 0 2675 200 OK - - - (empty) - - - text/plain; charset=us-ascii - - +1258577884.960135 UWkUyAuUGXf 192.168.1.104 1673 63.245.209.11 80 2 GET www.mozilla.org /script/urchin.js http://www.mozilla.org/projects/calendar/ Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.9.1.5) Gecko/20091102 Firefox/3.5.5 0 21421 200 OK - - - (empty) - - - text/plain; charset=us-ascii - - +1258577885.317160 UWkUyAuUGXf 192.168.1.104 1673 63.245.209.11 80 3 GET www.mozilla.org /images/template/screen/bullet_utility.png http://www.mozilla.org/style/screen.css Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.9.1.5) Gecko/20091102 Firefox/3.5.5 0 94 200 OK - - - (empty) - - - image/gif; charset=binary - - +1258577885.349639 UWkUyAuUGXf 192.168.1.104 1673 63.245.209.11 80 4 GET www.mozilla.org /images/template/screen/key-point-top.png http://www.mozilla.org/style/screen.css Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.9.1.5) Gecko/20091102 Firefox/3.5.5 0 2349 200 OK - - - (empty) - - - image/png; charset=binary e0029eea80812e9a8e57b8d05d52938a - +1258577885.394612 UWkUyAuUGXf 192.168.1.104 1673 63.245.209.11 80 5 GET www.mozilla.org /projects/calendar/images/header-sunbird.png http://www.mozilla.org/projects/calendar/calendar.css Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.9.1.5) Gecko/20091102 Firefox/3.5.5 0 27579 200 OK - - - (empty) - - - image/png; charset=binary 30aa926344f58019d047e85ba049ca1e - +#close 2013-03-22-16-25-59 diff --git a/testing/btest/Baseline/scripts.base.protocols.http.http-pipelining/http.log b/testing/btest/Baseline/scripts.base.protocols.http.http-pipelining/http.log index 6b5e395902..d7791097a9 100644 --- a/testing/btest/Baseline/scripts.base.protocols.http.http-pipelining/http.log +++ b/testing/btest/Baseline/scripts.base.protocols.http.http-pipelining/http.log @@ -3,12 +3,12 @@ #empty_field (empty) #unset_field - #path http -#open 2009-11-18-20-58-04 +#open 2013-03-22-14-38-28 #fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p trans_depth method host uri referrer user_agent request_body_len response_body_len status_code status_msg info_code info_msg filename tags username password proxied md5 extraction_file -#types time string addr port addr port count string string string string string count count count string count string string table[enum] string string table[string] string file +#types time string addr port addr port count string string string string string count count count string count string string table[enum] string string table[string] string string 1258577884.844956 UWkUyAuUGXf 192.168.1.104 1673 63.245.209.11 80 1 GET www.mozilla.org /style/enhanced.css http://www.mozilla.org/projects/calendar/ Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.9.1.5) Gecko/20091102 Firefox/3.5.5 0 2675 200 OK - - - (empty) - - - - - 1258577884.960135 UWkUyAuUGXf 192.168.1.104 1673 63.245.209.11 80 2 GET www.mozilla.org /script/urchin.js http://www.mozilla.org/projects/calendar/ Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.9.1.5) Gecko/20091102 Firefox/3.5.5 0 21421 200 OK - - - (empty) - - - - - 1258577885.317160 UWkUyAuUGXf 192.168.1.104 1673 63.245.209.11 80 3 GET www.mozilla.org /images/template/screen/bullet_utility.png http://www.mozilla.org/style/screen.css Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.9.1.5) Gecko/20091102 Firefox/3.5.5 0 94 200 OK - - - (empty) - - - - - 1258577885.349639 UWkUyAuUGXf 192.168.1.104 1673 63.245.209.11 80 4 GET www.mozilla.org /images/template/screen/key-point-top.png http://www.mozilla.org/style/screen.css Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.9.1.5) Gecko/20091102 Firefox/3.5.5 0 2349 200 OK - - - (empty) - - - - - 1258577885.394612 UWkUyAuUGXf 192.168.1.104 1673 63.245.209.11 80 5 GET www.mozilla.org /projects/calendar/images/header-sunbird.png http://www.mozilla.org/projects/calendar/calendar.css Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.9.1.5) Gecko/20091102 Firefox/3.5.5 0 27579 200 OK - - - (empty) - - - - - -#close 2009-11-18-20-58-32 +#close 2013-03-22-14-38-28 diff --git a/testing/btest/istate/events-ssl.bro b/testing/btest/istate/events-ssl.bro index e4440834a7..8eca37c581 100644 --- a/testing/btest/istate/events-ssl.bro +++ b/testing/btest/istate/events-ssl.bro @@ -40,7 +40,15 @@ redef tcp_close_delay = 0secs; redef ssl_ca_certificate = "../ca_cert.pem"; redef ssl_private_key = "../bro.pem"; redef ssl_passphrase = "my-password"; - + +# File analysis that populates fields in the http.log would make the sender's +# log differ from the receiver's since hooks don't get sent to peers. +hook FileAnalysis::policy(trig: FileAnalysis::Trigger, info: FileAnalysis::Info) + &priority=10 + { + FileAnalysis::stop(info$file_id); + } + @TEST-END-FILE ############# diff --git a/testing/btest/istate/events.bro b/testing/btest/istate/events.bro index c292f77113..2e81883a1e 100644 --- a/testing/btest/istate/events.bro +++ b/testing/btest/istate/events.bro @@ -36,6 +36,14 @@ redef peer_description = "events-send"; # it gets propagated but that's ok.) redef tcp_close_delay = 0secs; +# File analysis that populates fields in the http.log would make the sender's +# log differ from the receiver's since hooks don't get sent to peers. +hook FileAnalysis::policy(trig: FileAnalysis::Trigger, info: FileAnalysis::Info) + &priority=10 + { + FileAnalysis::stop(info$file_id); + } + @TEST-END-FILE ############# diff --git a/testing/btest/scripts/base/frameworks/file-analysis/actions/data_event.bro b/testing/btest/scripts/base/frameworks/file-analysis/actions/data_event.bro index d072a05c17..cc74f5f5c2 100644 --- a/testing/btest/scripts/base/frameworks/file-analysis/actions/data_event.bro +++ b/testing/btest/scripts/base/frameworks/file-analysis/actions/data_event.bro @@ -43,8 +43,9 @@ hook FileAnalysis::policy(trig: FileAnalysis::Trigger, info: FileAnalysis::Info) case FileAnalysis::TRIGGER_DONE: print info$file_id, info$seen_bytes, info$missing_bytes; - print info$conn_uids; - print info$conn_ids; + if ( info?$conns ) + for ( cid in info$conns ) + print cid; if ( info?$total_bytes ) print "total bytes: " + fmt("%s", info$total_bytes); diff --git a/testing/btest/scripts/base/frameworks/file-analysis/bifs/postpone_timeout.bro b/testing/btest/scripts/base/frameworks/file-analysis/bifs/postpone_timeout.bro index f8f727f28b..0082f9ee11 100644 --- a/testing/btest/scripts/base/frameworks/file-analysis/bifs/postpone_timeout.bro +++ b/testing/btest/scripts/base/frameworks/file-analysis/bifs/postpone_timeout.bro @@ -55,8 +55,9 @@ hook FileAnalysis::policy(trig: FileAnalysis::Trigger, info: FileAnalysis::Info) case FileAnalysis::TRIGGER_DONE: print info$file_id, info$seen_bytes, info$missing_bytes; - print info$conn_uids; - print info$conn_ids; + if ( info?$conns ) + for ( cid in info$conns ) + print cid; if ( info?$total_bytes ) print "total bytes: " + fmt("%s", info$total_bytes); diff --git a/testing/btest/scripts/base/frameworks/file-analysis/bifs/remove_action.bro b/testing/btest/scripts/base/frameworks/file-analysis/bifs/remove_action.bro index 90285701ca..24e9e4ca66 100644 --- a/testing/btest/scripts/base/frameworks/file-analysis/bifs/remove_action.bro +++ b/testing/btest/scripts/base/frameworks/file-analysis/bifs/remove_action.bro @@ -48,8 +48,9 @@ hook FileAnalysis::policy(trig: FileAnalysis::Trigger, info: FileAnalysis::Info) case FileAnalysis::TRIGGER_DONE: print info$file_id, info$seen_bytes, info$missing_bytes; - print info$conn_uids; - print info$conn_ids; + if ( info?$conns ) + for ( cid in info$conns ) + print cid; if ( info?$total_bytes ) print "total bytes: " + fmt("%s", info$total_bytes); diff --git a/testing/btest/scripts/base/frameworks/file-analysis/bifs/stop.bro b/testing/btest/scripts/base/frameworks/file-analysis/bifs/stop.bro index d8ff623c60..53dc52455b 100644 --- a/testing/btest/scripts/base/frameworks/file-analysis/bifs/stop.bro +++ b/testing/btest/scripts/base/frameworks/file-analysis/bifs/stop.bro @@ -43,8 +43,9 @@ hook FileAnalysis::policy(trig: FileAnalysis::Trigger, info: FileAnalysis::Info) case FileAnalysis::TRIGGER_DONE: print info$file_id, info$seen_bytes, info$missing_bytes; - print info$conn_uids; - print info$conn_ids; + if ( info?$conns ) + for ( cid in info$conns ) + print cid; if ( info?$total_bytes ) print "total bytes: " + fmt("%s", info$total_bytes); diff --git a/testing/btest/scripts/base/frameworks/file-analysis/ftp.bro b/testing/btest/scripts/base/frameworks/file-analysis/ftp.bro index 153eeb186c..38665c66a2 100644 --- a/testing/btest/scripts/base/frameworks/file-analysis/ftp.bro +++ b/testing/btest/scripts/base/frameworks/file-analysis/ftp.bro @@ -12,7 +12,7 @@ hook FileAnalysis::policy(trig: FileAnalysis::Trigger, info: FileAnalysis::Info) case FileAnalysis::TRIGGER_NEW: print info$file_id, info$seen_bytes, info$missing_bytes; - if ( info$source == "ftp-data" ) + if ( info$source == "FTP_DATA" ) { for ( act in actions ) FileAnalysis::add_action(info$file_id, act); @@ -34,8 +34,9 @@ hook FileAnalysis::policy(trig: FileAnalysis::Trigger, info: FileAnalysis::Info) case FileAnalysis::TRIGGER_EOF: print info$file_id, info$seen_bytes, info$missing_bytes; - print info$conn_uids; - print info$conn_ids; + if ( info?$conns ) + for ( cid in info$conns ) + print cid; if ( info?$total_bytes ) print "total bytes: " + fmt("%s", info$total_bytes); diff --git a/testing/btest/scripts/base/frameworks/file-analysis/http/get.bro b/testing/btest/scripts/base/frameworks/file-analysis/http/get.bro index 64a01119a4..38d4c444da 100644 --- a/testing/btest/scripts/base/frameworks/file-analysis/http/get.bro +++ b/testing/btest/scripts/base/frameworks/file-analysis/http/get.bro @@ -44,8 +44,9 @@ hook FileAnalysis::policy(trig: FileAnalysis::Trigger, info: FileAnalysis::Info) case FileAnalysis::TRIGGER_DONE: print info$file_id, info$seen_bytes, info$missing_bytes; - print info$conn_uids; - print info$conn_ids; + if ( info?$conns ) + for ( cid in info$conns ) + print cid; if ( info?$total_bytes ) print "total bytes: " + fmt("%s", info$total_bytes); diff --git a/testing/btest/scripts/base/frameworks/file-analysis/http/partial-content.bro b/testing/btest/scripts/base/frameworks/file-analysis/http/partial-content.bro index 7df9db5c0d..f5a8ac11d0 100644 --- a/testing/btest/scripts/base/frameworks/file-analysis/http/partial-content.bro +++ b/testing/btest/scripts/base/frameworks/file-analysis/http/partial-content.bro @@ -54,8 +54,9 @@ hook FileAnalysis::policy(trig: FileAnalysis::Trigger, info: FileAnalysis::Info) case FileAnalysis::TRIGGER_DONE: print info$file_id, info$seen_bytes, info$missing_bytes; - print info$conn_uids; - print info$conn_ids; + if ( info?$conns ) + for ( cid in info$conns ) + print cid; if ( info?$total_bytes ) print "total bytes: " + fmt("%s", info$total_bytes); diff --git a/testing/btest/scripts/base/frameworks/file-analysis/http/pipeline.bro b/testing/btest/scripts/base/frameworks/file-analysis/http/pipeline.bro index f1932426bd..010d99c33d 100644 --- a/testing/btest/scripts/base/frameworks/file-analysis/http/pipeline.bro +++ b/testing/btest/scripts/base/frameworks/file-analysis/http/pipeline.bro @@ -45,8 +45,9 @@ hook FileAnalysis::policy(trig: FileAnalysis::Trigger, info: FileAnalysis::Info) case FileAnalysis::TRIGGER_DONE: print info$file_id, info$seen_bytes, info$missing_bytes; - print info$conn_uids; - print info$conn_ids; + if ( info?$conns ) + for ( cid in info$conns ) + print cid; if ( info?$total_bytes ) print "total bytes: " + fmt("%s", info$total_bytes); diff --git a/testing/btest/scripts/base/frameworks/file-analysis/http/post.bro b/testing/btest/scripts/base/frameworks/file-analysis/http/post.bro index 35a31a06b2..a31f7129b8 100644 --- a/testing/btest/scripts/base/frameworks/file-analysis/http/post.bro +++ b/testing/btest/scripts/base/frameworks/file-analysis/http/post.bro @@ -42,8 +42,9 @@ hook FileAnalysis::policy(trig: FileAnalysis::Trigger, info: FileAnalysis::Info) case FileAnalysis::TRIGGER_DONE: print info$file_id, info$seen_bytes, info$missing_bytes; - print info$conn_uids; - print info$conn_ids; + if ( info?$conns ) + for ( cid in info$conns ) + print cid; if ( info?$total_bytes ) print "total bytes: " + fmt("%s", info$total_bytes); diff --git a/testing/btest/scripts/base/frameworks/file-analysis/irc.bro b/testing/btest/scripts/base/frameworks/file-analysis/irc.bro index 344a2ad6ab..e6d2fc29ac 100644 --- a/testing/btest/scripts/base/frameworks/file-analysis/irc.bro +++ b/testing/btest/scripts/base/frameworks/file-analysis/irc.bro @@ -12,7 +12,7 @@ hook FileAnalysis::policy(trig: FileAnalysis::Trigger, info: FileAnalysis::Info) case FileAnalysis::TRIGGER_NEW: print info$file_id, info$seen_bytes, info$missing_bytes; - if ( info$source == "irc-dcc-data" ) + if ( info$source == "IRC_DATA" ) { for ( act in actions ) FileAnalysis::add_action(info$file_id, act); @@ -34,8 +34,9 @@ hook FileAnalysis::policy(trig: FileAnalysis::Trigger, info: FileAnalysis::Info) case FileAnalysis::TRIGGER_EOF: print info$file_id, info$seen_bytes, info$missing_bytes; - print info$conn_uids; - print info$conn_ids; + if ( info?$conns ) + for ( cid in info$conns ) + print cid; if ( info?$total_bytes ) print "total bytes: " + fmt("%s", info$total_bytes); diff --git a/testing/btest/scripts/base/frameworks/file-analysis/logging.bro b/testing/btest/scripts/base/frameworks/file-analysis/logging.bro index ab12304ac5..8f48ccb38a 100644 --- a/testing/btest/scripts/base/frameworks/file-analysis/logging.bro +++ b/testing/btest/scripts/base/frameworks/file-analysis/logging.bro @@ -55,8 +55,9 @@ hook FileAnalysis::policy(trig: FileAnalysis::Trigger, info: FileAnalysis::Info) case FileAnalysis::TRIGGER_DONE: print info$file_id, info$seen_bytes, info$missing_bytes; - print info$conn_uids; - print info$conn_ids; + if ( info?$conns ) + for ( cid in info$conns ) + print cid; if ( info?$total_bytes ) print "total bytes: " + fmt("%s", info$total_bytes); diff --git a/testing/btest/scripts/base/frameworks/file-analysis/smtp.bro b/testing/btest/scripts/base/frameworks/file-analysis/smtp.bro index 48a97f4174..57a17a5339 100644 --- a/testing/btest/scripts/base/frameworks/file-analysis/smtp.bro +++ b/testing/btest/scripts/base/frameworks/file-analysis/smtp.bro @@ -42,8 +42,9 @@ hook FileAnalysis::policy(trig: FileAnalysis::Trigger, info: FileAnalysis::Info) case FileAnalysis::TRIGGER_EOF: print info$file_id, info$seen_bytes, info$missing_bytes; - print info$conn_uids; - print info$conn_ids; + if ( info?$conns ) + for ( cid in info$conns ) + print cid; if ( info?$total_bytes ) print "total bytes: " + fmt("%s", info$total_bytes); diff --git a/testing/btest/scripts/base/protocols/http/http-extract-files.bro b/testing/btest/scripts/base/protocols/http/http-extract-files.bro index 4338cddb47..ce9d3e7e04 100644 --- a/testing/btest/scripts/base/protocols/http/http-extract-files.bro +++ b/testing/btest/scripts/base/protocols/http/http-extract-files.bro @@ -1,5 +1,5 @@ # @TEST-EXEC: bro -C -r $TRACES/web.trace %INPUT # @TEST-EXEC: btest-diff http.log -# @TEST-EXEC: btest-diff http-item_141.42.64.125:56730-125.190.109.199:80_resp_1.dat +# @TEST-EXEC: btest-diff http-item-BFymS6bFgT3-0.dat -redef HTTP::extract_file_types += /text\/html/; \ No newline at end of file +redef HTTP::extract_file_types += /text\/html/; diff --git a/testing/btest/scripts/base/protocols/http/http-mime-and-md5.bro b/testing/btest/scripts/base/protocols/http/http-mime-and-md5.bro index c6bf4899f5..b35e491b4d 100644 --- a/testing/btest/scripts/base/protocols/http/http-mime-and-md5.bro +++ b/testing/btest/scripts/base/protocols/http/http-mime-and-md5.bro @@ -1,21 +1,6 @@ -# This tests md5 calculation for a specified mime type. The http.log -# will normalize mime types other than the target type to prevent sensitivity -# to varying versions of libmagic. +# This tests md5 calculation for a specified mime type. # @TEST-EXEC: bro -r $TRACES/http/pipelined-requests.trace %INPUT > output # @TEST-EXEC: btest-diff http.log redef HTTP::generate_md5 += /image\/png/; - -event bro_init() - { - Log::remove_default_filter(HTTP::LOG); - Log::add_filter(HTTP::LOG, [$name="normalized-mime-types", - $pred=function(rec: HTTP::Info): bool - { - if ( rec?$mime_type && HTTP::generate_md5 != rec$mime_type ) - rec$mime_type = "FAKE_MIME"; - return T; - } - ]); - }