From 1ebd938bed176fbfa4f67d38cfe8af36e40c66f2 Mon Sep 17 00:00:00 2001 From: Seth Hall Date: Mon, 9 Jan 2012 12:55:24 -0500 Subject: [PATCH 1/6] Adding the unified2 analyzer. - This isn't connected to anything yet. --- src/CMakeLists.txt | 3 + src/unified2-analyzer.pac | 26 +++++++++ src/unified2-file.pac | 118 ++++++++++++++++++++++++++++++++++++++ src/unified2.pac | 21 +++++++ 4 files changed, 168 insertions(+) create mode 100644 src/unified2-analyzer.pac create mode 100644 src/unified2-file.pac create mode 100644 src/unified2.pac diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 47314514f6..8f9da667e2 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -209,6 +209,9 @@ binpac_target(ssl.pac ssl-defs.pac ssl-protocol.pac ssl-analyzer.pac) binpac_target(syslog.pac syslog-protocol.pac syslog-analyzer.pac) +binpac_target(unified2.pac + unified2-file.pac unified2-analyzer.pac) + ######################################################################## ## bro target diff --git a/src/unified2-analyzer.pac b/src/unified2-analyzer.pac new file mode 100644 index 0000000000..9f4f206562 --- /dev/null +++ b/src/unified2-analyzer.pac @@ -0,0 +1,26 @@ + +refine flow Flow += { + + %member{ + %} + + %init{ + %} + + %eof{ + %} + + %cleanup{ + %} + + function proc_ids_event(ev: IDSEvent) : bool + %{ + printf("woo!\n"); + return true; + %} +}; + + +refine typeattr IDSEvent += &let { + proc : bool = $context.flow.proc_ids_event(this); +}; diff --git a/src/unified2-file.pac b/src/unified2-file.pac new file mode 100644 index 0000000000..26b72b14dd --- /dev/null +++ b/src/unified2-file.pac @@ -0,0 +1,118 @@ + +enum Types { + EVENT = 0, + PACKET = 1, + IDS_EVENT = 2, + IDS_EVENT_IPV6 = 72, + IDS_EVENT_MPLS = 99, + IDS_EVENT_IPV6_MPLS = 100, + IDS_EVENT_VLAN = 104, + IDS_EVENT_IPV6_VLAN = 105, + EXTRA_DATA = 110, +}; + + +type Time = record { + seconds: uint32; + microseconds: uint32; +} &byteorder=bigendian; + +type v4Addr = record { + u1: uint32; +}; + +type v6Addr = record { + u1: uint32; + u2: uint32; + u3: uint32; + u4: uint32; +}; + +type Addr(ip_ver: int) = case ip_ver of { + 4 -> v4: v4Addr; + 6 -> v6: v6Addr; +} &byteorder=bigendian; + +type Record = record { + rtype: uint32; + length: uint32; + data: case rtype of { + # EVENT -> event: Event(this); + PACKET -> packet: Packet(this); + IDS_EVENT -> ids_event: LegacyIDSEvent(this, 4); + IDS_EVENT_IPV6 -> ids_event_ipv6: LegacyIDSEvent(this, 6); + # IDS_EVENT_MPLS -> ids_event_mpls: IDSEvent(this, 4); + # IDS_EVENT_IPV6_MPLS -> ids_event_ipv6_mpls: IDSEvent(this, 6); + IDS_EVENT_VLAN -> ids_event_vlan: IDSEvent(this, 4); + IDS_EVENT_IPV6_VLAN -> ids_event_ipv6_vlan: IDSEvent(this, 6); + EXTRA_DATA -> extra_data: ExtraData(this); + default -> unknown_record_type: UnknownRecordType(this); + }; +} &byteorder=bigendian &length=length+8; + +type LegacyIDSEvent(rec: Record, ip_ver: int) = record { + sensor_id: uint32; + event_id: uint32; + ts: Time; + signature_id: uint32; + generator_id: uint32; + signature_revision: uint32; + classification_id: uint32; + priority_id: uint32; + src_ip: Addr(ip_ver); + dst_ip: Addr(ip_ver); + src_p: uint16; + dst_p: uint16; + protocol: uint8; + packet_action: uint8; +}; + +type IDSEvent(rec: Record, ip_ver: int) = record { + sensor_id: uint32; + event_id: uint32; + ts: Time; + signature_id: uint32; + generator_id: uint32; + signature_revision: uint32; + classification_id: uint32; + priority_id: uint32; + src_ip: Addr(ip_ver); + dst_ip: Addr(ip_ver); + src_p: uint16; + dst_p: uint16; + protocol: uint8; + impact_flag: uint8; + impact: uint8; + blocked: uint8; + mpls_label: uint32; + vlan_id: uint16; + : uint16; +} &byteorder=bigendian; + +type Packet(rec: Record) = record { + sensor_id: uint32; + event_id: uint32; + event_second: uint32; + packet_ts: Time; + link_type: uint32; + packet_len: uint32; + packet_data: bytestring &length=packet_len; +} &byteorder=bigendian &length=rec.length; + +type ExtraData(rec: Record) = record { + sensor_id: uint32; + event_id: uint32; + event_second: uint32; + extra_type: uint32; + data_type: uint32; + blob_len: uint32; + blob: bytestring &length=blob_len; +} &byteorder=bigendian &length=rec.length; + +type UnknownRecordType(rec: Record) = record { + data: bytestring &transient &length=rec.length; +} &byteorder=bigendian &length=rec.length; + +type File = record { + alerts: Record[] &transient &until($element <= 0); +} &byteorder=bigendian; diff --git a/src/unified2.pac b/src/unified2.pac new file mode 100644 index 0000000000..86820eb9f3 --- /dev/null +++ b/src/unified2.pac @@ -0,0 +1,21 @@ + +%include binpac.pac +%include bro.pac + +analyzer Unified2 withcontext { + analyzer: Unified2_Analyzer; + flow: Flow; +}; + +analyzer Unified2_Analyzer { + downflow = Flow; + upflow = Flow; +}; + +%include unified2-file.pac + +flow Flow { + flowunit = File withcontext(connection, this); +}; + +%include unified2-analyzer.pac From 04de4ce24b1d58ad1a11d61153cc38e2eaf6b7d7 Mon Sep 17 00:00:00 2001 From: Seth Hall Date: Sat, 10 Aug 2013 22:26:32 -0400 Subject: [PATCH 2/6] Unified2 file analyzer updated to new plugin style. --- scripts/base/files/unified2/__load__.bro | 1 + scripts/base/files/unified2/main.bro | 16 ++++++++ scripts/base/init-default.bro | 2 + src/binpac_bro.h | 2 + src/file_analysis/analyzer/CMakeLists.txt | 1 + .../analyzer/unified2/CMakeLists.txt | 10 +++++ src/file_analysis/analyzer/unified2/Plugin.cc | 29 ++++++++++++++ .../analyzer/unified2/Unified2.cc | 29 ++++++++++++++ .../analyzer/unified2/Unified2.h | 40 +++++++++++++++++++ .../analyzer/unified2/events.bif | 2 + src/file_analysis/analyzer/unified2/types.bif | 1 + .../analyzer/unified2/unified2-analyzer.pac | 35 ++++++++++++++++ .../analyzer/unified2}/unified2-file.pac | 1 - .../analyzer/unified2}/unified2.pac | 2 +- src/unified2-analyzer.pac | 26 ------------ 15 files changed, 169 insertions(+), 28 deletions(-) create mode 100644 scripts/base/files/unified2/__load__.bro create mode 100644 scripts/base/files/unified2/main.bro create mode 100644 src/file_analysis/analyzer/unified2/CMakeLists.txt create mode 100644 src/file_analysis/analyzer/unified2/Plugin.cc create mode 100644 src/file_analysis/analyzer/unified2/Unified2.cc create mode 100644 src/file_analysis/analyzer/unified2/Unified2.h create mode 100644 src/file_analysis/analyzer/unified2/events.bif create mode 100644 src/file_analysis/analyzer/unified2/types.bif create mode 100644 src/file_analysis/analyzer/unified2/unified2-analyzer.pac rename src/{ => file_analysis/analyzer/unified2}/unified2-file.pac (99%) rename src/{ => file_analysis/analyzer/unified2}/unified2.pac (82%) delete mode 100644 src/unified2-analyzer.pac diff --git a/scripts/base/files/unified2/__load__.bro b/scripts/base/files/unified2/__load__.bro new file mode 100644 index 0000000000..d551be57d3 --- /dev/null +++ b/scripts/base/files/unified2/__load__.bro @@ -0,0 +1 @@ +@load ./main \ No newline at end of file diff --git a/scripts/base/files/unified2/main.bro b/scripts/base/files/unified2/main.bro new file mode 100644 index 0000000000..5e5ff17e6f --- /dev/null +++ b/scripts/base/files/unified2/main.bro @@ -0,0 +1,16 @@ + + + +event file_new(f: fa_file) + { + print "found a file"; + print f$mime_type; + print Files::add_analyzer(f, Files::ANALYZER_UNIFIED2); + } + +event unified2_alert(f: fa_file, alert: count) + { + print "yaayyaya!!!"; + + print alert; + } \ No newline at end of file diff --git a/scripts/base/init-default.bro b/scripts/base/init-default.bro index 61376c7de4..c95c03899e 100644 --- a/scripts/base/init-default.bro +++ b/scripts/base/init-default.bro @@ -55,5 +55,7 @@ @load base/files/hash @load base/files/extract +@load base/files/unified2 + @load base/misc/find-checksum-offloading diff --git a/src/binpac_bro.h b/src/binpac_bro.h index 5902c52113..8dc3df9e6f 100644 --- a/src/binpac_bro.h +++ b/src/binpac_bro.h @@ -12,6 +12,7 @@ namespace analyzer { class Analyzer; } #include "event.bif.func_h" #include "TunnelEncapsulation.h" #include "analyzer/Analyzer.h" +#include "file_analysis/Analyzer.h" #include "Conn.h" #include "binpac.h" @@ -19,6 +20,7 @@ namespace analyzer { class Analyzer; } namespace binpac { typedef analyzer::Analyzer* BroAnalyzer; +typedef file_analysis::Analyzer BroFileAnalyzer; typedef Val* BroVal; typedef PortVal* BroPortVal; typedef StringVal* BroStringVal; diff --git a/src/file_analysis/analyzer/CMakeLists.txt b/src/file_analysis/analyzer/CMakeLists.txt index bfafcd2894..1e19b7bd11 100644 --- a/src/file_analysis/analyzer/CMakeLists.txt +++ b/src/file_analysis/analyzer/CMakeLists.txt @@ -1,3 +1,4 @@ add_subdirectory(data_event) add_subdirectory(extract) add_subdirectory(hash) +add_subdirectory(unified2) diff --git a/src/file_analysis/analyzer/unified2/CMakeLists.txt b/src/file_analysis/analyzer/unified2/CMakeLists.txt new file mode 100644 index 0000000000..26a3d88ba7 --- /dev/null +++ b/src/file_analysis/analyzer/unified2/CMakeLists.txt @@ -0,0 +1,10 @@ +include(BroPlugin) + +include_directories(BEFORE ${CMAKE_CURRENT_SOURCE_DIR} + ${CMAKE_CURRENT_BINARY_DIR}) + +bro_plugin_begin(Bro Unified2) +bro_plugin_cc(Unified2.cc Plugin.cc ../../Analyzer.cc) +bro_plugin_bif(events.bif) +bro_plugin_pac(unified2.pac unified2-file.pac unified2-analyzer.pac) +bro_plugin_end() diff --git a/src/file_analysis/analyzer/unified2/Plugin.cc b/src/file_analysis/analyzer/unified2/Plugin.cc new file mode 100644 index 0000000000..ae47ad270b --- /dev/null +++ b/src/file_analysis/analyzer/unified2/Plugin.cc @@ -0,0 +1,29 @@ +#include "plugin/Plugin.h" +#include "file_analysis/Component.h" + +#include "Unified2.h" + +namespace plugin { namespace Bro_Unified2 { + +class Plugin : public plugin::Plugin { +protected: + void InitPreScript() + { + SetName("Bro::Unified2"); + SetVersion(-1); + SetAPIVersion(BRO_PLUGIN_API_VERSION); + SetDynamicPlugin(false); + + SetDescription("Analyze Unified2 alert files."); + + AddComponent(new ::file_analysis::Component("UNIFIED2", + ::file_analysis::Unified2::Instantiate)); + + extern std::list > __bif_events_init(); + AddBifInitFunction(&__bif_events_init); + } +}; + +Plugin __plugin; + +} } diff --git a/src/file_analysis/analyzer/unified2/Unified2.cc b/src/file_analysis/analyzer/unified2/Unified2.cc new file mode 100644 index 0000000000..6b0579e2d3 --- /dev/null +++ b/src/file_analysis/analyzer/unified2/Unified2.cc @@ -0,0 +1,29 @@ +// See the file "COPYING" in the main distribution directory for copyright. + +#include + +#include "Unified2.h" +#include "file_analysis/Manager.h" + +using namespace file_analysis; + +Unified2::Unified2(RecordVal* args, File* file) + : file_analysis::Analyzer(file_mgr->GetComponentTag("UNIFIED2"), args, file) + { + interp = new binpac::Unified2::Unified2_Analyzer(this); + } + +Unified2::~Unified2() + { + } + +file_analysis::Analyzer* Unified2::Instantiate(RecordVal* args, File* file) + { + return new Unified2(args, file); + } + +bool Unified2::DeliverStream(const u_char* data, uint64 len) + { + interp->NewData(true, data, data+len); + return true; + } diff --git a/src/file_analysis/analyzer/unified2/Unified2.h b/src/file_analysis/analyzer/unified2/Unified2.h new file mode 100644 index 0000000000..d87bfb2009 --- /dev/null +++ b/src/file_analysis/analyzer/unified2/Unified2.h @@ -0,0 +1,40 @@ +// See the file "COPYING" in the main distribution directory for copyright. + +#ifndef FILE_ANALYSIS_UNIFIED2_H +#define FILE_ANALYSIS_UNIFIED2_H + +#include + +#include "Val.h" +#include "File.h" +#include "Analyzer.h" +#include "unified2_pac.h" + +namespace file_analysis { + +/** + * An analyzer to extract content of files to local disk. + */ +class Unified2 : public file_analysis::Analyzer { +public: + + virtual ~Unified2(); + + virtual bool DeliverStream(const u_char* data, uint64 len); + + static file_analysis::Analyzer* Instantiate(RecordVal* args, File* file); + +protected: + + Unified2(RecordVal* args, File* file); + +private: + binpac::Unified2::Unified2_Analyzer* interp; + + string filename; + int fd; +}; + +} // namespace file_analysis + +#endif diff --git a/src/file_analysis/analyzer/unified2/events.bif b/src/file_analysis/analyzer/unified2/events.bif new file mode 100644 index 0000000000..f1c3035606 --- /dev/null +++ b/src/file_analysis/analyzer/unified2/events.bif @@ -0,0 +1,2 @@ + +event unified2_alert%(f: fa_file, alert: count%); diff --git a/src/file_analysis/analyzer/unified2/types.bif b/src/file_analysis/analyzer/unified2/types.bif new file mode 100644 index 0000000000..fb385bf962 --- /dev/null +++ b/src/file_analysis/analyzer/unified2/types.bif @@ -0,0 +1 @@ +type Unified2Alert: record; diff --git a/src/file_analysis/analyzer/unified2/unified2-analyzer.pac b/src/file_analysis/analyzer/unified2/unified2-analyzer.pac new file mode 100644 index 0000000000..4743ebb5da --- /dev/null +++ b/src/file_analysis/analyzer/unified2/unified2-analyzer.pac @@ -0,0 +1,35 @@ +%extern{ +#include "Event.h" +#include "file_analysis/File.h" +#include "events.bif.h" +%} + +refine flow Flow += { + + %member{ + %} + + %init{ + %} + + %eof{ + %} + + %cleanup{ + %} + + function proc_ids_event(ev: IDSEvent) : bool + %{ + val_list* vl = new val_list(); + vl->append(connection()->bro_analyzer()->GetFile()->GetVal()->Ref()); + vl->append(new Val(${ev.signature_id}, TYPE_COUNT)); + mgr.QueueEvent(::unified2_alert, vl, SOURCE_LOCAL); + + return true; + %} +}; + + +refine typeattr IDSEvent += &let { + proc : bool = $context.flow.proc_ids_event(this); +}; diff --git a/src/unified2-file.pac b/src/file_analysis/analyzer/unified2/unified2-file.pac similarity index 99% rename from src/unified2-file.pac rename to src/file_analysis/analyzer/unified2/unified2-file.pac index 26b72b14dd..01497aaec3 100644 --- a/src/unified2-file.pac +++ b/src/file_analysis/analyzer/unified2/unified2-file.pac @@ -11,7 +11,6 @@ enum Types { EXTRA_DATA = 110, }; - type Time = record { seconds: uint32; microseconds: uint32; diff --git a/src/unified2.pac b/src/file_analysis/analyzer/unified2/unified2.pac similarity index 82% rename from src/unified2.pac rename to src/file_analysis/analyzer/unified2/unified2.pac index 86820eb9f3..c0947d3fb5 100644 --- a/src/unified2.pac +++ b/src/file_analysis/analyzer/unified2/unified2.pac @@ -7,7 +7,7 @@ analyzer Unified2 withcontext { flow: Flow; }; -analyzer Unified2_Analyzer { +analyzer Unified2_Analyzer(bro_analyzer: BroFileAnalyzer) { downflow = Flow; upflow = Flow; }; diff --git a/src/unified2-analyzer.pac b/src/unified2-analyzer.pac deleted file mode 100644 index 9f4f206562..0000000000 --- a/src/unified2-analyzer.pac +++ /dev/null @@ -1,26 +0,0 @@ - -refine flow Flow += { - - %member{ - %} - - %init{ - %} - - %eof{ - %} - - %cleanup{ - %} - - function proc_ids_event(ev: IDSEvent) : bool - %{ - printf("woo!\n"); - return true; - %} -}; - - -refine typeattr IDSEvent += &let { - proc : bool = $context.flow.proc_ids_event(this); -}; From 091c8f3ebc5b032b73af7b24ad70c7ba68761f63 Mon Sep 17 00:00:00 2001 From: Seth Hall Date: Mon, 12 Aug 2013 14:57:12 -0400 Subject: [PATCH 3/6] Working unified2 analyzer. - No output by default yet. Most of the activity is centered around generating the Unified2::alert event which ties together an IDSEvent and a packet. --- scripts/base/files/unified2/main.bro | 69 ++++++++- scripts/base/init-bare.bro | 33 ++++ scripts/base/utils/paths.bro | 5 +- .../analyzer/unified2/CMakeLists.txt | 2 +- src/file_analysis/analyzer/unified2/Plugin.cc | 4 + .../analyzer/unified2/Unified2.cc | 14 +- .../analyzer/unified2/events.bif | 7 +- src/file_analysis/analyzer/unified2/types.bif | 3 +- .../analyzer/unified2/unified2-analyzer.pac | 142 +++++++++++++++++- .../analyzer/unified2/unified2-file.pac | 70 +++------ .../analyzer/unified2/unified2.pac | 2 +- 11 files changed, 282 insertions(+), 69 deletions(-) diff --git a/scripts/base/files/unified2/main.bro b/scripts/base/files/unified2/main.bro index 5e5ff17e6f..25eef30d72 100644 --- a/scripts/base/files/unified2/main.bro +++ b/scripts/base/files/unified2/main.bro @@ -1,16 +1,71 @@ +@load base/utils/dir +@load base/utils/paths +module Unified2; + +export { + ## Directory to watch for Unified2 files. + const watch_file = "" &redef; + + ## File to watch for Unified2 records. + const watch_dir = "" &redef; + + ## Reconstructed "alert" which combines related events + ## and packets. + global alert: event(f: fa_file, ev: Unified2::IDSEvent, pkt: Unified2::Packet); + + type Info: record { + current_event: Unified2::IDSEvent &optional; + }; + + redef record fa_file += { + unified2: Info &optional; + }; +} + +event bro_init() + { + if ( watch_dir != "" ) + { + Dir::monitor(watch_dir, function(fname: string) + { + Input::add_analysis([$source=fname, + $reader=Input::READER_BINARY, + $mode=Input::MANUAL, + $name=fname]); + }, 10secs); + } + + if ( watch_file != "" ) + { + Input::add_analysis([$source=watch_file, + $reader=Input::READER_BINARY, + $mode=Input::MANUAL, + $name=watch_file]); + } + } event file_new(f: fa_file) { - print "found a file"; - print f$mime_type; - print Files::add_analyzer(f, Files::ANALYZER_UNIFIED2); + if ( f$source in watch_file || + compress_path(extract_path(f$source)) == compress_path(watch_dir) ) + { + Files::add_analyzer(f, Files::ANALYZER_UNIFIED2); + f$unified2 = Info(); + } } -event unified2_alert(f: fa_file, alert: count) +event unified2_event(f: fa_file, ev: Unified2::IDSEvent) { - print "yaayyaya!!!"; + f$unified2$current_event = ev; + } - print alert; - } \ No newline at end of file +event unified2_packet(f: fa_file, pkt: Unified2::Packet) + { + if ( f$unified2?$current_event ) + { + event Unified2::alert(f, f$unified2$current_event, pkt); + delete f$unified2$current_event; + } + } diff --git a/scripts/base/init-bare.bro b/scripts/base/init-bare.bro index ea79fb23a0..6d354c90bf 100644 --- a/scripts/base/init-bare.bro +++ b/scripts/base/init-bare.bro @@ -2686,6 +2686,39 @@ type ModbusHeaders: record { function_code: count; }; +module Unified2; +export { + type Unified2::IDSEvent: record { + sensor_id: count; + event_id: count; + ts: time; + signature_id: count; + generator_id: count; + signature_revision: count; + classification_id: count; + priority_id: count; + src_ip: addr; + dst_ip: addr; + src_p: port; + dst_p: port; + impact_flag: count; + impact: count; + blocked: count; + mpls_label: count &optional; ## Not available in "legacy" IDS events. + vlan_id: count &optional; ## Not available in "legacy" IDS events. + packet_action: count &optional; ## Only available in "legacy" IDS events. + }; + + type Unified2::Packet: record { + sensor_id: count; + event_id: count; + event_second: count; + packet_ts: time; + link_type: count; + packet_data: string; + }; +} + module SOCKS; export { ## This record is for a SOCKS client or server to provide either a diff --git a/scripts/base/utils/paths.bro b/scripts/base/utils/paths.bro index f8ad384ea7..04976c2b2b 100644 --- a/scripts/base/utils/paths.bro +++ b/scripts/base/utils/paths.bro @@ -9,13 +9,12 @@ const absolute_path_pat = /(\/|[A-Za-z]:[\\\/]).*/; ## Returns: the first absolute path found in input string, else an empty string function extract_path(input: string): string { - const dir_pattern = /(\/|[A-Za-z]:[\\\/])([^\"\ ]|(\\\ ))*/; + local dir_pattern = /([^\\\/]+)$/; local parts = split_all(input, dir_pattern); - if ( |parts| < 3 ) return ""; - return parts[2]; + return parts[1]; } ## Compresses a given path by removing '..'s and the parent directory it diff --git a/src/file_analysis/analyzer/unified2/CMakeLists.txt b/src/file_analysis/analyzer/unified2/CMakeLists.txt index 26a3d88ba7..9d831de936 100644 --- a/src/file_analysis/analyzer/unified2/CMakeLists.txt +++ b/src/file_analysis/analyzer/unified2/CMakeLists.txt @@ -5,6 +5,6 @@ include_directories(BEFORE ${CMAKE_CURRENT_SOURCE_DIR} bro_plugin_begin(Bro Unified2) bro_plugin_cc(Unified2.cc Plugin.cc ../../Analyzer.cc) -bro_plugin_bif(events.bif) +bro_plugin_bif(events.bif types.bif) bro_plugin_pac(unified2.pac unified2-file.pac unified2-analyzer.pac) bro_plugin_end() diff --git a/src/file_analysis/analyzer/unified2/Plugin.cc b/src/file_analysis/analyzer/unified2/Plugin.cc index ae47ad270b..cbc5fa4b0a 100644 --- a/src/file_analysis/analyzer/unified2/Plugin.cc +++ b/src/file_analysis/analyzer/unified2/Plugin.cc @@ -1,4 +1,5 @@ #include "plugin/Plugin.h" + #include "file_analysis/Component.h" #include "Unified2.h" @@ -21,6 +22,9 @@ protected: extern std::list > __bif_events_init(); AddBifInitFunction(&__bif_events_init); + + extern std::list > __bif_types_init(); + AddBifInitFunction(&__bif_types_init); } }; diff --git a/src/file_analysis/analyzer/unified2/Unified2.cc b/src/file_analysis/analyzer/unified2/Unified2.cc index 6b0579e2d3..dac60daa37 100644 --- a/src/file_analysis/analyzer/unified2/Unified2.cc +++ b/src/file_analysis/analyzer/unified2/Unified2.cc @@ -1,7 +1,5 @@ // See the file "COPYING" in the main distribution directory for copyright. -#include - #include "Unified2.h" #include "file_analysis/Manager.h" @@ -15,6 +13,7 @@ Unified2::Unified2(RecordVal* args, File* file) Unified2::~Unified2() { + delete interp; } file_analysis::Analyzer* Unified2::Instantiate(RecordVal* args, File* file) @@ -24,6 +23,15 @@ file_analysis::Analyzer* Unified2::Instantiate(RecordVal* args, File* file) bool Unified2::DeliverStream(const u_char* data, uint64 len) { - interp->NewData(true, data, data+len); + try + { + interp->NewData(true, data, data + len); + } + catch ( const binpac::Exception& e ) + { + printf("Binpac exception: %s\n", e.c_msg()); + return false; + } + return true; } diff --git a/src/file_analysis/analyzer/unified2/events.bif b/src/file_analysis/analyzer/unified2/events.bif index f1c3035606..c5f3dda6a4 100644 --- a/src/file_analysis/analyzer/unified2/events.bif +++ b/src/file_analysis/analyzer/unified2/events.bif @@ -1,2 +1,7 @@ -event unified2_alert%(f: fa_file, alert: count%); +## Abstract all of the various Unified2 event formats into +## a single event. +event unified2_event%(f: fa_file, ev: Unified2::IDSEvent%); + +## The Unified2 packet format event. +event unified2_packet%(f: fa_file, pkt: Unified2::Packet%); diff --git a/src/file_analysis/analyzer/unified2/types.bif b/src/file_analysis/analyzer/unified2/types.bif index fb385bf962..20965f7038 100644 --- a/src/file_analysis/analyzer/unified2/types.bif +++ b/src/file_analysis/analyzer/unified2/types.bif @@ -1 +1,2 @@ -type Unified2Alert: record; +type Unified2::IDSEvent: record; +type Unified2::Packet: record; diff --git a/src/file_analysis/analyzer/unified2/unified2-analyzer.pac b/src/file_analysis/analyzer/unified2/unified2-analyzer.pac index 4743ebb5da..490754c47d 100644 --- a/src/file_analysis/analyzer/unified2/unified2-analyzer.pac +++ b/src/file_analysis/analyzer/unified2/unified2-analyzer.pac @@ -2,6 +2,8 @@ #include "Event.h" #include "file_analysis/File.h" #include "events.bif.h" +#include "types.bif.h" +#include "IPAddr.h" %} refine flow Flow += { @@ -18,18 +20,150 @@ refine flow Flow += { %cleanup{ %} + function ts_to_double(ts: Time): double + %{ + double t = ${ts.seconds} + (${ts.microseconds} / 1000000); + return t; + %} + + function unified2_addr_to_bro_addr(a: uint32[]): AddrVal + %{ + if ( a->size() == 1 ) + { + return new AddrVal(IPAddr(IPv4, &(a->at(0)), IPAddr::Host)); + } + else if ( a->size() == 4 ) + { + uint32 tmp[4] = { a->at(0), a->at(1), a->at(2), a->at(3) }; + return new AddrVal(IPAddr(IPv6, tmp, IPAddr::Host)); + } + else + { + // Should never reach here. + return new AddrVal(1); + } + %} + + function to_port(n: uint16, p: uint8): PortVal + %{ + TransportProto proto = TRANSPORT_UNKNOWN; + switch ( p ) { + case 1: proto = TRANSPORT_ICMP; break; + case 6: proto = TRANSPORT_TCP; break; + case 17: proto = TRANSPORT_UDP; break; + } + + return new PortVal(n, proto); + %} + + #function proc_record(rec: Record) : bool + # %{ + # return true; + # %} + + function proc_legacy_ids_event(ev: LegacyIDSEvent) : bool + %{ + if ( ::unified2_event ) + { + RecordVal* ids_event = new RecordVal(BifType::Record::Unified2::IDSEvent); + ids_event->Assign(0, new Val(${ev.sensor_id}, TYPE_COUNT)); + ids_event->Assign(1, new Val(${ev.event_id}, TYPE_COUNT)); + ids_event->Assign(2, new Val(ts_to_double(${ev.ts}), TYPE_TIME)); + ids_event->Assign(3, new Val(${ev.signature_id}, TYPE_COUNT)); + ids_event->Assign(4, new Val(${ev.generator_id}, TYPE_COUNT)); + ids_event->Assign(5, new Val(${ev.signature_revision}, TYPE_COUNT)); + ids_event->Assign(6, new Val(${ev.classification_id}, TYPE_COUNT)); + ids_event->Assign(7, new Val(${ev.priority_id}, TYPE_COUNT)); + ids_event->Assign(8, unified2_addr_to_bro_addr(${ev.src_ip})); + ids_event->Assign(9, unified2_addr_to_bro_addr(${ev.dst_ip})); + ids_event->Assign(10, to_port(${ev.src_p}, ${ev.protocol})); + ids_event->Assign(11, to_port(${ev.dst_p}, ${ev.protocol})); + ids_event->Assign(17, new Val(${ev.packet_action}, TYPE_COUNT)); + + val_list* vl = new val_list(); + vl->append(connection()->bro_analyzer()->GetFile()->GetVal()->Ref()); + vl->append(ids_event); + mgr.QueueEvent(::unified2_event, vl, SOURCE_LOCAL); + } + return true; + %} + function proc_ids_event(ev: IDSEvent) : bool %{ - val_list* vl = new val_list(); - vl->append(connection()->bro_analyzer()->GetFile()->GetVal()->Ref()); - vl->append(new Val(${ev.signature_id}, TYPE_COUNT)); - mgr.QueueEvent(::unified2_alert, vl, SOURCE_LOCAL); + if ( ::unified2_event ) + { + RecordVal* ids_event = new RecordVal(BifType::Record::Unified2::IDSEvent); + ids_event->Assign(0, new Val(${ev.sensor_id}, TYPE_COUNT)); + ids_event->Assign(1, new Val(${ev.event_id}, TYPE_COUNT)); + ids_event->Assign(2, new Val(ts_to_double(${ev.ts}), TYPE_TIME)); + ids_event->Assign(3, new Val(${ev.signature_id}, TYPE_COUNT)); + ids_event->Assign(4, new Val(${ev.generator_id}, TYPE_COUNT)); + ids_event->Assign(5, new Val(${ev.signature_revision}, TYPE_COUNT)); + ids_event->Assign(6, new Val(${ev.classification_id}, TYPE_COUNT)); + ids_event->Assign(7, new Val(${ev.priority_id}, TYPE_COUNT)); + ids_event->Assign(8, unified2_addr_to_bro_addr(${ev.src_ip})); + ids_event->Assign(9, unified2_addr_to_bro_addr(${ev.dst_ip})); + ids_event->Assign(10, to_port(${ev.src_p}, ${ev.protocol})); + ids_event->Assign(11, to_port(${ev.dst_p}, ${ev.protocol})); + ids_event->Assign(12, new Val(${ev.impact_flag}, TYPE_COUNT)); + ids_event->Assign(13, new Val(${ev.impact}, TYPE_COUNT)); + ids_event->Assign(14, new Val(${ev.blocked}, TYPE_COUNT)); + ids_event->Assign(15, new Val(${ev.mpls_label}, TYPE_COUNT)); + ids_event->Assign(16, new Val(${ev.vlan_id}, TYPE_COUNT)); + + val_list* vl = new val_list(); + vl->append(connection()->bro_analyzer()->GetFile()->GetVal()->Ref()); + vl->append(ids_event); + mgr.QueueEvent(::unified2_event, vl, SOURCE_LOCAL); + } return true; %} + + function proc_packet(pkt: Packet) : bool + %{ + if ( ::unified2_packet ) + { + RecordVal* packet = new RecordVal(BifType::Record::Unified2::Packet); + packet->Assign(0, new Val(${pkt.sensor_id}, TYPE_COUNT)); + packet->Assign(1, new Val(${pkt.event_id}, TYPE_COUNT)); + packet->Assign(2, new Val(${pkt.event_second}, TYPE_COUNT)); + packet->Assign(3, new Val(ts_to_double(${pkt.packet_ts}), TYPE_TIME)); + packet->Assign(4, new Val(${pkt.link_type}, TYPE_COUNT)); + packet->Assign(5, bytestring_to_val(${pkt.packet_data})); + + val_list* vl = new val_list(); + vl->append(connection()->bro_analyzer()->GetFile()->GetVal()->Ref()); + vl->append(packet); + mgr.QueueEvent(::unified2_packet, vl, SOURCE_LOCAL); + } + + return true; + %} + + #function proc_unknown_record_type(rec: UnknownRecordType) : bool + # %{ + # printf("unknown packet type\n"); + # return true; + # %} }; +#refine typeattr Record += &let { +# proc : bool = $context.flow.proc_record(this); +#}; + +refine typeattr LegacyIDSEvent += &let { + proc : bool = $context.flow.proc_legacy_ids_event(this); +}; refine typeattr IDSEvent += &let { proc : bool = $context.flow.proc_ids_event(this); }; + +refine typeattr Packet += &let { + proc : bool = $context.flow.proc_packet(this); +}; + +#refine typeattr UnknownRecordType += &let { +# proc : bool = $context.flow.proc_unknown_record_type(this); +#}; \ No newline at end of file diff --git a/src/file_analysis/analyzer/unified2/unified2-file.pac b/src/file_analysis/analyzer/unified2/unified2-file.pac index 01497aaec3..72f5c959bf 100644 --- a/src/file_analysis/analyzer/unified2/unified2-file.pac +++ b/src/file_analysis/analyzer/unified2/unified2-file.pac @@ -1,14 +1,11 @@ enum Types { - EVENT = 0, - PACKET = 1, - IDS_EVENT = 2, - IDS_EVENT_IPV6 = 72, - IDS_EVENT_MPLS = 99, - IDS_EVENT_IPV6_MPLS = 100, - IDS_EVENT_VLAN = 104, - IDS_EVENT_IPV6_VLAN = 105, - EXTRA_DATA = 110, + PACKET = 2, + IDS_EVENT = 7, + IDS_EVENT_IPV6 = 72, + IDS_EVENT_2 = 104, + IDS_EVENT_IPV6_2 = 105, + EXTRA_DATA = 110, }; type Time = record { @@ -16,40 +13,21 @@ type Time = record { microseconds: uint32; } &byteorder=bigendian; -type v4Addr = record { - u1: uint32; -}; - -type v6Addr = record { - u1: uint32; - u2: uint32; - u3: uint32; - u4: uint32; -}; - -type Addr(ip_ver: int) = case ip_ver of { - 4 -> v4: v4Addr; - 6 -> v6: v6Addr; -} &byteorder=bigendian; - type Record = record { rtype: uint32; length: uint32; data: case rtype of { - # EVENT -> event: Event(this); - PACKET -> packet: Packet(this); - IDS_EVENT -> ids_event: LegacyIDSEvent(this, 4); - IDS_EVENT_IPV6 -> ids_event_ipv6: LegacyIDSEvent(this, 6); - # IDS_EVENT_MPLS -> ids_event_mpls: IDSEvent(this, 4); - # IDS_EVENT_IPV6_MPLS -> ids_event_ipv6_mpls: IDSEvent(this, 6); - IDS_EVENT_VLAN -> ids_event_vlan: IDSEvent(this, 4); - IDS_EVENT_IPV6_VLAN -> ids_event_ipv6_vlan: IDSEvent(this, 6); - EXTRA_DATA -> extra_data: ExtraData(this); - default -> unknown_record_type: UnknownRecordType(this); + PACKET -> packet: Packet(this); + IDS_EVENT -> ids_event: LegacyIDSEvent(this, 1); + IDS_EVENT_IPV6 -> ids_event_ipv6: LegacyIDSEvent(this, 4); + IDS_EVENT_2 -> ids_event_vlan: IDSEvent(this, 1); + IDS_EVENT_IPV6_2 -> ids_event_ipv6_vlan: IDSEvent(this, 4); + #EXTRA_DATA -> extra_data: ExtraData(this); + default -> unknown_record_type: UnknownRecordType(this); }; } &byteorder=bigendian &length=length+8; -type LegacyIDSEvent(rec: Record, ip_ver: int) = record { +type LegacyIDSEvent(rec: Record, ip_len: int) = record { sensor_id: uint32; event_id: uint32; ts: Time; @@ -58,15 +36,15 @@ type LegacyIDSEvent(rec: Record, ip_ver: int) = record { signature_revision: uint32; classification_id: uint32; priority_id: uint32; - src_ip: Addr(ip_ver); - dst_ip: Addr(ip_ver); + src_ip: uint32[ip_len]; + dst_ip: uint32[ip_len]; src_p: uint16; dst_p: uint16; protocol: uint8; packet_action: uint8; -}; +} &byteorder=bigendian; -type IDSEvent(rec: Record, ip_ver: int) = record { +type IDSEvent(rec: Record, ip_len: int) = record { sensor_id: uint32; event_id: uint32; ts: Time; @@ -75,8 +53,8 @@ type IDSEvent(rec: Record, ip_ver: int) = record { signature_revision: uint32; classification_id: uint32; priority_id: uint32; - src_ip: Addr(ip_ver); - dst_ip: Addr(ip_ver); + src_ip: uint32[ip_len]; + dst_ip: uint32[ip_len]; src_p: uint16; dst_p: uint16; protocol: uint8; @@ -85,7 +63,7 @@ type IDSEvent(rec: Record, ip_ver: int) = record { blocked: uint8; mpls_label: uint32; vlan_id: uint16; - : uint16; + pad: uint16; } &byteorder=bigendian; type Packet(rec: Record) = record { @@ -96,7 +74,7 @@ type Packet(rec: Record) = record { link_type: uint32; packet_len: uint32; packet_data: bytestring &length=packet_len; -} &byteorder=bigendian &length=rec.length; +} &byteorder=bigendian; type ExtraData(rec: Record) = record { sensor_id: uint32; @@ -111,7 +89,3 @@ type ExtraData(rec: Record) = record { type UnknownRecordType(rec: Record) = record { data: bytestring &transient &length=rec.length; } &byteorder=bigendian &length=rec.length; - -type File = record { - alerts: Record[] &transient &until($element <= 0); -} &byteorder=bigendian; diff --git a/src/file_analysis/analyzer/unified2/unified2.pac b/src/file_analysis/analyzer/unified2/unified2.pac index c0947d3fb5..ddc7dc5315 100644 --- a/src/file_analysis/analyzer/unified2/unified2.pac +++ b/src/file_analysis/analyzer/unified2/unified2.pac @@ -15,7 +15,7 @@ analyzer Unified2_Analyzer(bro_analyzer: BroFileAnalyzer) { %include unified2-file.pac flow Flow { - flowunit = File withcontext(connection, this); + flowunit = Record withcontext(connection, this); }; %include unified2-analyzer.pac From 95161a920cfbcd94c9f4877b6afbeae4f5610ab2 Mon Sep 17 00:00:00 2001 From: Seth Hall Date: Mon, 12 Aug 2013 15:31:31 -0400 Subject: [PATCH 4/6] Fixed some tests. --- doc/scripts/DocSourcesList.cmake | 3 +++ scripts/base/files/unified2/main.bro | 11 ++++++++++- scripts/base/init-bare.bro | 9 ++++++--- scripts/base/utils/paths.bro | 5 +++-- .../canonified_loaded_scripts.log | 6 ++++-- .../canonified_loaded_scripts.log | 8 ++++++-- .../files.log | 6 +++--- 7 files changed, 35 insertions(+), 13 deletions(-) diff --git a/doc/scripts/DocSourcesList.cmake b/doc/scripts/DocSourcesList.cmake index bd88f5cd54..56a27b18ac 100644 --- a/doc/scripts/DocSourcesList.cmake +++ b/doc/scripts/DocSourcesList.cmake @@ -70,6 +70,8 @@ rest_target(${CMAKE_BINARY_DIR}/scripts base/bif/plugins/Bro_TCP.events.bif.bro) rest_target(${CMAKE_BINARY_DIR}/scripts base/bif/plugins/Bro_TCP.functions.bif.bro) rest_target(${CMAKE_BINARY_DIR}/scripts base/bif/plugins/Bro_Teredo.events.bif.bro) rest_target(${CMAKE_BINARY_DIR}/scripts base/bif/plugins/Bro_UDP.events.bif.bro) +rest_target(${CMAKE_BINARY_DIR}/scripts base/bif/plugins/Bro_Unified2.events.bif.bro) +rest_target(${CMAKE_BINARY_DIR}/scripts base/bif/plugins/Bro_Unified2.types.bif.bro) rest_target(${CMAKE_BINARY_DIR}/scripts base/bif/plugins/Bro_ZIP.events.bif.bro) rest_target(${CMAKE_BINARY_DIR}/scripts base/bif/reporter.bif.bro) rest_target(${CMAKE_BINARY_DIR}/scripts base/bif/strings.bif.bro) @@ -77,6 +79,7 @@ rest_target(${CMAKE_BINARY_DIR}/scripts base/bif/top-k.bif.bro) rest_target(${CMAKE_BINARY_DIR}/scripts base/bif/types.bif.bro) rest_target(${psd} base/files/extract/main.bro) rest_target(${psd} base/files/hash/main.bro) +rest_target(${psd} base/files/unified2/main.bro) rest_target(${psd} base/frameworks/analyzer/main.bro) rest_target(${psd} base/frameworks/cluster/main.bro) rest_target(${psd} base/frameworks/cluster/nodes/manager.bro) diff --git a/scripts/base/files/unified2/main.bro b/scripts/base/files/unified2/main.bro index 25eef30d72..4d540cca3b 100644 --- a/scripts/base/files/unified2/main.bro +++ b/scripts/base/files/unified2/main.bro @@ -16,10 +16,14 @@ export { global alert: event(f: fa_file, ev: Unified2::IDSEvent, pkt: Unified2::Packet); type Info: record { + ## The last received IDS event. This is primarily used + ## for tying together Unified2 events and packets. current_event: Unified2::IDSEvent &optional; }; redef record fa_file += { + ## Add a field to store per-file state about Unified2 + ## files. unified2: Info &optional; }; } @@ -48,8 +52,13 @@ event bro_init() event file_new(f: fa_file) { + local file_dir = ""; + local parts = split_all(f$source, /\/[^\/]*$/); + if ( |parts| == 3 ) + file_dir = parts[1]; + if ( f$source in watch_file || - compress_path(extract_path(f$source)) == compress_path(watch_dir) ) + compress_path(watch_dir) == file_dir ) { Files::add_analyzer(f, Files::ANALYZER_UNIFIED2); f$unified2 = Info(); diff --git a/scripts/base/init-bare.bro b/scripts/base/init-bare.bro index 6d354c90bf..ac8f13f1f6 100644 --- a/scripts/base/init-bare.bro +++ b/scripts/base/init-bare.bro @@ -2704,9 +2704,12 @@ export { impact_flag: count; impact: count; blocked: count; - mpls_label: count &optional; ## Not available in "legacy" IDS events. - vlan_id: count &optional; ## Not available in "legacy" IDS events. - packet_action: count &optional; ## Only available in "legacy" IDS events. + ## Not available in "legacy" IDS events. + mpls_label: count &optional; + ## Not available in "legacy" IDS events. + vlan_id: count &optional; + ## Only available in "legacy" IDS events. + packet_action: count &optional; }; type Unified2::Packet: record { diff --git a/scripts/base/utils/paths.bro b/scripts/base/utils/paths.bro index 04976c2b2b..f8ad384ea7 100644 --- a/scripts/base/utils/paths.bro +++ b/scripts/base/utils/paths.bro @@ -9,12 +9,13 @@ const absolute_path_pat = /(\/|[A-Za-z]:[\\\/]).*/; ## Returns: the first absolute path found in input string, else an empty string function extract_path(input: string): string { - local dir_pattern = /([^\\\/]+)$/; + const dir_pattern = /(\/|[A-Za-z]:[\\\/])([^\"\ ]|(\\\ ))*/; local parts = split_all(input, dir_pattern); + if ( |parts| < 3 ) return ""; - return parts[1]; + return parts[2]; } ## Compresses a given path by removing '..'s and the parent directory it diff --git a/testing/btest/Baseline/coverage.bare-load-baseline/canonified_loaded_scripts.log b/testing/btest/Baseline/coverage.bare-load-baseline/canonified_loaded_scripts.log index ca22ba568c..32bd473b63 100644 --- a/testing/btest/Baseline/coverage.bare-load-baseline/canonified_loaded_scripts.log +++ b/testing/btest/Baseline/coverage.bare-load-baseline/canonified_loaded_scripts.log @@ -3,7 +3,7 @@ #empty_field (empty) #unset_field - #path loaded_scripts -#open 2013-08-09-16-13-58 +#open 2013-08-12-19-04-42 #fields name #types string scripts/base/init-bare.bro @@ -61,6 +61,8 @@ scripts/base/init-bare.bro build/scripts/base/bif/plugins/Bro_TCP.functions.bif.bro build/scripts/base/bif/plugins/Bro_Teredo.events.bif.bro build/scripts/base/bif/plugins/Bro_UDP.events.bif.bro + build/scripts/base/bif/plugins/Bro_Unified2.events.bif.bro + build/scripts/base/bif/plugins/Bro_Unified2.types.bif.bro build/scripts/base/bif/plugins/Bro_ZIP.events.bif.bro scripts/base/frameworks/logging/__load__.bro scripts/base/frameworks/logging/main.bro @@ -94,4 +96,4 @@ scripts/base/init-bare.bro build/scripts/base/bif/top-k.bif.bro scripts/policy/misc/loaded-scripts.bro scripts/base/utils/paths.bro -#close 2013-08-09-16-13-58 +#close 2013-08-12-19-04-42 diff --git a/testing/btest/Baseline/coverage.default-load-baseline/canonified_loaded_scripts.log b/testing/btest/Baseline/coverage.default-load-baseline/canonified_loaded_scripts.log index ab30771ef3..f237ebf027 100644 --- a/testing/btest/Baseline/coverage.default-load-baseline/canonified_loaded_scripts.log +++ b/testing/btest/Baseline/coverage.default-load-baseline/canonified_loaded_scripts.log @@ -3,7 +3,7 @@ #empty_field (empty) #unset_field - #path loaded_scripts -#open 2013-08-09-16-13-37 +#open 2013-08-12-19-05-30 #fields name #types string scripts/base/init-bare.bro @@ -61,6 +61,8 @@ scripts/base/init-bare.bro build/scripts/base/bif/plugins/Bro_TCP.functions.bif.bro build/scripts/base/bif/plugins/Bro_Teredo.events.bif.bro build/scripts/base/bif/plugins/Bro_UDP.events.bif.bro + build/scripts/base/bif/plugins/Bro_Unified2.events.bif.bro + build/scripts/base/bif/plugins/Bro_Unified2.types.bif.bro build/scripts/base/bif/plugins/Bro_ZIP.events.bif.bro scripts/base/frameworks/logging/__load__.bro scripts/base/frameworks/logging/main.bro @@ -206,6 +208,8 @@ scripts/base/init-default.bro scripts/base/files/hash/main.bro scripts/base/files/extract/__load__.bro scripts/base/files/extract/main.bro + scripts/base/files/unified2/__load__.bro + scripts/base/files/unified2/main.bro scripts/base/misc/find-checksum-offloading.bro scripts/policy/misc/loaded-scripts.bro -#close 2013-08-09-16-13-37 +#close 2013-08-12-19-05-30 diff --git a/testing/btest/Baseline/scripts.base.frameworks.file-analysis.logging/files.log b/testing/btest/Baseline/scripts.base.frameworks.file-analysis.logging/files.log index 2663184b88..1e46209d46 100644 --- a/testing/btest/Baseline/scripts.base.frameworks.file-analysis.logging/files.log +++ b/testing/btest/Baseline/scripts.base.frameworks.file-analysis.logging/files.log @@ -3,8 +3,8 @@ #empty_field (empty) #unset_field - #path files -#open 2013-07-25-16-57-31 +#open 2013-08-12-19-07-37 #fields ts fuid tx_hosts rx_hosts conn_uids source depth analyzers mime_type filename duration local_orig is_orig seen_bytes total_bytes missing_bytes overflow_bytes timedout parent_fuid md5 sha1 sha256 extracted #types time string table[addr] table[addr] table[string] string count table[string] string string interval bool bool count count count count bool string string string string string -1362692527.009721 G75mcAsU764 192.150.187.43 141.142.228.5 UWkUyAuUGXf HTTP 0 SHA256,DATA_EVENT,MD5,EXTRACT,SHA1 text/plain - 0.000054 - F 4705 4705 0 0 F - 397168fd09991a0e712254df7bc639ac 1dd7ac0398df6cbc0696445a91ec681facf4dc47 4e7c7ef0984119447e743e3ec77e1de52713e345cde03fe7df753a35849bed18 G75mcAsU764-file -#close 2013-07-25-16-57-31 +1362692527.009721 G75mcAsU764 192.150.187.43 141.142.228.5 UWkUyAuUGXf HTTP 0 UNIFIED2,SHA256,DATA_EVENT,MD5,EXTRACT,SHA1 text/plain - 0.000054 - F 4705 4705 0 0 F - 397168fd09991a0e712254df7bc639ac 1dd7ac0398df6cbc0696445a91ec681facf4dc47 4e7c7ef0984119447e743e3ec77e1de52713e345cde03fe7df753a35849bed18 G75mcAsU764-file +#close 2013-08-12-19-07-37 From f7c6dd7f7e7a680aebc31558bbf05756f87ad45a Mon Sep 17 00:00:00 2001 From: Seth Hall Date: Tue, 13 Aug 2013 03:21:43 -0400 Subject: [PATCH 5/6] Finished work on unified2 analyzer. --- scripts/base/files/unified2/__load__.bro | 2 +- scripts/base/files/unified2/main.bro | 164 ++++++++++++++++-- scripts/base/init-bare.bro | 12 +- .../analyzer/unified2/unified2-analyzer.pac | 12 +- .../analyzer/unified2/unified2-file.pac | 12 +- .../unified2.log | 11 ++ testing/btest/Files/unified2.u2 | Bin 0 -> 576 bytes testing/btest/btest.cfg | 1 + .../scripts/base/files/unified2/alert.bro | 76 ++++++++ 9 files changed, 260 insertions(+), 30 deletions(-) create mode 100644 testing/btest/Baseline/scripts.base.files.unified2.alert/unified2.log create mode 100755 testing/btest/Files/unified2.u2 create mode 100644 testing/btest/scripts/base/files/unified2/alert.bro diff --git a/scripts/base/files/unified2/__load__.bro b/scripts/base/files/unified2/__load__.bro index d551be57d3..a10fe855df 100644 --- a/scripts/base/files/unified2/__load__.bro +++ b/scripts/base/files/unified2/__load__.bro @@ -1 +1 @@ -@load ./main \ No newline at end of file +@load ./main diff --git a/scripts/base/files/unified2/main.bro b/scripts/base/files/unified2/main.bro index 4d540cca3b..6d873a54cb 100644 --- a/scripts/base/files/unified2/main.bro +++ b/scripts/base/files/unified2/main.bro @@ -5,31 +5,148 @@ module Unified2; export { + redef enum Log::ID += { LOG }; + ## Directory to watch for Unified2 files. const watch_file = "" &redef; ## File to watch for Unified2 records. const watch_dir = "" &redef; + ## The sid-msg.map file you would like to use for your alerts. + const sid_msg = "" &redef; + + ## The gen-msg.map file you would like to use for your alerts. + const gen_msg = "" &redef; + + ## The classification.config file you would like to use for your alerts. + const classification_config = "" &redef; + ## Reconstructed "alert" which combines related events ## and packets. global alert: event(f: fa_file, ev: Unified2::IDSEvent, pkt: Unified2::Packet); + type PacketID: record { + src_ip: addr; + src_p: port; + dst_ip: addr; + dst_p: port; + } &log; + type Info: record { - ## The last received IDS event. This is primarily used - ## for tying together Unified2 events and packets. - current_event: Unified2::IDSEvent &optional; - }; + ## Timestamp attached to the alert. + ts: time &log; + ## Addresses and ports for the connection. + id: PacketID &log; + ## Sensor that originated this event. + sensor_id: count &log; + ## Sig id for this generator. + signature_id: count &log; + ## A string representation of the "signature_id" field if a sid_msg.map file was loaded. + signature: string &log &optional; + ## Which generator generated the alert? + generator_id: count &log; + ## A string representation of the "generator_id" field if a gen_msg.map file was loaded. + generator: string &log &optional; + ## Sig revision for this id. + signature_revision: count &log; + ## Event classification. + classification_id: count &log; + ## Descriptive classification string, + classification: string &log &optional; + ## Event priority. + priority_id: count &log; + ## Event ID. + event_id: count &log; + ## Some of the packet data. + packet: string &log; + } &log; redef record fa_file += { - ## Add a field to store per-file state about Unified2 - ## files. + ## Add a field to store per-file state about Unified2 files. unified2: Info &optional; + + ## Recently received IDS events. This is primarily used + ## for tying together Unified2 events and packets. + u2_events: table[count] of Unified2::IDSEvent &optional &create_expire=5sec; }; + + ## The event for accessing logged records. + global log_unified2: event(rec: Info); } +# Mappings for extended information from alerts. +global classification_map: table[count] of string; +global sid_map: table[count] of string; +global gen_map: table[count] of string; + +# For reading in config files. +type OneLine: record { + line: string; +}; + +event Unified2::read_sid_msg_line(desc: Input::EventDescription, tpe: Input::Event, line: string) + { + local parts = split_n(line, / \|\| /, F, 100); + if ( |parts| >= 2 && /^[0-9]+$/ in parts[1] ) + sid_map[to_count(parts[1])] = parts[2]; + } + +event Unified2::read_gen_msg_line(desc: Input::EventDescription, tpe: Input::Event, line: string) + { + local parts = split_n(line, / \|\| /, F, 3); + if ( |parts| >= 2 && /^[0-9]+$/ in parts[1] ) + gen_map[to_count(parts[1])] = parts[3]; + } + +event Unified2::read_classification_line(desc: Input::EventDescription, tpe: Input::Event, line: string) + { + local parts = split_n(line, /: /, F, 2); + if ( |parts| == 2 ) + { + local parts2 = split_n(parts[2], /,/, F, 4); + if ( |parts2| > 1 ) + classification_map[|classification_map|+1] = parts2[1]; + } + } + event bro_init() { + Log::create_stream(Unified2::LOG, [$columns=Info, $ev=log_unified2]); + + if ( sid_msg != "" ) + { + Input::add_event([$source=sid_msg, + $reader=Input::READER_RAW, + $mode=Input::REREAD, + $name=sid_msg, + $fields=Unified2::OneLine, + $want_record=F, + $ev=Unified2::read_sid_msg_line]); + } + + if ( gen_msg != "" ) + { + Input::add_event([$source=gen_msg, + $name=gen_msg, + $reader=Input::READER_RAW, + $mode=Input::REREAD, + $fields=Unified2::OneLine, + $want_record=F, + $ev=Unified2::read_gen_msg_line]); + } + + if ( classification_config != "" ) + { + Input::add_event([$source=classification_config, + $name=classification_config, + $reader=Input::READER_RAW, + $mode=Input::REREAD, + $fields=Unified2::OneLine, + $want_record=F, + $ev=Unified2::read_classification_line]); + } + if ( watch_dir != "" ) { Dir::monitor(watch_dir, function(fname: string) @@ -61,20 +178,45 @@ event file_new(f: fa_file) compress_path(watch_dir) == file_dir ) { Files::add_analyzer(f, Files::ANALYZER_UNIFIED2); - f$unified2 = Info(); + f$u2_events = table(); } } event unified2_event(f: fa_file, ev: Unified2::IDSEvent) { - f$unified2$current_event = ev; + f$u2_events[ev$event_id] = ev; } event unified2_packet(f: fa_file, pkt: Unified2::Packet) { - if ( f$unified2?$current_event ) + if ( f?$u2_events && pkt$event_id in f$u2_events) { - event Unified2::alert(f, f$unified2$current_event, pkt); - delete f$unified2$current_event; + local ev = f$u2_events[pkt$event_id]; + event Unified2::alert(f, ev, pkt); + delete f$u2_events[pkt$event_id]; } } + +event Unified2::alert(f: fa_file, ev: IDSEvent, pkt: Packet) + { + local info = Info($ts=ev$ts, + $id=PacketID($src_ip=ev$src_ip, $src_p=ev$src_p, + $dst_ip=ev$dst_ip, $dst_p=ev$dst_p), + $sensor_id=ev$sensor_id, + $signature_id=ev$signature_id, + $generator_id=ev$generator_id, + $signature_revision=ev$signature_revision, + $classification_id=ev$classification_id, + $priority_id=ev$priority_id, + $event_id=ev$event_id, + $packet=pkt$data); + + if ( ev$signature_id in sid_map ) + info$signature=sid_map[ev$signature_id]; + if ( ev$generator_id in gen_map ) + info$generator=gen_map[ev$generator_id]; + if ( ev$classification_id in classification_map ) + info$classification=classification_map[ev$classification_id]; + + Log::write(LOG, info); + } diff --git a/scripts/base/init-bare.bro b/scripts/base/init-bare.bro index ac8f13f1f6..a911b0946e 100644 --- a/scripts/base/init-bare.bro +++ b/scripts/base/init-bare.bro @@ -2713,12 +2713,12 @@ export { }; type Unified2::Packet: record { - sensor_id: count; - event_id: count; - event_second: count; - packet_ts: time; - link_type: count; - packet_data: string; + sensor_id: count; + event_id: count; + event_second: count; + packet_ts: time; + link_type: count; + data: string; }; } diff --git a/src/file_analysis/analyzer/unified2/unified2-analyzer.pac b/src/file_analysis/analyzer/unified2/unified2-analyzer.pac index 490754c47d..cef788096a 100644 --- a/src/file_analysis/analyzer/unified2/unified2-analyzer.pac +++ b/src/file_analysis/analyzer/unified2/unified2-analyzer.pac @@ -61,7 +61,7 @@ refine flow Flow += { # return true; # %} - function proc_legacy_ids_event(ev: LegacyIDSEvent) : bool + function proc_ids_event(ev: IDS_Event) : bool %{ if ( ::unified2_event ) { @@ -88,7 +88,7 @@ refine flow Flow += { return true; %} - function proc_ids_event(ev: IDSEvent) : bool + function proc_ids_event_2(ev: IDS_Event_2) : bool %{ if ( ::unified2_event ) { @@ -152,12 +152,12 @@ refine flow Flow += { # proc : bool = $context.flow.proc_record(this); #}; -refine typeattr LegacyIDSEvent += &let { - proc : bool = $context.flow.proc_legacy_ids_event(this); +refine typeattr IDS_Event += &let { + proc : bool = $context.flow.proc_ids_event(this); }; -refine typeattr IDSEvent += &let { - proc : bool = $context.flow.proc_ids_event(this); +refine typeattr IDS_Event_2 += &let { + proc : bool = $context.flow.proc_ids_event_2(this); }; refine typeattr Packet += &let { diff --git a/src/file_analysis/analyzer/unified2/unified2-file.pac b/src/file_analysis/analyzer/unified2/unified2-file.pac index 72f5c959bf..f0d8ca9187 100644 --- a/src/file_analysis/analyzer/unified2/unified2-file.pac +++ b/src/file_analysis/analyzer/unified2/unified2-file.pac @@ -18,16 +18,16 @@ type Record = record { length: uint32; data: case rtype of { PACKET -> packet: Packet(this); - IDS_EVENT -> ids_event: LegacyIDSEvent(this, 1); - IDS_EVENT_IPV6 -> ids_event_ipv6: LegacyIDSEvent(this, 4); - IDS_EVENT_2 -> ids_event_vlan: IDSEvent(this, 1); - IDS_EVENT_IPV6_2 -> ids_event_ipv6_vlan: IDSEvent(this, 4); + IDS_EVENT -> ids_event: IDS_Event(this, 1); + IDS_EVENT_IPV6 -> ids_event_ipv6: IDS_Event(this, 4); + IDS_EVENT_2 -> ids_event_vlan: IDS_Event_2(this, 1); + IDS_EVENT_IPV6_2 -> ids_event_ipv6_vlan: IDS_Event_2(this, 4); #EXTRA_DATA -> extra_data: ExtraData(this); default -> unknown_record_type: UnknownRecordType(this); }; } &byteorder=bigendian &length=length+8; -type LegacyIDSEvent(rec: Record, ip_len: int) = record { +type IDS_Event(rec: Record, ip_len: int) = record { sensor_id: uint32; event_id: uint32; ts: Time; @@ -44,7 +44,7 @@ type LegacyIDSEvent(rec: Record, ip_len: int) = record { packet_action: uint8; } &byteorder=bigendian; -type IDSEvent(rec: Record, ip_len: int) = record { +type IDS_Event_2(rec: Record, ip_len: int) = record { sensor_id: uint32; event_id: uint32; ts: Time; diff --git a/testing/btest/Baseline/scripts.base.files.unified2.alert/unified2.log b/testing/btest/Baseline/scripts.base.files.unified2.alert/unified2.log new file mode 100644 index 0000000000..b86dedf23b --- /dev/null +++ b/testing/btest/Baseline/scripts.base.files.unified2.alert/unified2.log @@ -0,0 +1,11 @@ +#separator \x09 +#set_separator , +#empty_field (empty) +#unset_field - +#path unified2 +#open 2013-08-13-07-16-01 +#fields ts id.src_ip id.src_p id.dst_ip id.dst_p sensor_id signature_id signature generator_id generator signature_revision classification_id classification priority_id event_id packet +#types time addr port addr port count count string count string count count string count count string +1323827323.000000 192.168.1.72 50185 74.125.225.49 80 0 2003058 ET MALWARE 180solutions (Zango) Spyware Installer Download 1 snort general alert 5 21 trojan-activity 1 2 \xd80bH\xc5\xb5x\xca9\xb7\xe4r\x08\x00E\x10\x00\\x1a\xce@\x00@\x062\x1f\xc0\xa8\x01HJ}\xe11\xc4\x09\x00P*\xa8bv]z/\xde\x80\x18\x82+\x88,\x00\x00\x01\x01\x08\x0a\x17J\x83Q\xfe\xad\xac\x1aGET /Zango/ZangoInstaller.exe HTTP/1.0\x0d\x0a +1323827344.000000 192.168.1.72 49862 199.47.216.144 80 0 2012647 ET POLICY Dropbox.com Offsite File Backup in Use 1 snort general alert 3 33 policy-violation 1 3 \xd80bH\xc5\xb5x\xca9\xb7\xe4r\x08\x00E\x00\x00\xf8Q\xdf@\x00@\x06\x86p\xc0\xa8\x01H\xc7/\xd8\x90\xc2\xc6\x00P\x9cm\x97U\xf07\x084\x80\x18\x82\x18%<\x00\x00\x01\x01\x08\x0a\x17J\xd7\xde\x00\x92\x81\xc5GET /subscribe?host_int=43112345&ns_map=123456_1234524412104916591&ts=1323827344 HTTP/1.1\x0d\x0aHost: notify1.dropbox.com\x0d\x0aAccept-Encoding: identity\x0d\x0aConnection: keep-alive\x0d\x0aX-Dropbox-Locale: en_US\x0d\x0a\x0d\x0a +#close 2013-08-13-07-16-01 diff --git a/testing/btest/Files/unified2.u2 b/testing/btest/Files/unified2.u2 new file mode 100755 index 0000000000000000000000000000000000000000..2619152e1449dcba038027753738d44954a1eed5 GIT binary patch literal 576 zcmZQzV8~!#V6XvV5Mc6q!BEY>6lTjHH=zi~X9QwaAQlC&53FGH@Tz@ic!ZN7fDI^s z1ek#AHYgu#2HYHo8Cf?Bl01%XtvF@5{YeoAgR1~TjMO;?1_w4Hd1O?iO;=z=WiHV_+p@E5|p_!?rp;}3?t)a1z zv4xSjv55&R5Da;_Jb-3dDdgpsWTsUb>ZKIr7bN9Z=q2ap@^U#QC#M#a=(^@5=ci=m zrCTXvrljVTWR_I&ayjSc<)tQ Date: Tue, 13 Aug 2013 08:55:11 -0400 Subject: [PATCH 6/6] Fixes in case a packet isn't seen that matches an event. --- scripts/base/files/unified2/main.bro | 84 ++++++++++++++++++---------- 1 file changed, 53 insertions(+), 31 deletions(-) diff --git a/scripts/base/files/unified2/main.bro b/scripts/base/files/unified2/main.bro index 6d873a54cb..83deee27d4 100644 --- a/scripts/base/files/unified2/main.bro +++ b/scripts/base/files/unified2/main.bro @@ -59,18 +59,9 @@ export { ## Event ID. event_id: count &log; ## Some of the packet data. - packet: string &log; + packet: string &log &optional; } &log; - redef record fa_file += { - ## Add a field to store per-file state about Unified2 files. - unified2: Info &optional; - - ## Recently received IDS events. This is primarily used - ## for tying together Unified2 events and packets. - u2_events: table[count] of Unified2::IDSEvent &optional &create_expire=5sec; - }; - ## The event for accessing logged records. global log_unified2: event(rec: Info); } @@ -85,6 +76,41 @@ type OneLine: record { line: string; }; +function create_info(ev: IDSEvent): Info + { + local info = Info($ts=ev$ts, + $id=PacketID($src_ip=ev$src_ip, $src_p=ev$src_p, + $dst_ip=ev$dst_ip, $dst_p=ev$dst_p), + $sensor_id=ev$sensor_id, + $signature_id=ev$signature_id, + $generator_id=ev$generator_id, + $signature_revision=ev$signature_revision, + $classification_id=ev$classification_id, + $priority_id=ev$priority_id, + $event_id=ev$event_id); + + if ( ev$signature_id in sid_map ) + info$signature=sid_map[ev$signature_id]; + if ( ev$generator_id in gen_map ) + info$generator=gen_map[ev$generator_id]; + if ( ev$classification_id in classification_map ) + info$classification=classification_map[ev$classification_id]; + + return info; + } + +redef record fa_file += { + ## Recently received IDS events. This is primarily used + ## for tying together Unified2 events and packets. + u2_events: table[count] of Unified2::IDSEvent + &optional &create_expire=5sec + &expire_func=function(t: table[count] of Unified2::IDSEvent, event_id: count): interval + { + Log::write(LOG, create_info(t[event_id])); + return 0secs; + }; +}; + event Unified2::read_sid_msg_line(desc: Input::EventDescription, tpe: Input::Event, line: string) { local parts = split_n(line, / \|\| /, F, 100); @@ -153,7 +179,7 @@ event bro_init() { Input::add_analysis([$source=fname, $reader=Input::READER_BINARY, - $mode=Input::MANUAL, + $mode=Input::STREAM, $name=fname]); }, 10secs); } @@ -162,7 +188,7 @@ event bro_init() { Input::add_analysis([$source=watch_file, $reader=Input::READER_BINARY, - $mode=Input::MANUAL, + $mode=Input::STREAM, $name=watch_file]); } } @@ -199,24 +225,20 @@ event unified2_packet(f: fa_file, pkt: Unified2::Packet) event Unified2::alert(f: fa_file, ev: IDSEvent, pkt: Packet) { - local info = Info($ts=ev$ts, - $id=PacketID($src_ip=ev$src_ip, $src_p=ev$src_p, - $dst_ip=ev$dst_ip, $dst_p=ev$dst_p), - $sensor_id=ev$sensor_id, - $signature_id=ev$signature_id, - $generator_id=ev$generator_id, - $signature_revision=ev$signature_revision, - $classification_id=ev$classification_id, - $priority_id=ev$priority_id, - $event_id=ev$event_id, - $packet=pkt$data); - - if ( ev$signature_id in sid_map ) - info$signature=sid_map[ev$signature_id]; - if ( ev$generator_id in gen_map ) - info$generator=gen_map[ev$generator_id]; - if ( ev$classification_id in classification_map ) - info$classification=classification_map[ev$classification_id]; - + local info = create_info(ev); + info$packet=pkt$data; Log::write(LOG, info); } + +event file_state_remove(f: fa_file) + { + if ( f?$u2_events ) + { + # In case any events never had matching packets, flush + # the extras to the log. + for ( i in f$u2_events ) + { + Log::write(LOG, create_info(f$u2_events[i])); + } + } + } \ No newline at end of file