diff --git a/CHANGES b/CHANGES index 84517ab170..e5b7477b97 100644 --- a/CHANGES +++ b/CHANGES @@ -1,4 +1,11 @@ +2.1-1072 | 2013-08-14 10:28:51 -0700 + + * Adding the unified2 analyzer that reads unified2 files from disk, + turning them into events. (Seth Hall) + + * Fixing intel framework tests. (Seth Hall) + 2.1-1059 | 2013-08-13 23:52:41 -0400 * Add file name support to intel framework. (Seth Hall) diff --git a/VERSION b/VERSION index ab5b18284b..11615b28ab 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -2.1-1059 +2.1-1072 diff --git a/doc/scripts/DocSourcesList.cmake b/doc/scripts/DocSourcesList.cmake index 5756d75ea3..7abc3e6bb8 100644 --- a/doc/scripts/DocSourcesList.cmake +++ b/doc/scripts/DocSourcesList.cmake @@ -71,6 +71,8 @@ rest_target(${CMAKE_BINARY_DIR}/scripts base/bif/plugins/Bro_TCP.events.bif.bro) rest_target(${CMAKE_BINARY_DIR}/scripts base/bif/plugins/Bro_TCP.functions.bif.bro) rest_target(${CMAKE_BINARY_DIR}/scripts base/bif/plugins/Bro_Teredo.events.bif.bro) rest_target(${CMAKE_BINARY_DIR}/scripts base/bif/plugins/Bro_UDP.events.bif.bro) +rest_target(${CMAKE_BINARY_DIR}/scripts base/bif/plugins/Bro_Unified2.events.bif.bro) +rest_target(${CMAKE_BINARY_DIR}/scripts base/bif/plugins/Bro_Unified2.types.bif.bro) rest_target(${CMAKE_BINARY_DIR}/scripts base/bif/plugins/Bro_ZIP.events.bif.bro) rest_target(${CMAKE_BINARY_DIR}/scripts base/bif/reporter.bif.bro) rest_target(${CMAKE_BINARY_DIR}/scripts base/bif/strings.bif.bro) @@ -78,6 +80,7 @@ rest_target(${CMAKE_BINARY_DIR}/scripts base/bif/top-k.bif.bro) rest_target(${CMAKE_BINARY_DIR}/scripts base/bif/types.bif.bro) rest_target(${psd} base/files/extract/main.bro) rest_target(${psd} base/files/hash/main.bro) +rest_target(${psd} base/files/unified2/main.bro) rest_target(${psd} base/frameworks/analyzer/main.bro) rest_target(${psd} base/frameworks/cluster/main.bro) rest_target(${psd} base/frameworks/cluster/nodes/manager.bro) diff --git a/scripts/base/files/unified2/__load__.bro b/scripts/base/files/unified2/__load__.bro new file mode 100644 index 0000000000..a10fe855df --- /dev/null +++ b/scripts/base/files/unified2/__load__.bro @@ -0,0 +1 @@ +@load ./main diff --git a/scripts/base/files/unified2/main.bro b/scripts/base/files/unified2/main.bro new file mode 100644 index 0000000000..a74c10e908 --- /dev/null +++ b/scripts/base/files/unified2/main.bro @@ -0,0 +1,244 @@ + +@load base/utils/dir +@load base/utils/paths + +module Unified2; + +export { + redef enum Log::ID += { LOG }; + + ## Directory to watch for Unified2 files. + const watch_file = "" &redef; + + ## File to watch for Unified2 records. + const watch_dir = "" &redef; + + ## The sid-msg.map file you would like to use for your alerts. + const sid_msg = "" &redef; + + ## The gen-msg.map file you would like to use for your alerts. + const gen_msg = "" &redef; + + ## The classification.config file you would like to use for your alerts. + const classification_config = "" &redef; + + ## Reconstructed "alert" which combines related events + ## and packets. + global alert: event(f: fa_file, ev: Unified2::IDSEvent, pkt: Unified2::Packet); + + type PacketID: record { + src_ip: addr; + src_p: port; + dst_ip: addr; + dst_p: port; + } &log; + + type Info: record { + ## Timestamp attached to the alert. + ts: time &log; + ## Addresses and ports for the connection. + id: PacketID &log; + ## Sensor that originated this event. + sensor_id: count &log; + ## Sig id for this generator. + signature_id: count &log; + ## A string representation of the "signature_id" field if a sid_msg.map file was loaded. + signature: string &log &optional; + ## Which generator generated the alert? + generator_id: count &log; + ## A string representation of the "generator_id" field if a gen_msg.map file was loaded. + generator: string &log &optional; + ## Sig revision for this id. + signature_revision: count &log; + ## Event classification. + classification_id: count &log; + ## Descriptive classification string, + classification: string &log &optional; + ## Event priority. + priority_id: count &log; + ## Event ID. + event_id: count &log; + ## Some of the packet data. + packet: string &log &optional; + } &log; + + ## The event for accessing logged records. + global log_unified2: event(rec: Info); +} + +# Mappings for extended information from alerts. +global classification_map: table[count] of string; +global sid_map: table[count] of string; +global gen_map: table[count] of string; + +# For reading in config files. +type OneLine: record { + line: string; +}; + +function create_info(ev: IDSEvent): Info + { + local info = Info($ts=ev$ts, + $id=PacketID($src_ip=ev$src_ip, $src_p=ev$src_p, + $dst_ip=ev$dst_ip, $dst_p=ev$dst_p), + $sensor_id=ev$sensor_id, + $signature_id=ev$signature_id, + $generator_id=ev$generator_id, + $signature_revision=ev$signature_revision, + $classification_id=ev$classification_id, + $priority_id=ev$priority_id, + $event_id=ev$event_id); + + if ( ev$signature_id in sid_map ) + info$signature=sid_map[ev$signature_id]; + if ( ev$generator_id in gen_map ) + info$generator=gen_map[ev$generator_id]; + if ( ev$classification_id in classification_map ) + info$classification=classification_map[ev$classification_id]; + + return info; + } + +redef record fa_file += { + ## Recently received IDS events. This is primarily used + ## for tying together Unified2 events and packets. + u2_events: table[count] of Unified2::IDSEvent + &optional &create_expire=5sec + &expire_func=function(t: table[count] of Unified2::IDSEvent, event_id: count): interval + { + Log::write(LOG, create_info(t[event_id])); + return 0secs; + }; +}; + +event Unified2::read_sid_msg_line(desc: Input::EventDescription, tpe: Input::Event, line: string) + { + local parts = split_n(line, / \|\| /, F, 100); + if ( |parts| >= 2 && /^[0-9]+$/ in parts[1] ) + sid_map[to_count(parts[1])] = parts[2]; + } + +event Unified2::read_gen_msg_line(desc: Input::EventDescription, tpe: Input::Event, line: string) + { + local parts = split_n(line, / \|\| /, F, 3); + if ( |parts| >= 2 && /^[0-9]+$/ in parts[1] ) + gen_map[to_count(parts[1])] = parts[3]; + } + +event Unified2::read_classification_line(desc: Input::EventDescription, tpe: Input::Event, line: string) + { + local parts = split_n(line, /: /, F, 2); + if ( |parts| == 2 ) + { + local parts2 = split_n(parts[2], /,/, F, 4); + if ( |parts2| > 1 ) + classification_map[|classification_map|+1] = parts2[1]; + } + } + +event bro_init() &priority=5 + { + Log::create_stream(Unified2::LOG, [$columns=Info, $ev=log_unified2]); + + if ( sid_msg != "" ) + { + Input::add_event([$source=sid_msg, + $reader=Input::READER_RAW, + $mode=Input::REREAD, + $name=sid_msg, + $fields=Unified2::OneLine, + $want_record=F, + $ev=Unified2::read_sid_msg_line]); + } + + if ( gen_msg != "" ) + { + Input::add_event([$source=gen_msg, + $name=gen_msg, + $reader=Input::READER_RAW, + $mode=Input::REREAD, + $fields=Unified2::OneLine, + $want_record=F, + $ev=Unified2::read_gen_msg_line]); + } + + if ( classification_config != "" ) + { + Input::add_event([$source=classification_config, + $name=classification_config, + $reader=Input::READER_RAW, + $mode=Input::REREAD, + $fields=Unified2::OneLine, + $want_record=F, + $ev=Unified2::read_classification_line]); + } + + if ( watch_dir != "" ) + { + Dir::monitor(watch_dir, function(fname: string) + { + Input::add_analysis([$source=fname, + $reader=Input::READER_BINARY, + $mode=Input::STREAM, + $name=fname]); + }, 10secs); + } + + if ( watch_file != "" ) + { + Input::add_analysis([$source=watch_file, + $reader=Input::READER_BINARY, + $mode=Input::STREAM, + $name=watch_file]); + } + } + +event file_new(f: fa_file) + { + local file_dir = ""; + local parts = split_all(f$source, /\/[^\/]*$/); + if ( |parts| == 3 ) + file_dir = parts[1]; + + if ( (watch_file != "" && f$source == watch_file) || + (watch_dir != "" && compress_path(watch_dir) == file_dir) ) + { + Files::add_analyzer(f, Files::ANALYZER_UNIFIED2); + f$u2_events = table(); + } + } + +event unified2_event(f: fa_file, ev: Unified2::IDSEvent) + { + f$u2_events[ev$event_id] = ev; + } + +event unified2_packet(f: fa_file, pkt: Unified2::Packet) + { + if ( f?$u2_events && pkt$event_id in f$u2_events) + { + local ev = f$u2_events[pkt$event_id]; + event Unified2::alert(f, ev, pkt); + delete f$u2_events[pkt$event_id]; + } + } + +event Unified2::alert(f: fa_file, ev: IDSEvent, pkt: Packet) + { + local info = create_info(ev); + info$packet=pkt$data; + Log::write(LOG, info); + } + +event file_state_remove(f: fa_file) + { + if ( f?$u2_events ) + { + # In case any events never had matching packets, flush + # the extras to the log. + for ( i in f$u2_events ) + { + Log::write(LOG, create_info(f$u2_events[i])); + } + } + } diff --git a/scripts/base/init-bare.bro b/scripts/base/init-bare.bro index ea79fb23a0..cc69fd702e 100644 --- a/scripts/base/init-bare.bro +++ b/scripts/base/init-bare.bro @@ -2686,6 +2686,42 @@ type ModbusHeaders: record { function_code: count; }; +module Unified2; +export { + type Unified2::IDSEvent: record { + sensor_id: count; + event_id: count; + ts: time; + signature_id: count; + generator_id: count; + signature_revision: count; + classification_id: count; + priority_id: count; + src_ip: addr; + dst_ip: addr; + src_p: port; + dst_p: port; + impact_flag: count; + impact: count; + blocked: count; + ## Not available in "legacy" IDS events. + mpls_label: count &optional; + ## Not available in "legacy" IDS events. + vlan_id: count &optional; + ## Only available in "legacy" IDS events. + packet_action: count &optional; + }; + + type Unified2::Packet: record { + sensor_id: count; + event_id: count; + event_second: count; + packet_ts: time; + link_type: count; + data: string; + }; +} + module SOCKS; export { ## This record is for a SOCKS client or server to provide either a diff --git a/scripts/base/init-default.bro b/scripts/base/init-default.bro index 913aac321d..202f8eaaab 100644 --- a/scripts/base/init-default.bro +++ b/scripts/base/init-default.bro @@ -56,5 +56,7 @@ @load base/files/hash @load base/files/extract +@load base/files/unified2 + @load base/misc/find-checksum-offloading diff --git a/src/binpac_bro.h b/src/binpac_bro.h index 5902c52113..8dc3df9e6f 100644 --- a/src/binpac_bro.h +++ b/src/binpac_bro.h @@ -12,6 +12,7 @@ namespace analyzer { class Analyzer; } #include "event.bif.func_h" #include "TunnelEncapsulation.h" #include "analyzer/Analyzer.h" +#include "file_analysis/Analyzer.h" #include "Conn.h" #include "binpac.h" @@ -19,6 +20,7 @@ namespace analyzer { class Analyzer; } namespace binpac { typedef analyzer::Analyzer* BroAnalyzer; +typedef file_analysis::Analyzer BroFileAnalyzer; typedef Val* BroVal; typedef PortVal* BroPortVal; typedef StringVal* BroStringVal; diff --git a/src/file_analysis/analyzer/CMakeLists.txt b/src/file_analysis/analyzer/CMakeLists.txt index bfafcd2894..1e19b7bd11 100644 --- a/src/file_analysis/analyzer/CMakeLists.txt +++ b/src/file_analysis/analyzer/CMakeLists.txt @@ -1,3 +1,4 @@ add_subdirectory(data_event) add_subdirectory(extract) add_subdirectory(hash) +add_subdirectory(unified2) diff --git a/src/file_analysis/analyzer/unified2/CMakeLists.txt b/src/file_analysis/analyzer/unified2/CMakeLists.txt new file mode 100644 index 0000000000..4a9b11ef92 --- /dev/null +++ b/src/file_analysis/analyzer/unified2/CMakeLists.txt @@ -0,0 +1,11 @@ + +include(BroPlugin) + +include_directories(BEFORE ${CMAKE_CURRENT_SOURCE_DIR} + ${CMAKE_CURRENT_BINARY_DIR}) + +bro_plugin_begin(Bro Unified2) +bro_plugin_cc(Unified2.cc Plugin.cc ../../Analyzer.cc) +bro_plugin_bif(events.bif types.bif) +bro_plugin_pac(unified2.pac unified2-file.pac unified2-analyzer.pac) +bro_plugin_end() diff --git a/src/file_analysis/analyzer/unified2/Plugin.cc b/src/file_analysis/analyzer/unified2/Plugin.cc new file mode 100644 index 0000000000..130ed89ce9 --- /dev/null +++ b/src/file_analysis/analyzer/unified2/Plugin.cc @@ -0,0 +1,35 @@ +// See the file "COPYING" in the main distribution directory for copyright. + +#include "plugin/Plugin.h" + +#include "file_analysis/Component.h" + +#include "Unified2.h" + +namespace plugin { namespace Bro_Unified2 { + +class Plugin : public plugin::Plugin { +protected: + void InitPreScript() + { + SetName("Bro::Unified2"); + SetVersion(-1); + SetAPIVersion(BRO_PLUGIN_API_VERSION); + SetDynamicPlugin(false); + + SetDescription("Analyze Unified2 alert files."); + + AddComponent(new ::file_analysis::Component("UNIFIED2", + ::file_analysis::Unified2::Instantiate)); + + extern std::list > __bif_events_init(); + AddBifInitFunction(&__bif_events_init); + + extern std::list > __bif_types_init(); + AddBifInitFunction(&__bif_types_init); + } +}; + +Plugin __plugin; + +} } diff --git a/src/file_analysis/analyzer/unified2/Unified2.cc b/src/file_analysis/analyzer/unified2/Unified2.cc new file mode 100644 index 0000000000..0fbb17bb93 --- /dev/null +++ b/src/file_analysis/analyzer/unified2/Unified2.cc @@ -0,0 +1,38 @@ +// See the file "COPYING" in the main distribution directory for copyright. + +#include "Unified2.h" +#include "file_analysis/Manager.h" + +using namespace file_analysis; + +Unified2::Unified2(RecordVal* args, File* file) + : file_analysis::Analyzer(file_mgr->GetComponentTag("UNIFIED2"), args, file) + { + interp = new binpac::Unified2::Unified2_Analyzer(this); + } + +Unified2::~Unified2() + { + delete interp; + } + +file_analysis::Analyzer* Unified2::Instantiate(RecordVal* args, File* file) + { + return new Unified2(args, file); + } + +bool Unified2::DeliverStream(const u_char* data, uint64 len) + { + try + { + interp->NewData(true, data, data + len); + } + + catch ( const binpac::Exception& e ) + { + printf("Binpac exception: %s\n", e.c_msg()); + return false; + } + + return true; + } diff --git a/src/file_analysis/analyzer/unified2/Unified2.h b/src/file_analysis/analyzer/unified2/Unified2.h new file mode 100644 index 0000000000..bd172d2b6b --- /dev/null +++ b/src/file_analysis/analyzer/unified2/Unified2.h @@ -0,0 +1,38 @@ +// See the file "COPYING" in the main distribution directory for copyright. + +#ifndef FILE_ANALYSIS_UNIFIED2_H +#define FILE_ANALYSIS_UNIFIED2_H + +#include + +#include "Val.h" +#include "File.h" +#include "Analyzer.h" +#include "unified2_pac.h" + +namespace file_analysis { + +/** + * An analyzer to extract content of files from local disk. + */ +class Unified2 : public file_analysis::Analyzer { +public: + virtual ~Unified2(); + + virtual bool DeliverStream(const u_char* data, uint64 len); + + static file_analysis::Analyzer* Instantiate(RecordVal* args, File* file); + +protected: + Unified2(RecordVal* args, File* file); + +private: + binpac::Unified2::Unified2_Analyzer* interp; + + string filename; + int fd; +}; + +} // namespace file_analysis + +#endif diff --git a/src/file_analysis/analyzer/unified2/events.bif b/src/file_analysis/analyzer/unified2/events.bif new file mode 100644 index 0000000000..c5f3dda6a4 --- /dev/null +++ b/src/file_analysis/analyzer/unified2/events.bif @@ -0,0 +1,7 @@ + +## Abstract all of the various Unified2 event formats into +## a single event. +event unified2_event%(f: fa_file, ev: Unified2::IDSEvent%); + +## The Unified2 packet format event. +event unified2_packet%(f: fa_file, pkt: Unified2::Packet%); diff --git a/src/file_analysis/analyzer/unified2/types.bif b/src/file_analysis/analyzer/unified2/types.bif new file mode 100644 index 0000000000..20965f7038 --- /dev/null +++ b/src/file_analysis/analyzer/unified2/types.bif @@ -0,0 +1,2 @@ +type Unified2::IDSEvent: record; +type Unified2::Packet: record; diff --git a/src/file_analysis/analyzer/unified2/unified2-analyzer.pac b/src/file_analysis/analyzer/unified2/unified2-analyzer.pac new file mode 100644 index 0000000000..11072f140b --- /dev/null +++ b/src/file_analysis/analyzer/unified2/unified2-analyzer.pac @@ -0,0 +1,170 @@ + +%extern{ +#include "Event.h" +#include "file_analysis/File.h" +#include "events.bif.h" +#include "types.bif.h" +#include "IPAddr.h" +%} + +refine flow Flow += { + + %member{ + %} + + %init{ + %} + + %eof{ + %} + + %cleanup{ + %} + + function ts_to_double(ts: Time): double + %{ + double t = ${ts.seconds} + (${ts.microseconds} / 1000000); + return t; + %} + + function unified2_addr_to_bro_addr(a: uint32[]): AddrVal + %{ + if ( a->size() == 1 ) + { + return new AddrVal(IPAddr(IPv4, &(a->at(0)), IPAddr::Host)); + } + else if ( a->size() == 4 ) + { + uint32 tmp[4] = { a->at(0), a->at(1), a->at(2), a->at(3) }; + return new AddrVal(IPAddr(IPv6, tmp, IPAddr::Host)); + } + else + { + // Should never reach here. + return new AddrVal(1); + } + %} + + function to_port(n: uint16, p: uint8): PortVal + %{ + TransportProto proto = TRANSPORT_UNKNOWN; + switch ( p ) { + case 1: proto = TRANSPORT_ICMP; break; + case 6: proto = TRANSPORT_TCP; break; + case 17: proto = TRANSPORT_UDP; break; + } + + return new PortVal(n, proto); + %} + + #function proc_record(rec: Record) : bool + # %{ + # return true; + # %} + + function proc_ids_event(ev: IDS_Event) : bool + %{ + if ( ::unified2_event ) + { + RecordVal* ids_event = new RecordVal(BifType::Record::Unified2::IDSEvent); + ids_event->Assign(0, new Val(${ev.sensor_id}, TYPE_COUNT)); + ids_event->Assign(1, new Val(${ev.event_id}, TYPE_COUNT)); + ids_event->Assign(2, new Val(ts_to_double(${ev.ts}), TYPE_TIME)); + ids_event->Assign(3, new Val(${ev.signature_id}, TYPE_COUNT)); + ids_event->Assign(4, new Val(${ev.generator_id}, TYPE_COUNT)); + ids_event->Assign(5, new Val(${ev.signature_revision}, TYPE_COUNT)); + ids_event->Assign(6, new Val(${ev.classification_id}, TYPE_COUNT)); + ids_event->Assign(7, new Val(${ev.priority_id}, TYPE_COUNT)); + ids_event->Assign(8, unified2_addr_to_bro_addr(${ev.src_ip})); + ids_event->Assign(9, unified2_addr_to_bro_addr(${ev.dst_ip})); + ids_event->Assign(10, to_port(${ev.src_p}, ${ev.protocol})); + ids_event->Assign(11, to_port(${ev.dst_p}, ${ev.protocol})); + ids_event->Assign(17, new Val(${ev.packet_action}, TYPE_COUNT)); + + val_list* vl = new val_list(); + vl->append(connection()->bro_analyzer()->GetFile()->GetVal()->Ref()); + vl->append(ids_event); + mgr.QueueEvent(::unified2_event, vl, SOURCE_LOCAL); + } + return true; + %} + + function proc_ids_event_2(ev: IDS_Event_2) : bool + %{ + if ( ::unified2_event ) + { + RecordVal* ids_event = new RecordVal(BifType::Record::Unified2::IDSEvent); + ids_event->Assign(0, new Val(${ev.sensor_id}, TYPE_COUNT)); + ids_event->Assign(1, new Val(${ev.event_id}, TYPE_COUNT)); + ids_event->Assign(2, new Val(ts_to_double(${ev.ts}), TYPE_TIME)); + ids_event->Assign(3, new Val(${ev.signature_id}, TYPE_COUNT)); + ids_event->Assign(4, new Val(${ev.generator_id}, TYPE_COUNT)); + ids_event->Assign(5, new Val(${ev.signature_revision}, TYPE_COUNT)); + ids_event->Assign(6, new Val(${ev.classification_id}, TYPE_COUNT)); + ids_event->Assign(7, new Val(${ev.priority_id}, TYPE_COUNT)); + ids_event->Assign(8, unified2_addr_to_bro_addr(${ev.src_ip})); + ids_event->Assign(9, unified2_addr_to_bro_addr(${ev.dst_ip})); + ids_event->Assign(10, to_port(${ev.src_p}, ${ev.protocol})); + ids_event->Assign(11, to_port(${ev.dst_p}, ${ev.protocol})); + ids_event->Assign(12, new Val(${ev.impact_flag}, TYPE_COUNT)); + ids_event->Assign(13, new Val(${ev.impact}, TYPE_COUNT)); + ids_event->Assign(14, new Val(${ev.blocked}, TYPE_COUNT)); + ids_event->Assign(15, new Val(${ev.mpls_label}, TYPE_COUNT)); + ids_event->Assign(16, new Val(${ev.vlan_id}, TYPE_COUNT)); + + val_list* vl = new val_list(); + vl->append(connection()->bro_analyzer()->GetFile()->GetVal()->Ref()); + vl->append(ids_event); + mgr.QueueEvent(::unified2_event, vl, SOURCE_LOCAL); + } + + return true; + %} + + function proc_packet(pkt: Packet) : bool + %{ + if ( ::unified2_packet ) + { + RecordVal* packet = new RecordVal(BifType::Record::Unified2::Packet); + packet->Assign(0, new Val(${pkt.sensor_id}, TYPE_COUNT)); + packet->Assign(1, new Val(${pkt.event_id}, TYPE_COUNT)); + packet->Assign(2, new Val(${pkt.event_second}, TYPE_COUNT)); + packet->Assign(3, new Val(ts_to_double(${pkt.packet_ts}), TYPE_TIME)); + packet->Assign(4, new Val(${pkt.link_type}, TYPE_COUNT)); + packet->Assign(5, bytestring_to_val(${pkt.packet_data})); + + val_list* vl = new val_list(); + vl->append(connection()->bro_analyzer()->GetFile()->GetVal()->Ref()); + vl->append(packet); + mgr.QueueEvent(::unified2_packet, vl, SOURCE_LOCAL); + } + + return true; + %} + + #function proc_unknown_record_type(rec: UnknownRecordType) : bool + # %{ + # printf("unknown packet type\n"); + # return true; + # %} +}; + +#refine typeattr Record += &let { +# proc : bool = $context.flow.proc_record(this); +#}; + +refine typeattr IDS_Event += &let { + proc : bool = $context.flow.proc_ids_event(this); +}; + +refine typeattr IDS_Event_2 += &let { + proc : bool = $context.flow.proc_ids_event_2(this); +}; + +refine typeattr Packet += &let { + proc : bool = $context.flow.proc_packet(this); +}; + +#refine typeattr UnknownRecordType += &let { +# proc : bool = $context.flow.proc_unknown_record_type(this); +#}; diff --git a/src/file_analysis/analyzer/unified2/unified2-file.pac b/src/file_analysis/analyzer/unified2/unified2-file.pac new file mode 100644 index 0000000000..f0d8ca9187 --- /dev/null +++ b/src/file_analysis/analyzer/unified2/unified2-file.pac @@ -0,0 +1,91 @@ + +enum Types { + PACKET = 2, + IDS_EVENT = 7, + IDS_EVENT_IPV6 = 72, + IDS_EVENT_2 = 104, + IDS_EVENT_IPV6_2 = 105, + EXTRA_DATA = 110, +}; + +type Time = record { + seconds: uint32; + microseconds: uint32; +} &byteorder=bigendian; + +type Record = record { + rtype: uint32; + length: uint32; + data: case rtype of { + PACKET -> packet: Packet(this); + IDS_EVENT -> ids_event: IDS_Event(this, 1); + IDS_EVENT_IPV6 -> ids_event_ipv6: IDS_Event(this, 4); + IDS_EVENT_2 -> ids_event_vlan: IDS_Event_2(this, 1); + IDS_EVENT_IPV6_2 -> ids_event_ipv6_vlan: IDS_Event_2(this, 4); + #EXTRA_DATA -> extra_data: ExtraData(this); + default -> unknown_record_type: UnknownRecordType(this); + }; +} &byteorder=bigendian &length=length+8; + +type IDS_Event(rec: Record, ip_len: int) = record { + sensor_id: uint32; + event_id: uint32; + ts: Time; + signature_id: uint32; + generator_id: uint32; + signature_revision: uint32; + classification_id: uint32; + priority_id: uint32; + src_ip: uint32[ip_len]; + dst_ip: uint32[ip_len]; + src_p: uint16; + dst_p: uint16; + protocol: uint8; + packet_action: uint8; +} &byteorder=bigendian; + +type IDS_Event_2(rec: Record, ip_len: int) = record { + sensor_id: uint32; + event_id: uint32; + ts: Time; + signature_id: uint32; + generator_id: uint32; + signature_revision: uint32; + classification_id: uint32; + priority_id: uint32; + src_ip: uint32[ip_len]; + dst_ip: uint32[ip_len]; + src_p: uint16; + dst_p: uint16; + protocol: uint8; + impact_flag: uint8; + impact: uint8; + blocked: uint8; + mpls_label: uint32; + vlan_id: uint16; + pad: uint16; +} &byteorder=bigendian; + +type Packet(rec: Record) = record { + sensor_id: uint32; + event_id: uint32; + event_second: uint32; + packet_ts: Time; + link_type: uint32; + packet_len: uint32; + packet_data: bytestring &length=packet_len; +} &byteorder=bigendian; + +type ExtraData(rec: Record) = record { + sensor_id: uint32; + event_id: uint32; + event_second: uint32; + extra_type: uint32; + data_type: uint32; + blob_len: uint32; + blob: bytestring &length=blob_len; +} &byteorder=bigendian &length=rec.length; + +type UnknownRecordType(rec: Record) = record { + data: bytestring &transient &length=rec.length; +} &byteorder=bigendian &length=rec.length; diff --git a/src/file_analysis/analyzer/unified2/unified2.pac b/src/file_analysis/analyzer/unified2/unified2.pac new file mode 100644 index 0000000000..ddc7dc5315 --- /dev/null +++ b/src/file_analysis/analyzer/unified2/unified2.pac @@ -0,0 +1,21 @@ + +%include binpac.pac +%include bro.pac + +analyzer Unified2 withcontext { + analyzer: Unified2_Analyzer; + flow: Flow; +}; + +analyzer Unified2_Analyzer(bro_analyzer: BroFileAnalyzer) { + downflow = Flow; + upflow = Flow; +}; + +%include unified2-file.pac + +flow Flow { + flowunit = Record withcontext(connection, this); +}; + +%include unified2-analyzer.pac diff --git a/testing/btest/Baseline/coverage.bare-load-baseline/canonified_loaded_scripts.log b/testing/btest/Baseline/coverage.bare-load-baseline/canonified_loaded_scripts.log index 5506dcd01f..545bf70e7e 100644 --- a/testing/btest/Baseline/coverage.bare-load-baseline/canonified_loaded_scripts.log +++ b/testing/btest/Baseline/coverage.bare-load-baseline/canonified_loaded_scripts.log @@ -3,7 +3,7 @@ #empty_field (empty) #unset_field - #path loaded_scripts -#open 2013-08-12-18-22-48 +#open 2013-08-14-01-19-27 #fields name #types string scripts/base/init-bare.bro @@ -62,6 +62,8 @@ scripts/base/init-bare.bro build/scripts/base/bif/plugins/Bro_TCP.functions.bif.bro build/scripts/base/bif/plugins/Bro_Teredo.events.bif.bro build/scripts/base/bif/plugins/Bro_UDP.events.bif.bro + build/scripts/base/bif/plugins/Bro_Unified2.events.bif.bro + build/scripts/base/bif/plugins/Bro_Unified2.types.bif.bro build/scripts/base/bif/plugins/Bro_ZIP.events.bif.bro scripts/base/frameworks/logging/__load__.bro scripts/base/frameworks/logging/main.bro @@ -95,4 +97,4 @@ scripts/base/init-bare.bro build/scripts/base/bif/top-k.bif.bro scripts/policy/misc/loaded-scripts.bro scripts/base/utils/paths.bro -#close 2013-08-12-18-22-48 +#close 2013-08-14-01-19-27 diff --git a/testing/btest/Baseline/coverage.default-load-baseline/canonified_loaded_scripts.log b/testing/btest/Baseline/coverage.default-load-baseline/canonified_loaded_scripts.log index 3cb217e814..d37fbb117c 100644 --- a/testing/btest/Baseline/coverage.default-load-baseline/canonified_loaded_scripts.log +++ b/testing/btest/Baseline/coverage.default-load-baseline/canonified_loaded_scripts.log @@ -3,7 +3,7 @@ #empty_field (empty) #unset_field - #path loaded_scripts -#open 2013-08-12-18-22-49 +#open 2013-08-14-01-19-28 #fields name #types string scripts/base/init-bare.bro @@ -62,6 +62,8 @@ scripts/base/init-bare.bro build/scripts/base/bif/plugins/Bro_TCP.functions.bif.bro build/scripts/base/bif/plugins/Bro_Teredo.events.bif.bro build/scripts/base/bif/plugins/Bro_UDP.events.bif.bro + build/scripts/base/bif/plugins/Bro_Unified2.events.bif.bro + build/scripts/base/bif/plugins/Bro_Unified2.types.bif.bro build/scripts/base/bif/plugins/Bro_ZIP.events.bif.bro scripts/base/frameworks/logging/__load__.bro scripts/base/frameworks/logging/main.bro @@ -210,6 +212,8 @@ scripts/base/init-default.bro scripts/base/files/hash/main.bro scripts/base/files/extract/__load__.bro scripts/base/files/extract/main.bro + scripts/base/files/unified2/__load__.bro + scripts/base/files/unified2/main.bro scripts/base/misc/find-checksum-offloading.bro scripts/policy/misc/loaded-scripts.bro -#close 2013-08-12-18-22-49 +#close 2013-08-14-01-19-28 diff --git a/testing/btest/Baseline/scripts.base.files.unified2.alert/unified2.log b/testing/btest/Baseline/scripts.base.files.unified2.alert/unified2.log new file mode 100644 index 0000000000..b86dedf23b --- /dev/null +++ b/testing/btest/Baseline/scripts.base.files.unified2.alert/unified2.log @@ -0,0 +1,11 @@ +#separator \x09 +#set_separator , +#empty_field (empty) +#unset_field - +#path unified2 +#open 2013-08-13-07-16-01 +#fields ts id.src_ip id.src_p id.dst_ip id.dst_p sensor_id signature_id signature generator_id generator signature_revision classification_id classification priority_id event_id packet +#types time addr port addr port count count string count string count count string count count string +1323827323.000000 192.168.1.72 50185 74.125.225.49 80 0 2003058 ET MALWARE 180solutions (Zango) Spyware Installer Download 1 snort general alert 5 21 trojan-activity 1 2 \xd80bH\xc5\xb5x\xca9\xb7\xe4r\x08\x00E\x10\x00\\x1a\xce@\x00@\x062\x1f\xc0\xa8\x01HJ}\xe11\xc4\x09\x00P*\xa8bv]z/\xde\x80\x18\x82+\x88,\x00\x00\x01\x01\x08\x0a\x17J\x83Q\xfe\xad\xac\x1aGET /Zango/ZangoInstaller.exe HTTP/1.0\x0d\x0a +1323827344.000000 192.168.1.72 49862 199.47.216.144 80 0 2012647 ET POLICY Dropbox.com Offsite File Backup in Use 1 snort general alert 3 33 policy-violation 1 3 \xd80bH\xc5\xb5x\xca9\xb7\xe4r\x08\x00E\x00\x00\xf8Q\xdf@\x00@\x06\x86p\xc0\xa8\x01H\xc7/\xd8\x90\xc2\xc6\x00P\x9cm\x97U\xf07\x084\x80\x18\x82\x18%<\x00\x00\x01\x01\x08\x0a\x17J\xd7\xde\x00\x92\x81\xc5GET /subscribe?host_int=43112345&ns_map=123456_1234524412104916591&ts=1323827344 HTTP/1.1\x0d\x0aHost: notify1.dropbox.com\x0d\x0aAccept-Encoding: identity\x0d\x0aConnection: keep-alive\x0d\x0aX-Dropbox-Locale: en_US\x0d\x0a\x0d\x0a +#close 2013-08-13-07-16-01 diff --git a/testing/btest/Baseline/scripts.base.frameworks.file-analysis.logging/files.log b/testing/btest/Baseline/scripts.base.frameworks.file-analysis.logging/files.log index 2663184b88..344ffa7802 100644 --- a/testing/btest/Baseline/scripts.base.frameworks.file-analysis.logging/files.log +++ b/testing/btest/Baseline/scripts.base.frameworks.file-analysis.logging/files.log @@ -3,8 +3,8 @@ #empty_field (empty) #unset_field - #path files -#open 2013-07-25-16-57-31 +#open 2013-08-14-04-50-17 #fields ts fuid tx_hosts rx_hosts conn_uids source depth analyzers mime_type filename duration local_orig is_orig seen_bytes total_bytes missing_bytes overflow_bytes timedout parent_fuid md5 sha1 sha256 extracted #types time string table[addr] table[addr] table[string] string count table[string] string string interval bool bool count count count count bool string string string string string 1362692527.009721 G75mcAsU764 192.150.187.43 141.142.228.5 UWkUyAuUGXf HTTP 0 SHA256,DATA_EVENT,MD5,EXTRACT,SHA1 text/plain - 0.000054 - F 4705 4705 0 0 F - 397168fd09991a0e712254df7bc639ac 1dd7ac0398df6cbc0696445a91ec681facf4dc47 4e7c7ef0984119447e743e3ec77e1de52713e345cde03fe7df753a35849bed18 G75mcAsU764-file -#close 2013-07-25-16-57-31 +#close 2013-08-14-04-50-17 diff --git a/testing/btest/Files/unified2.u2 b/testing/btest/Files/unified2.u2 new file mode 100755 index 0000000000..2619152e14 Binary files /dev/null and b/testing/btest/Files/unified2.u2 differ diff --git a/testing/btest/btest.cfg b/testing/btest/btest.cfg index 7ccf99eea8..cc06eb4d5d 100644 --- a/testing/btest/btest.cfg +++ b/testing/btest/btest.cfg @@ -14,6 +14,7 @@ LC_ALL=C BTEST_PATH=%(testbase)s/../../aux/btest PATH=%(testbase)s/../../build/src:%(testbase)s/../scripts:%(testbase)s/../../aux/btest:%(testbase)s/../../aux/bro-aux/bro-cut:%(default_path)s TRACES=%(testbase)s/Traces +FILES=%(testbase)s/Files SCRIPTS=%(testbase)s/../scripts DIST=%(testbase)s/../.. BUILD=%(testbase)s/../../build diff --git a/testing/btest/scripts/base/files/unified2/alert.bro b/testing/btest/scripts/base/files/unified2/alert.bro new file mode 100644 index 0000000000..eca1ca036c --- /dev/null +++ b/testing/btest/scripts/base/files/unified2/alert.bro @@ -0,0 +1,76 @@ +# @TEST-EXEC: bro -b %INPUT Unified2::watch_file=$FILES/unified2.u2 +# @TEST-EXEC: btest-diff unified2.log + +@TEST-START-FILE sid_msg.map +2003058 || ET MALWARE 180solutions (Zango) Spyware Installer Download || url,doc.emergingthreats.net/bin/view/Main/2003058 || url,securityresponse.symantec.com/avcenter/venc/data/pf/adware.180search.html +2012647 || ET POLICY Dropbox.com Offsite File Backup in Use || url,dereknewton.com/2011/04/dropbox-authentication-static-host-ids/ || url,www.dropbox.com +@TEST-END-FILE + +@TEST-START-FILE gen_msg.map +1 || 1 || snort general alert +2 || 1 || tag: Tagged Packet +3 || 1 || snort dynamic alert +100 || 1 || spp_portscan: Portscan Detected +100 || 2 || spp_portscan: Portscan Status +100 || 3 || spp_portscan: Portscan Ended +101 || 1 || spp_minfrag: minfrag alert +@TEST-END-FILE + +@TEST-START-FILE classification.config +# +# config classification:shortname,short description,priority +# + +#Traditional classifications. These will be replaced soon + +config classification: not-suspicious,Not Suspicious Traffic,3 +config classification: unknown,Unknown Traffic,3 +config classification: bad-unknown,Potentially Bad Traffic, 2 +config classification: attempted-recon,Attempted Information Leak,2 +config classification: successful-recon-limited,Information Leak,2 +config classification: successful-recon-largescale,Large Scale Information Leak,2 +config classification: attempted-dos,Attempted Denial of Service,2 +config classification: successful-dos,Denial of Service,2 +config classification: attempted-user,Attempted User Privilege Gain,1 +config classification: unsuccessful-user,Unsuccessful User Privilege Gain,1 +config classification: successful-user,Successful User Privilege Gain,1 +config classification: attempted-admin,Attempted Administrator Privilege Gain,1 +config classification: successful-admin,Successful Administrator Privilege Gain,1 +config classification: rpc-portmap-decode,Decode of an RPC Query,2 +config classification: shellcode-detect,Executable Code was Detected,1 +config classification: string-detect,A Suspicious String was Detected,3 +config classification: suspicious-filename-detect,A Suspicious Filename was Detected,2 +config classification: suspicious-login,An Attempted Login Using a Suspicious Username was Detected,2 +config classification: system-call-detect,A System Call was Detected,2 +config classification: tcp-connection,A TCP Connection was Detected,4 +config classification: trojan-activity,A Network Trojan was Detected, 1 +config classification: unusual-client-port-connection,A Client was Using an Unusual Port,2 +config classification: network-scan,Detection of a Network Scan,3 +config classification: denial-of-service,Detection of a Denial of Service Attack,2 +config classification: non-standard-protocol,Detection of a Non-Standard Protocol or Event,2 +config classification: protocol-command-decode,Generic Protocol Command Decode,3 +config classification: web-application-activity,Access to a Potentially Vulnerable Web Application,2 +config classification: web-application-attack,Web Application Attack,1 +config classification: misc-activity,Misc activity,3 +config classification: misc-attack,Misc Attack,2 +config classification: icmp-event,Generic ICMP event,3 +config classification: inappropriate-content,Inappropriate Content was Detected,1 +config classification: policy-violation,Potential Corporate Privacy Violation,1 +config classification: default-login-attempt,Attempt to Login By a Default Username and Password,2 +@TEST-END-FILE + +redef exit_only_after_terminate = T; + +@load base/files/unified2 + +redef Unified2::sid_msg = @DIR+"/sid_msg.map"; +redef Unified2::gen_msg = @DIR+"/gen_msg.map"; +redef Unified2::classification_config = @DIR+"/classification.config"; +global i = 0; + +event Unified2::alert(f: fa_file, ev: Unified2::IDSEvent, pkt: Unified2::Packet) + { + ++i; + if ( i == 2 ) + terminate(); + } \ No newline at end of file