diff --git a/scripts/base/files/unified2/__load__.bro b/scripts/base/files/unified2/__load__.bro index d551be57d3..a10fe855df 100644 --- a/scripts/base/files/unified2/__load__.bro +++ b/scripts/base/files/unified2/__load__.bro @@ -1 +1 @@ -@load ./main \ No newline at end of file +@load ./main diff --git a/scripts/base/files/unified2/main.bro b/scripts/base/files/unified2/main.bro index 4d540cca3b..6d873a54cb 100644 --- a/scripts/base/files/unified2/main.bro +++ b/scripts/base/files/unified2/main.bro @@ -5,31 +5,148 @@ module Unified2; export { + redef enum Log::ID += { LOG }; + ## Directory to watch for Unified2 files. const watch_file = "" &redef; ## File to watch for Unified2 records. const watch_dir = "" &redef; + ## The sid-msg.map file you would like to use for your alerts. + const sid_msg = "" &redef; + + ## The gen-msg.map file you would like to use for your alerts. + const gen_msg = "" &redef; + + ## The classification.config file you would like to use for your alerts. + const classification_config = "" &redef; + ## Reconstructed "alert" which combines related events ## and packets. global alert: event(f: fa_file, ev: Unified2::IDSEvent, pkt: Unified2::Packet); + type PacketID: record { + src_ip: addr; + src_p: port; + dst_ip: addr; + dst_p: port; + } &log; + type Info: record { - ## The last received IDS event. This is primarily used - ## for tying together Unified2 events and packets. - current_event: Unified2::IDSEvent &optional; - }; + ## Timestamp attached to the alert. + ts: time &log; + ## Addresses and ports for the connection. + id: PacketID &log; + ## Sensor that originated this event. + sensor_id: count &log; + ## Sig id for this generator. + signature_id: count &log; + ## A string representation of the "signature_id" field if a sid_msg.map file was loaded. + signature: string &log &optional; + ## Which generator generated the alert? + generator_id: count &log; + ## A string representation of the "generator_id" field if a gen_msg.map file was loaded. + generator: string &log &optional; + ## Sig revision for this id. + signature_revision: count &log; + ## Event classification. + classification_id: count &log; + ## Descriptive classification string, + classification: string &log &optional; + ## Event priority. + priority_id: count &log; + ## Event ID. + event_id: count &log; + ## Some of the packet data. + packet: string &log; + } &log; redef record fa_file += { - ## Add a field to store per-file state about Unified2 - ## files. + ## Add a field to store per-file state about Unified2 files. unified2: Info &optional; + + ## Recently received IDS events. This is primarily used + ## for tying together Unified2 events and packets. + u2_events: table[count] of Unified2::IDSEvent &optional &create_expire=5sec; }; + + ## The event for accessing logged records. + global log_unified2: event(rec: Info); } +# Mappings for extended information from alerts. +global classification_map: table[count] of string; +global sid_map: table[count] of string; +global gen_map: table[count] of string; + +# For reading in config files. +type OneLine: record { + line: string; +}; + +event Unified2::read_sid_msg_line(desc: Input::EventDescription, tpe: Input::Event, line: string) + { + local parts = split_n(line, / \|\| /, F, 100); + if ( |parts| >= 2 && /^[0-9]+$/ in parts[1] ) + sid_map[to_count(parts[1])] = parts[2]; + } + +event Unified2::read_gen_msg_line(desc: Input::EventDescription, tpe: Input::Event, line: string) + { + local parts = split_n(line, / \|\| /, F, 3); + if ( |parts| >= 2 && /^[0-9]+$/ in parts[1] ) + gen_map[to_count(parts[1])] = parts[3]; + } + +event Unified2::read_classification_line(desc: Input::EventDescription, tpe: Input::Event, line: string) + { + local parts = split_n(line, /: /, F, 2); + if ( |parts| == 2 ) + { + local parts2 = split_n(parts[2], /,/, F, 4); + if ( |parts2| > 1 ) + classification_map[|classification_map|+1] = parts2[1]; + } + } + event bro_init() { + Log::create_stream(Unified2::LOG, [$columns=Info, $ev=log_unified2]); + + if ( sid_msg != "" ) + { + Input::add_event([$source=sid_msg, + $reader=Input::READER_RAW, + $mode=Input::REREAD, + $name=sid_msg, + $fields=Unified2::OneLine, + $want_record=F, + $ev=Unified2::read_sid_msg_line]); + } + + if ( gen_msg != "" ) + { + Input::add_event([$source=gen_msg, + $name=gen_msg, + $reader=Input::READER_RAW, + $mode=Input::REREAD, + $fields=Unified2::OneLine, + $want_record=F, + $ev=Unified2::read_gen_msg_line]); + } + + if ( classification_config != "" ) + { + Input::add_event([$source=classification_config, + $name=classification_config, + $reader=Input::READER_RAW, + $mode=Input::REREAD, + $fields=Unified2::OneLine, + $want_record=F, + $ev=Unified2::read_classification_line]); + } + if ( watch_dir != "" ) { Dir::monitor(watch_dir, function(fname: string) @@ -61,20 +178,45 @@ event file_new(f: fa_file) compress_path(watch_dir) == file_dir ) { Files::add_analyzer(f, Files::ANALYZER_UNIFIED2); - f$unified2 = Info(); + f$u2_events = table(); } } event unified2_event(f: fa_file, ev: Unified2::IDSEvent) { - f$unified2$current_event = ev; + f$u2_events[ev$event_id] = ev; } event unified2_packet(f: fa_file, pkt: Unified2::Packet) { - if ( f$unified2?$current_event ) + if ( f?$u2_events && pkt$event_id in f$u2_events) { - event Unified2::alert(f, f$unified2$current_event, pkt); - delete f$unified2$current_event; + local ev = f$u2_events[pkt$event_id]; + event Unified2::alert(f, ev, pkt); + delete f$u2_events[pkt$event_id]; } } + +event Unified2::alert(f: fa_file, ev: IDSEvent, pkt: Packet) + { + local info = Info($ts=ev$ts, + $id=PacketID($src_ip=ev$src_ip, $src_p=ev$src_p, + $dst_ip=ev$dst_ip, $dst_p=ev$dst_p), + $sensor_id=ev$sensor_id, + $signature_id=ev$signature_id, + $generator_id=ev$generator_id, + $signature_revision=ev$signature_revision, + $classification_id=ev$classification_id, + $priority_id=ev$priority_id, + $event_id=ev$event_id, + $packet=pkt$data); + + if ( ev$signature_id in sid_map ) + info$signature=sid_map[ev$signature_id]; + if ( ev$generator_id in gen_map ) + info$generator=gen_map[ev$generator_id]; + if ( ev$classification_id in classification_map ) + info$classification=classification_map[ev$classification_id]; + + Log::write(LOG, info); + } diff --git a/scripts/base/init-bare.bro b/scripts/base/init-bare.bro index ac8f13f1f6..a911b0946e 100644 --- a/scripts/base/init-bare.bro +++ b/scripts/base/init-bare.bro @@ -2713,12 +2713,12 @@ export { }; type Unified2::Packet: record { - sensor_id: count; - event_id: count; - event_second: count; - packet_ts: time; - link_type: count; - packet_data: string; + sensor_id: count; + event_id: count; + event_second: count; + packet_ts: time; + link_type: count; + data: string; }; } diff --git a/src/file_analysis/analyzer/unified2/unified2-analyzer.pac b/src/file_analysis/analyzer/unified2/unified2-analyzer.pac index 490754c47d..cef788096a 100644 --- a/src/file_analysis/analyzer/unified2/unified2-analyzer.pac +++ b/src/file_analysis/analyzer/unified2/unified2-analyzer.pac @@ -61,7 +61,7 @@ refine flow Flow += { # return true; # %} - function proc_legacy_ids_event(ev: LegacyIDSEvent) : bool + function proc_ids_event(ev: IDS_Event) : bool %{ if ( ::unified2_event ) { @@ -88,7 +88,7 @@ refine flow Flow += { return true; %} - function proc_ids_event(ev: IDSEvent) : bool + function proc_ids_event_2(ev: IDS_Event_2) : bool %{ if ( ::unified2_event ) { @@ -152,12 +152,12 @@ refine flow Flow += { # proc : bool = $context.flow.proc_record(this); #}; -refine typeattr LegacyIDSEvent += &let { - proc : bool = $context.flow.proc_legacy_ids_event(this); +refine typeattr IDS_Event += &let { + proc : bool = $context.flow.proc_ids_event(this); }; -refine typeattr IDSEvent += &let { - proc : bool = $context.flow.proc_ids_event(this); +refine typeattr IDS_Event_2 += &let { + proc : bool = $context.flow.proc_ids_event_2(this); }; refine typeattr Packet += &let { diff --git a/src/file_analysis/analyzer/unified2/unified2-file.pac b/src/file_analysis/analyzer/unified2/unified2-file.pac index 72f5c959bf..f0d8ca9187 100644 --- a/src/file_analysis/analyzer/unified2/unified2-file.pac +++ b/src/file_analysis/analyzer/unified2/unified2-file.pac @@ -18,16 +18,16 @@ type Record = record { length: uint32; data: case rtype of { PACKET -> packet: Packet(this); - IDS_EVENT -> ids_event: LegacyIDSEvent(this, 1); - IDS_EVENT_IPV6 -> ids_event_ipv6: LegacyIDSEvent(this, 4); - IDS_EVENT_2 -> ids_event_vlan: IDSEvent(this, 1); - IDS_EVENT_IPV6_2 -> ids_event_ipv6_vlan: IDSEvent(this, 4); + IDS_EVENT -> ids_event: IDS_Event(this, 1); + IDS_EVENT_IPV6 -> ids_event_ipv6: IDS_Event(this, 4); + IDS_EVENT_2 -> ids_event_vlan: IDS_Event_2(this, 1); + IDS_EVENT_IPV6_2 -> ids_event_ipv6_vlan: IDS_Event_2(this, 4); #EXTRA_DATA -> extra_data: ExtraData(this); default -> unknown_record_type: UnknownRecordType(this); }; } &byteorder=bigendian &length=length+8; -type LegacyIDSEvent(rec: Record, ip_len: int) = record { +type IDS_Event(rec: Record, ip_len: int) = record { sensor_id: uint32; event_id: uint32; ts: Time; @@ -44,7 +44,7 @@ type LegacyIDSEvent(rec: Record, ip_len: int) = record { packet_action: uint8; } &byteorder=bigendian; -type IDSEvent(rec: Record, ip_len: int) = record { +type IDS_Event_2(rec: Record, ip_len: int) = record { sensor_id: uint32; event_id: uint32; ts: Time; diff --git a/testing/btest/Baseline/scripts.base.files.unified2.alert/unified2.log b/testing/btest/Baseline/scripts.base.files.unified2.alert/unified2.log new file mode 100644 index 0000000000..b86dedf23b --- /dev/null +++ b/testing/btest/Baseline/scripts.base.files.unified2.alert/unified2.log @@ -0,0 +1,11 @@ +#separator \x09 +#set_separator , +#empty_field (empty) +#unset_field - +#path unified2 +#open 2013-08-13-07-16-01 +#fields ts id.src_ip id.src_p id.dst_ip id.dst_p sensor_id signature_id signature generator_id generator signature_revision classification_id classification priority_id event_id packet +#types time addr port addr port count count string count string count count string count count string +1323827323.000000 192.168.1.72 50185 74.125.225.49 80 0 2003058 ET MALWARE 180solutions (Zango) Spyware Installer Download 1 snort general alert 5 21 trojan-activity 1 2 \xd80bH\xc5\xb5x\xca9\xb7\xe4r\x08\x00E\x10\x00\\x1a\xce@\x00@\x062\x1f\xc0\xa8\x01HJ}\xe11\xc4\x09\x00P*\xa8bv]z/\xde\x80\x18\x82+\x88,\x00\x00\x01\x01\x08\x0a\x17J\x83Q\xfe\xad\xac\x1aGET /Zango/ZangoInstaller.exe HTTP/1.0\x0d\x0a +1323827344.000000 192.168.1.72 49862 199.47.216.144 80 0 2012647 ET POLICY Dropbox.com Offsite File Backup in Use 1 snort general alert 3 33 policy-violation 1 3 \xd80bH\xc5\xb5x\xca9\xb7\xe4r\x08\x00E\x00\x00\xf8Q\xdf@\x00@\x06\x86p\xc0\xa8\x01H\xc7/\xd8\x90\xc2\xc6\x00P\x9cm\x97U\xf07\x084\x80\x18\x82\x18%<\x00\x00\x01\x01\x08\x0a\x17J\xd7\xde\x00\x92\x81\xc5GET /subscribe?host_int=43112345&ns_map=123456_1234524412104916591&ts=1323827344 HTTP/1.1\x0d\x0aHost: notify1.dropbox.com\x0d\x0aAccept-Encoding: identity\x0d\x0aConnection: keep-alive\x0d\x0aX-Dropbox-Locale: en_US\x0d\x0a\x0d\x0a +#close 2013-08-13-07-16-01 diff --git a/testing/btest/Files/unified2.u2 b/testing/btest/Files/unified2.u2 new file mode 100755 index 0000000000..2619152e14 Binary files /dev/null and b/testing/btest/Files/unified2.u2 differ diff --git a/testing/btest/btest.cfg b/testing/btest/btest.cfg index 7ccf99eea8..cc06eb4d5d 100644 --- a/testing/btest/btest.cfg +++ b/testing/btest/btest.cfg @@ -14,6 +14,7 @@ LC_ALL=C BTEST_PATH=%(testbase)s/../../aux/btest PATH=%(testbase)s/../../build/src:%(testbase)s/../scripts:%(testbase)s/../../aux/btest:%(testbase)s/../../aux/bro-aux/bro-cut:%(default_path)s TRACES=%(testbase)s/Traces +FILES=%(testbase)s/Files SCRIPTS=%(testbase)s/../scripts DIST=%(testbase)s/../.. BUILD=%(testbase)s/../../build diff --git a/testing/btest/scripts/base/files/unified2/alert.bro b/testing/btest/scripts/base/files/unified2/alert.bro new file mode 100644 index 0000000000..eca1ca036c --- /dev/null +++ b/testing/btest/scripts/base/files/unified2/alert.bro @@ -0,0 +1,76 @@ +# @TEST-EXEC: bro -b %INPUT Unified2::watch_file=$FILES/unified2.u2 +# @TEST-EXEC: btest-diff unified2.log + +@TEST-START-FILE sid_msg.map +2003058 || ET MALWARE 180solutions (Zango) Spyware Installer Download || url,doc.emergingthreats.net/bin/view/Main/2003058 || url,securityresponse.symantec.com/avcenter/venc/data/pf/adware.180search.html +2012647 || ET POLICY Dropbox.com Offsite File Backup in Use || url,dereknewton.com/2011/04/dropbox-authentication-static-host-ids/ || url,www.dropbox.com +@TEST-END-FILE + +@TEST-START-FILE gen_msg.map +1 || 1 || snort general alert +2 || 1 || tag: Tagged Packet +3 || 1 || snort dynamic alert +100 || 1 || spp_portscan: Portscan Detected +100 || 2 || spp_portscan: Portscan Status +100 || 3 || spp_portscan: Portscan Ended +101 || 1 || spp_minfrag: minfrag alert +@TEST-END-FILE + +@TEST-START-FILE classification.config +# +# config classification:shortname,short description,priority +# + +#Traditional classifications. These will be replaced soon + +config classification: not-suspicious,Not Suspicious Traffic,3 +config classification: unknown,Unknown Traffic,3 +config classification: bad-unknown,Potentially Bad Traffic, 2 +config classification: attempted-recon,Attempted Information Leak,2 +config classification: successful-recon-limited,Information Leak,2 +config classification: successful-recon-largescale,Large Scale Information Leak,2 +config classification: attempted-dos,Attempted Denial of Service,2 +config classification: successful-dos,Denial of Service,2 +config classification: attempted-user,Attempted User Privilege Gain,1 +config classification: unsuccessful-user,Unsuccessful User Privilege Gain,1 +config classification: successful-user,Successful User Privilege Gain,1 +config classification: attempted-admin,Attempted Administrator Privilege Gain,1 +config classification: successful-admin,Successful Administrator Privilege Gain,1 +config classification: rpc-portmap-decode,Decode of an RPC Query,2 +config classification: shellcode-detect,Executable Code was Detected,1 +config classification: string-detect,A Suspicious String was Detected,3 +config classification: suspicious-filename-detect,A Suspicious Filename was Detected,2 +config classification: suspicious-login,An Attempted Login Using a Suspicious Username was Detected,2 +config classification: system-call-detect,A System Call was Detected,2 +config classification: tcp-connection,A TCP Connection was Detected,4 +config classification: trojan-activity,A Network Trojan was Detected, 1 +config classification: unusual-client-port-connection,A Client was Using an Unusual Port,2 +config classification: network-scan,Detection of a Network Scan,3 +config classification: denial-of-service,Detection of a Denial of Service Attack,2 +config classification: non-standard-protocol,Detection of a Non-Standard Protocol or Event,2 +config classification: protocol-command-decode,Generic Protocol Command Decode,3 +config classification: web-application-activity,Access to a Potentially Vulnerable Web Application,2 +config classification: web-application-attack,Web Application Attack,1 +config classification: misc-activity,Misc activity,3 +config classification: misc-attack,Misc Attack,2 +config classification: icmp-event,Generic ICMP event,3 +config classification: inappropriate-content,Inappropriate Content was Detected,1 +config classification: policy-violation,Potential Corporate Privacy Violation,1 +config classification: default-login-attempt,Attempt to Login By a Default Username and Password,2 +@TEST-END-FILE + +redef exit_only_after_terminate = T; + +@load base/files/unified2 + +redef Unified2::sid_msg = @DIR+"/sid_msg.map"; +redef Unified2::gen_msg = @DIR+"/gen_msg.map"; +redef Unified2::classification_config = @DIR+"/classification.config"; +global i = 0; + +event Unified2::alert(f: fa_file, ev: Unified2::IDSEvent, pkt: Unified2::Packet) + { + ++i; + if ( i == 2 ) + terminate(); + } \ No newline at end of file