From f5668e41a2cf2fb6daa9c04df0911c23608fd8f8 Mon Sep 17 00:00:00 2001 From: Seth Hall Date: Thu, 17 Mar 2011 14:29:07 -0400 Subject: [PATCH] Updates to make scripts work with logging-internals code. --- policy/conn.bro | 8 ++- policy/ftp.bro | 84 ++++++++++++++------------- policy/http.bro | 24 ++++---- policy/known-hosts.bro | 16 +++--- policy/known-services.bro | 56 ++++++++++++------ policy/notice.bro | 12 +++- policy/software.bro | 118 +++++++++++++++++++++++++------------- policy/ssh.bro | 16 +++--- 8 files changed, 203 insertions(+), 131 deletions(-) diff --git a/policy/conn.bro b/policy/conn.bro index 34bbcab65a..5e8688f614 100644 --- a/policy/conn.bro +++ b/policy/conn.bro @@ -36,12 +36,14 @@ export { # This is where users can get access to the active Log record for a # connection so they can extend and enhance the logged data. global active_conns: table[conn_id] of Log; + + global log_conn: event(rec: Log); } event bro_init() { - Log::create_stream("CONN", "Conn::Log"); - Log::add_default_filter("CONN"); + Log::create_stream(CONN, [$columns=Conn::Log, $ev=log_conn]); + Log::add_default_filter(CONN); } function conn_state(c: connection, trans: transport_proto): string @@ -143,7 +145,7 @@ event connection_established(c: connection) &priority = 10 event connection_state_remove(c: connection) &priority = -10 { local conn_log = get_conn_log(c); - Log::write("CONN", conn_log); + Log::write(CONN, conn_log); if ( c$id in active_conns ) delete active_conns[c$id]; diff --git a/policy/ftp.bro b/policy/ftp.bro index 4191b4af24..55e1c17dea 100644 --- a/policy/ftp.bro +++ b/policy/ftp.bro @@ -23,8 +23,15 @@ redef enum Notice::Type += { FTP_Site_Exec_Success, }; +redef enum Log::ID += { FTP }; + +# Configure DPD +const ports = { 21/tcp } &redef; +redef capture_filters += { ["ftp"] = "port 21" }; +redef dpd_config += { [ANALYZER_FTP] = [$ports = ports] }; + + export { - redef enum Log::ID += { FTP }; type LogTags: enum { UNKNOWN }; @@ -47,7 +54,7 @@ export { type SessionInfo: record { log: Log; ## By setting the CWD to '/.', we can indicate that unless something - ## more concrete is discovered that the exiting but unknown + ## more concrete is discovered that the existing but unknown ## directory is ok to use. cwd: string &default="/."; command: CmdArg &optional; @@ -82,19 +89,15 @@ export { ## This tracks all of the currently established FTP control sessions. global active_conns: table[conn_id] of SessionInfo &read_expire=5mins; + global log_ftp: event(rec: Log); } global ftp_data_expected: table[addr, port] of ExpectedConn &create_expire=5mins; -# Configure DPD -const ports = { 21/tcp } &redef; -redef capture_filters += { ["ftp"] = "port 21" }; -redef dpd_config += { [ANALYZER_FTP] = [$ports = ports] }; - event bro_init() { - Log::create_stream("FTP", "FTP::Log"); - Log::add_default_filter("FTP"); + Log::create_stream(FTP, [$columns=FTP::Log, $ev=log_ftp]); + Log::add_default_filter(FTP); } # A set of commands where the argument can be expected to refer @@ -128,7 +131,7 @@ function parse_ftp_reply_code(code: count): ReplyCode return a; } -function new_ftp_session(c: connection) +function get_ftp_session(c: connection): SessionInfo { local id = c$id; @@ -141,13 +144,14 @@ function new_ftp_session(c: connection) add_pending_cmd(info$pending_commands, "", ""); active_conns[id] = info; + return info; } function ftp_message(s: SessionInfo) { # If it either has a tag associated with it (something detected) # or it's a deliberately logged command. - if ( |s$log$tags| > 0 || s$command$cmd in logged_commands ) + if ( |s$log$tags| > 0 || (s?$command && s$command$cmd in logged_commands) ) { local pass = "\\N"; if ( to_lower(s$log$user) in guest_ids && s$log?$password ) @@ -162,7 +166,7 @@ function ftp_message(s: SessionInfo) s$log$arg=arg; # TODO: does the framework do this atomicly or do I need the copy? - Log::write("FTP", copy(s$log)); + Log::write(FTP, copy(s$log)); } # The MIME and file_size fields are specific to file transfer commands @@ -184,9 +188,7 @@ event ftp_request(c: connection, command: string, arg: string) &priority=1 #if ( is_string_binary(command) ) return; local id = c$id; - if ( id !in active_conns ) - new_ftp_session(c); - local session = active_conns[id]; + local session = get_ftp_session(c); # Log the previous command when a new command is seen. # The downside here is that commands definitely aren't logged until the @@ -233,9 +235,7 @@ event ftp_reply(c: connection, code: count, msg: string, cont_resp: bool) &prior if ( cont_resp ) return; local id = c$id; - if ( id !in active_conns ) - new_ftp_session(c); - local session = active_conns[id]; + local session = get_ftp_session(c); session$command = get_pending_cmd(session$pending_commands, code, msg); @@ -247,6 +247,8 @@ event ftp_reply(c: connection, code: count, msg: string, cont_resp: bool) &prior #if ( response_xyz$x == 2 && # successful # session$command$cmd == "PASS" ) # do_ftp_login(c, session); + if ( session$command$cmd == "PASS" ) + print fmt("Woo: %s %s", session$log$user, session$log$password); if ( code == 150 && session$command$cmd == "RETR" ) { @@ -333,27 +335,27 @@ event connection_state_remove(c: connection) &priority=1 delete active_conns[id]; } -event expected_connection_seen(c: connection, a: count) &priority=1 - { - local id = c$id; - if ( [id$resp_h, id$resp_p] in ftp_data_expected ) - add c$service["ftp-data"]; - } +#event expected_connection_seen(c: connection, a: count) &priority=1 +# { +# local id = c$id; +# if ( [id$resp_h, id$resp_p] in ftp_data_expected ) +# add c$service["ftp-data"]; +# } -event file_transferred(c: connection, prefix: string, descr: string, - mime_type: string) &priority=1 - { - local id = c$id; - if ( [id$resp_h, id$resp_p] in ftp_data_expected ) - { - local expected = ftp_data_expected[id$resp_h, id$resp_p]; - local s = expected$session; - s$log$mime_type = mime_type; - s$log$mime_desc = descr; - - # TODO: not sure if it's ok to delete this here, but it should - # always be called since the file analyzer is always attached - # to ftp-data sessions. - delete ftp_data_expected[id$resp_h, id$resp_p]; - } - } +#event file_transferred(c: connection, prefix: string, descr: string, +# mime_type: string) &priority=1 +# { +# local id = c$id; +# if ( [id$resp_h, id$resp_p] in ftp_data_expected ) +# { +# local expected = ftp_data_expected[id$resp_h, id$resp_p]; +# local s = expected$session; +# s$log$mime_type = mime_type; +# s$log$mime_desc = descr; +# +# # TODO: not sure if it's ok to delete this here, but it should +# # always be called since the file analyzer is always attached +# # to ftp-data sessions. +# delete ftp_data_expected[id$resp_h, id$resp_p]; +# } +# } diff --git a/policy/http.bro b/policy/http.bro index 8d3940c999..e4775ca984 100644 --- a/policy/http.bro +++ b/policy/http.bro @@ -8,13 +8,14 @@ module HTTP; -redef enum Log::ID += { HTTP }; redef enum Software::Type += { WEB_SERVER, WEB_BROWSER, WEB_BROWSER_PLUGIN, }; +redef enum Log::ID += { HTTP }; + export { type LogTags: enum { ## Indicator of a URI based SQL injection attack. @@ -113,12 +114,13 @@ export { ## List of all active HTTP session indexed by conn_id. global active_conns: table[conn_id] of SessionInfo &read_expire=5mins; + global log_http: event(rec: Log); } event bro_init() { - Log::create_stream("HTTP", "HTTP::Log"); - Log::add_default_filter("HTTP"); + Log::create_stream(HTTP, [$columns=HTTP::Log, $ev=log_http]); + Log::add_default_filter(HTTP); } # DPD configuration. @@ -137,11 +139,7 @@ function new_http_log(id: conn_id): Log { local tags: set[LogTags] = set(); local proxied: set[string] = set(); - local log: Log = [$ts=network_time(), $id=id, $tags=tags, $proxied=proxied, - # TODO: some bug with record default initialization - $user_agent="", $request_body_size=0, $response_body_size=0, $status_code=0, $status_msg="", $username="", $password="", $referrer="", $host="" - ]; - return log; + return [$ts=network_time(), $id=id, $tags=tags, $proxied=proxied]; } function get_http_session(id: conn_id): SessionInfo @@ -250,7 +248,7 @@ event http_header(c: connection, is_orig: bool, name: string, value: string) &pr { if ( name == "SERVER" ) { - local si = Software::default_parse(value, c$id$resp_h, WEB_SERVER); + local si = Software::parse(value, c$id$resp_h, WEB_SERVER); Software::found(c, si); } else if ( name == "CONTENT-LENGTH" ) @@ -264,10 +262,10 @@ event http_begin_entity(c: connection, is_orig: bool) &priority=1 if ( is_orig ) if ( sess$log_point == AFTER_REQUEST ) - Log::write("HTTP", sess$log); + Log::write(HTTP, sess$log); else if ( sess$log_point == AFTER_REQUEST ) - Log::write("HTTP", sess$log); + Log::write(HTTP, sess$log); } event http_message_done(c: connection, is_orig: bool, stat: http_message_stat) &priority=1 @@ -276,10 +274,10 @@ event http_message_done(c: connection, is_orig: bool, stat: http_message_stat) & if ( is_orig ) if ( sess$log_point == AFTER_REQUEST_BODY ) - Log::write("HTTP", sess$log); + Log::write(HTTP, sess$log); else if ( sess$log_point == AFTER_REPLY_BODY ) - Log::write("HTTP", sess$log); + Log::write(HTTP, sess$log); } diff --git a/policy/known-hosts.bro b/policy/known-hosts.bro index 804f30536e..d75a5b8f5b 100644 --- a/policy/known-hosts.bro +++ b/policy/known-hosts.bro @@ -2,9 +2,9 @@ module KnownHosts; +redef enum Log::ID += { KNOWN_HOSTS }; + export { - redef enum Log::ID += { KNOWN_HOSTS }; - type Log: record { ts: time; address: addr; @@ -20,13 +20,15 @@ export { # Maintain the list of known hosts for 24 hours so that the existence # of each individual address is logged each day. - global known_hosts: set[addr] &create_expire=1day; + global known_hosts: set[addr] &create_expire=1day &syncronized; + + global log_known_hosts: event(rec: Log); } event bro_init() { - Log::create_stream("KNOWN_HOSTS", "KnownHosts::Log"); - Log::add_default_filter("KNOWN_HOSTS"); + Log::create_stream(KNOWN_HOSTS, [$columns=KnownHosts::Log, $ev=log_known_hosts]); + Log::add_default_filter(KNOWN_HOSTS); } event connection_established(c: connection) @@ -36,11 +38,11 @@ event connection_established(c: connection) if ( id$orig_h !in known_hosts && addr_matches_hosts(id$orig_h, logging) ) { add known_hosts[id$orig_h]; - Log::write("KNOWN_HOSTS", [$ts=network_time(), $address=id$orig_h]); + Log::write(KNOWN_HOSTS, [$ts=network_time(), $address=id$orig_h]); } if ( id$resp_h !in known_hosts && addr_matches_hosts(id$resp_h, logging) ) { add known_hosts[id$resp_h]; - Log::write("KNOWN_HOSTS", [$ts=network_time(), $address=id$resp_h]); + Log::write(KNOWN_HOSTS, [$ts=network_time(), $address=id$resp_h]); } } diff --git a/policy/known-services.bro b/policy/known-services.bro index ec7108a08e..53f327a335 100644 --- a/policy/known-services.bro +++ b/policy/known-services.bro @@ -1,33 +1,43 @@ -#@load global-ext @load functions module KnownServices; +redef enum Log::ID += { KNOWN_SERVICES }; + export { - redef enum Log::ID += { KNOWN_SERVICES }; + type LogPoints: enum { + AFTER_PROTOCOL_DETECTION, + AT_CONNECTION_END, + }; + type Log: record { ts: time; host: addr; port_num: port; -# port_num: count; # split 'em? -# port_proto: string; - service: string &default=""; + port_proto: transport_proto; + service: set[string]; + log_point: LogPoints; }; - + + # The hosts whose services should be logged. const logged_hosts = LocalHosts &redef; + + const default_log_point = AFTER_PROTOCOL_DETECTION &redef; global known_services: set[addr, port] &create_expire=1day &synchronized; + + global log_known_services: event(rec: Log); } # The temporary holding place for new, unknown services. -global established_conns: set[addr, port] &create_expire=1day &redef; - +global established_conns: table[addr, port] of Log &create_expire=1day &redef; event bro_init() { - Log::create_stream("KNOWN_SERVICES", "KnownServices::Log"); - Log::add_default_filter("KNOWN_SERVICES"); + Log::create_stream(KNOWN_SERVICES, [$columns=KnownServices::Log, + $ev=log_known_services]); + Log::add_default_filter(KNOWN_SERVICES); } event connection_established(c: connection) @@ -38,7 +48,7 @@ event connection_established(c: connection) add established_conns[id$resp_h, id$resp_p]; } -event known_services_done(c: connection) +function known_services_done(c: connection) { local id = c$id; if ( [id$resp_h, id$resp_p] !in known_services && @@ -46,20 +56,30 @@ event known_services_done(c: connection) "ftp-data" !in c$service ) # don't include ftp data sessions { add known_services[id$resp_h, id$resp_p]; - Log::write( "KNOWN_SERVICES", [ $ts=c$start_time, $host=id$resp_h, - $port_num=id$resp_p, $service=c$service] ); + Log::write(KNOWN_SERVICES, [$ts=c$start_time, + $host=id$resp_h, + $port_num=id$resp_p, + $port_proto=get_port_transport_proto(id$resp_p), + $service=c$service] ); } } -event connection_state_remove(c: connection) +event connection_established(c: connection) { - event known_services_done(c); + } - - + +# Log the event after protocol detection if event protocol_confirmation(c: connection, atype: count, aid: count) &priority=-10 { - event known_services_done(c); + local l = established_conns[c$id$resp, c$id$resp_p]; + if ( l$log_point == AFTER_PROTOCOL_DETECTION ) + known_services_done(c); } +# Handle the connection ending in case no protocol was ever detected. +event connection_state_remove(c: connection) + { + known_services_done(c); + } diff --git a/policy/notice.bro b/policy/notice.bro index 8adfe38cc5..3bb7925b40 100644 --- a/policy/notice.bro +++ b/policy/notice.bro @@ -2,6 +2,10 @@ module Notice; +## This couldn't be named NOTICE because that name is already used by the +## global function NOTICE(). +redef enum Log::ID += { NOTICE_LOG }; + export { type Type: enum { NoticeNone, # placeholder @@ -96,6 +100,8 @@ export { # These are implemented below global email_notice_to: function(n: Notice::Info, dest: string) &redef; global notice: function(n: Notice::Info); + + global log_notice: event(rec: Info); } # Each notice has a unique ID associated with it. @@ -105,8 +111,8 @@ redef new_notice_tag = function(): string event bro_init() { - Log::create_stream("NOTICE", "Notice::Info"); - Log::add_default_filter("NOTICE"); + Log::create_stream(NOTICE_LOG, [$columns=Notice::Info, $ev=log_notice]); + Log::add_default_filter(NOTICE_LOG); } function add_notice_tag(c: connection): string @@ -255,7 +261,7 @@ function NOTICE(n: Notice::Info) { # Build the info here after we had a chance to set the # $dropped field. - Log::write("NOTICE", n); + Log::write(NOTICE_LOG, n); if ( action != NOTICE_FILE && n$do_alarm ) { diff --git a/policy/software.bro b/policy/software.bro index 4f01026dc0..97507b4e68 100644 --- a/policy/software.bro +++ b/policy/software.bro @@ -17,12 +17,14 @@ redef enum Notice::Type += { Software_Version_Change, }; +redef enum Log::ID += { SOFTWARE }; + export { type Version: record { - major: count &default=0; ##< Major version number - minor: count &default=0; ##< Minor version number - minor2: count &default=0; ##< Minor subversion number - addl: string &default=""; ##< Additional version string (e.g. "beta42") + major: count &optional; ##< Major version number + minor: count &optional; ##< Minor version number + minor2: count &optional; ##< Minor subversion number + addl: string &optional; ##< Additional version string (e.g. "beta42") }; type Type: enum { @@ -38,21 +40,20 @@ export { PRINTER, }; - redef enum Log::ID += { SOFTWARE }; type Info: record { ## The time at which the software was first detected. ts: time; ## The IP address detected running the software. - host: addr &default=0.0.0.0; + host: addr; ## The type of software detected (e.g. WEB_SERVER) software_type: Type &default=UNKNOWN; ## Name of the software (e.g. Apache) - name: string &default=""; + name: string; ## Version of the software version: Version; ## The full unparsed version string found because the version parsing ## doesn't work 100% reliably and this acts as a fall back in the logs. - unparsed_version: string &default=""; + unparsed_version: string &optional; }; ## The hosts whose software should be logged. @@ -79,7 +80,7 @@ export { ## This function can take many software version strings and parse them into ## a sensible Software::Version record. There are still many cases where ## scripts may have to have their own specific version parsing though. - global default_parse: function(unparsed_version: string, + global parse: function(unparsed_version: string, host: addr, software_type: Type): Info; @@ -93,19 +94,21 @@ export { type SoftwareSet: table[string] of Info; # The set of software associated with an address. global tracked_software: table[addr] of SoftwareSet &create_expire=1day &synchronized; + + global log_software: event(rec: Info); } event bro_init() { - Log::create_stream("SOFTWARE", "Software::Info"); - Log::add_default_filter("SOFTWARE"); + Log::create_stream(SOFTWARE, [$columns=Software::Info, $ev=log_software]); + Log::add_default_filter(SOFTWARE); } # Don't even try to understand this now, just make sure the tests are # working. -function default_parse(unparsed_version: string, - host: addr, - software_type: Type): Info +function parse(unparsed_version: string, + host: addr, + software_type: Type): Info { local software_name = ""; local v: Version; @@ -113,30 +116,29 @@ function default_parse(unparsed_version: string, # The regular expression should match the complete version number # and software name. local version_parts = split_n(unparsed_version, /[0-9\/\-\._ ]{2,}/, T, 1); + if ( 1 in version_parts ) + software_name = version_parts[1]; if ( |version_parts| >= 2 ) { - software_name = version_parts[1]; # Remove the name/version separator because it's left at the begining # of the version number from the previous split_all. local sv = version_parts[2]; if ( /^[\/\-\._ ]/ in sv ) sv = sub(version_parts[2], /^[\/\-\._ ]/, ""); local version_numbers = split_n(sv, /[\-\._,\[\(\{ ]/, F, 4); - local addl = ""; if ( 4 in version_numbers && version_numbers[4] != "" ) - addl = version_numbers[4]; + v$addl = version_numbers[4]; else if ( 3 in version_parts && version_parts[3] != "" ) { # TODO: there's a bug with do_split! local vp = split_n(version_parts[3], /[\-\._,\[\]\(\)\{\} ]/, F, 2); if ( |vp| >= 1 && vp[1] != "" ) - addl = vp[1]; - else if ( |vp| >= 2 ) - addl = vp[2]; + v$addl = vp[1]; + else if ( |vp| >= 2 && vp[2] != "" ) + v$addl = vp[2]; else - addl = version_parts[3]; + v$addl = version_parts[3]; } - v$addl = addl; if ( |version_numbers| >= 3 ) v$minor2 = to_count(version_numbers[3]); @@ -153,22 +155,60 @@ function default_parse(unparsed_version: string, function cmp_versions(v1: Version, v2: Version): int { - if ( v1$major < v2$major ) - return -1; - if ( v1$major > v2$major ) - return 1; - - if ( v1$minor < v2$minor ) - return -1; - if ( v1$minor > v2$minor ) - return 1; - - if ( v1$minor2 < v2$minor2 ) - return -1; - if ( v1$minor2 > v2$minor2 ) - return 1; - - return strcmp(v1$addl, v2$addl); + if ( v1?$major && v2?$major ) + { + if ( v1$major < v2$major ) + return -1; + if ( v1$major > v2$major ) + return 1; + } + else + { + if ( !v1?$major && !v2?$major ) + return 0; + else + return -1; + } + + if ( v1?$minor && v2?$minor ) + { + if ( v1$minor < v2$minor ) + return -1; + if ( v1$minor > v2$minor ) + return 1; + } + else + { + if ( !v1?$minor && !v2?$minor ) + return 0; + else + return -1; + } + + if ( v1?$minor2 && v2?$minor2 ) + { + if ( v1$minor2 < v2$minor2 ) + return -1; + if ( v1$minor2 > v2$minor2 ) + return 1; + } + else + { + if ( !v1?$minor2 && !v2?$minor2 ) + { return 0; } + else + {print "super WTF!"; return -1; } + } + + if ( v1?$addl && v2?$addl ) + return strcmp(v1$addl, v2$addl); + else + { + if ( !v1?$minor2 && !v2?$minor2 ) + return 0; + else + return -1; + } } function software_endpoint_name(c: connection, host: addr): string @@ -225,7 +265,7 @@ event software_register(c: connection, info: Info) } } - Log::write("SOFTWARE", info); + Log::write(SOFTWARE, info); ts[info$name] = info; } diff --git a/policy/ssh.bro b/policy/ssh.bro index 9958a3e717..3760708256 100644 --- a/policy/ssh.bro +++ b/policy/ssh.bro @@ -12,6 +12,8 @@ redef enum Notice::Type += { SSH_Bytecount_Inconsistency, }; +redef enum Log::ID += { SSH }; + redef enum Software::Type += { SSH_SERVER, SSH_CLIENT, @@ -22,8 +24,6 @@ redef capture_filters += { ["ssh"] = "tcp port 22" }; redef dpd_config += { [ANALYZER_SSH] = [$ports = set(22/tcp)] }; export { - # Create a new ID for our log stream - redef enum Log::ID += { SSH }; type Log: record { ts: time; id: conn_id; @@ -86,6 +86,8 @@ export { # The list of active SSH connections and the associated session info. global active_conns: table[conn_id] of Log &read_expire=2mins; + + global log_ssh: event(rec: Log); } function local_filter(rec: record { id: conn_id; } ): bool @@ -98,9 +100,9 @@ event bro_init() # Create the stream. # First argument is the ID for the stream. # Second argument is the log record type. - Log::create_stream("SSH", "SSH::Log"); + Log::create_stream(SSH, [$columns=SSH::Log, $ev=log_ssh]); # Add a default filter that simply logs everything to "ssh.log" using the default writer. - Log::add_default_filter("SSH"); + Log::add_default_filter(SSH); } event check_ssh_connection(c: connection, done: bool) @@ -204,7 +206,7 @@ event check_ssh_connection(c: connection, done: bool) ssh_log$direction = direction; ssh_log$resp_size = c$resp$size; - Log::write("SSH", ssh_log); + Log::write(SSH, ssh_log); delete active_conns[c$id]; # Stop watching this connection, we don't care about it anymore. @@ -247,7 +249,7 @@ event ssh_client_version(c: connection, version: string) # Get rid of the protocol information when passing to the software framework. local cleaned_version = sub(version, /^SSH[0-9\.\-]+/, ""); - local si = Software::default_parse(cleaned_version, c$id$orig_h, SSH_CLIENT); + local si = Software::parse(cleaned_version, c$id$orig_h, SSH_CLIENT); Software::found(c, si); } @@ -258,7 +260,7 @@ event ssh_server_version(c: connection, version: string) # Get rid of the protocol information when passing to the software framework. local cleaned_version = sub(version, /SSH[0-9\.\-]{2,}/, ""); - local si = Software::default_parse(cleaned_version, c$id$resp_h, SSH_SERVER); + local si = Software::parse(cleaned_version, c$id$resp_h, SSH_SERVER); Software::found(c, si); }