diff --git a/aux/binpac b/aux/binpac index 106cd9782f..c3c7ef0dfd 160000 --- a/aux/binpac +++ b/aux/binpac @@ -1 +1 @@ -Subproject commit 106cd9782f1f9443743b49adccae26f0ee72621c +Subproject commit c3c7ef0dfddb0746d3762e41086ba42928e68483 diff --git a/aux/bro-aux b/aux/bro-aux index 764b616792..98f92eeb40 160000 --- a/aux/bro-aux +++ b/aux/bro-aux @@ -1 +1 @@ -Subproject commit 764b6167922662051b62d269ec0fbd14e2ce0c02 +Subproject commit 98f92eeb40281045159097764abddc428fb49bf2 diff --git a/aux/broccoli b/aux/broccoli index d693ba5f14..48d473398e 160000 --- a/aux/broccoli +++ b/aux/broccoli @@ -1 +1 @@ -Subproject commit d693ba5f14c33a97e4f149e49196281cc075d289 +Subproject commit 48d473398e577893b6c7f77d605ccdf266a2f93b diff --git a/aux/broctl b/aux/broctl index b26dc6dc6a..532dcd5aa5 160000 --- a/aux/broctl +++ b/aux/broctl @@ -1 +1 @@ -Subproject commit b26dc6dc6a9080abe7a062d0882e813acbb63248 +Subproject commit 532dcd5aa51c8b29b2d71cd37e1d7c21e33cc715 diff --git a/aux/btest b/aux/btest index 409bda3a00..a2b04952ae 160000 --- a/aux/btest +++ b/aux/btest @@ -1 +1 @@ -Subproject commit 409bda3a003b18c4736ef168595f20118f4d0038 +Subproject commit a2b04952ae91dcd27d5e68a42d5d26c291ecb1f5 diff --git a/policy/all.bro b/policy/all.bro index 4bbe3e8afe..637e0a3391 100644 --- a/policy/all.bro +++ b/policy/all.bro @@ -53,10 +53,8 @@ @load http-identified-files.bro @load http-reply @load http-request -@load http-rewriter @load http @load icmp -@load ident-rewriter @load ident @load inactivity @load interconn @@ -111,7 +109,6 @@ @load site @load smb @load smtp-relay -@load smtp-rewriter @load smtp @load snort @load software diff --git a/policy/bro.init b/policy/bro.init index e25038e1e7..ba180e5045 100644 --- a/policy/bro.init +++ b/policy/bro.init @@ -1117,14 +1117,6 @@ type bt_tracker_headers: table[string] of string; @load event.bif.bro -@load common-rw.bif.bro -@load finger-rw.bif.bro -@load ftp-rw.bif.bro -@load ident-rw.bif.bro -@load smtp-rw.bif.bro -@load http-rw.bif.bro -@load dns-rw.bif.bro - function subst(s: string, from: pattern, to: string): string { local p = split_all(s, from); @@ -1404,34 +1396,3 @@ const skip_http_data = F &redef; # Whether the analysis engine parses IP packets encapsulated in # UDP tunnels. See also: udp_tunnel_port, policy/udp-tunnel.bro. const parse_udp_tunnels = F &redef; - -# Whether a commitment is required before writing the transformed -# trace for a connection into the dump file. -const requires_trace_commitment = F &redef; - -# Whether IP address anonymization is enabled. -const anonymize_ip_addr = F &redef; - -# Whether to omit place holder packets when rewriting. -const omit_rewrite_place_holder = T &redef; - -# Whether trace of various protocols is being rewritten. -const rewriting_http_trace = F &redef; -const rewriting_smtp_trace = F &redef; -const rewriting_ftp_trace = F &redef; -const rewriting_ident_trace = F &redef; -const rewriting_finger_trace = F &redef; -const rewriting_dns_trace = F &redef; -const rewriting_smb_trace = F &redef; - -# Whether we dump selected original packets to the output trace. -const dump_selected_source_packets = F &redef; - -# If true, we dump original packets to the output trace *if and only if* -# the connection is not rewritten; if false, the policy script can decide -# whether to dump a particular connection by calling dump_packets_of_connection. -# -# NOTE: DO NOT SET THIS TO TRUE WHEN ANONYMIZING A TRACE! -# (TODO: this variable should be disabled when using '-A' option) -const dump_original_packets_if_not_rewriting = F &redef; - diff --git a/policy/dns-anonymizer.bro b/policy/dns-anonymizer.bro deleted file mode 100644 index d85f31a395..0000000000 --- a/policy/dns-anonymizer.bro +++ /dev/null @@ -1,107 +0,0 @@ -# $Id:$ - -@load dns -@load anon - -module DNS; - -redef rewriting_dns_trace = T; - -event dns_message(c: connection, is_orig: bool, msg: dns_msg, len: count) - { - if ( get_conn_transport_proto(c$id) == udp ) - rewrite_dns_message(c, is_orig, msg, len); - } - -event dns_request(c: connection, msg: dns_msg, query: string, - qtype: count, qclass: count) - { - if ( get_conn_transport_proto(c$id) == udp ) - rewrite_dns_request(c, anonymize_host(query), - msg, qtype, qclass); - } - -event dns_end(c: connection, msg: dns_msg) - { - if ( get_conn_transport_proto(c$id) == udp ) - rewrite_dns_end(c, T); - } - -event dns_query_reply(c: connection, msg: dns_msg, query: string, - qtype: count, qclass: count) - { - rewrite_dns_reply_question(c, msg, anonymize_host(query), - qtype, qclass); - } - -event dns_A_reply(c: connection, msg: dns_msg, ans: dns_answer, a: addr) - { - ans$query = anonymize_host(ans$query); - rewrite_dns_A_reply(c, msg, ans, anonymize_address(a, c$id)); - } - -#### FIXME: ANONYMIZE! -event dns_AAAA_reply(c: connection, msg: dns_msg, ans: dns_answer, - a: addr, astr: string) - { - ans$query = anonymize_host(ans$query); - astr = "::"; - a = anonymize_address(a, c$id); - rewrite_dns_AAAA_reply(c, msg, ans, a, astr); - } - -event dns_NS_reply(c: connection, msg: dns_msg, ans: dns_answer, name: string) - { - ans$query = anonymize_host(ans$query); - rewrite_dns_NS_reply(c, msg, ans, anonymize_host(name)); - } - -event dns_CNAME_reply(c: connection, msg: dns_msg, ans: dns_answer, - name: string) - { - ans$query = anonymize_host(ans$query); - rewrite_dns_CNAME_reply(c, msg, ans, anonymize_host(name)); - } - -event dns_MX_reply(c: connection, msg: dns_msg, ans: dns_answer, - name: string, preference: count) - { - ans$query = anonymize_host(ans$query); - rewrite_dns_MX_reply(c, msg, ans, anonymize_host(name), preference); - } - -event dns_PTR_reply(c: connection, msg: dns_msg, ans: dns_answer, name: string) - { - ans$query = anonymize_host(ans$query); - rewrite_dns_PTR_reply(c, msg, ans, anonymize_host(name)); - } - -event dns_SOA_reply(c: connection, msg: dns_msg, ans: dns_answer, soa: dns_soa) - { - soa$mname = anonymize_host(soa$mname); - soa$rname = anonymize_host(soa$rname); - ans$query = anonymize_host(ans$query); - rewrite_dns_SOA_reply(c, msg, ans, soa); - } - -event dns_TXT_reply(c: connection, msg: dns_msg, ans: dns_answer, - str: string) - { - str = anonymize_string(str); - ans$query = anonymize_host(ans$query); - rewrite_dns_TXT_reply(c, msg, ans, str); - } - -event dns_EDNS_addl (c: connection, msg: dns_msg, ans: dns_edns_additional) - { - rewrite_dns_EDNS_addl(c, msg, ans); - } - -event dns_rejected(c: connection, msg: dns_msg, query: string, - qtype: count, qclass: count) - { - #### Hmmm, this is probably not right - we are going to have to look - # at the question type to determine how to anonymize this. - rewrite_dns_reply_question(c, msg, anonymize_host(query), - qtype, qclass); - } diff --git a/policy/finger.bro b/policy/finger.bro index 8cf1dbba65..7765ce45c6 100644 --- a/policy/finger.bro +++ b/policy/finger.bro @@ -60,19 +60,6 @@ event finger_request(c: connection, full: bool, username: string, hostname: stri req = fmt("(%s)", req); append_addl_marker(c, req, " *"); - - if ( rewriting_finger_trace ) - rewrite_finger_request(c, full, - public_user(username) ? username : "private user", - hostname); - } - -event finger_reply(c: connection, reply_line: string) - { - local id = c$id; - if ( rewriting_finger_trace ) - rewrite_finger_reply(c, - authorized_client(id$orig_h) ? "finger reply ..." : reply_line); } function is_finger_conn(c: connection): bool @@ -80,10 +67,3 @@ function is_finger_conn(c: connection): bool return c$id$resp_p == finger; } -event connection_state_remove(c: connection) - { - if ( rewriting_finger_trace && requires_trace_commitment && - is_finger_conn(c) ) - # Commit queued packets and all packets in future. - rewrite_commit_trace(c, T, T); - } diff --git a/policy/ftp-anonymizer.bro b/policy/ftp-anonymizer.bro deleted file mode 100644 index 560b85119b..0000000000 --- a/policy/ftp-anonymizer.bro +++ /dev/null @@ -1,846 +0,0 @@ -# $Id: ftp-anonymizer.bro 47 2004-06-11 07:26:32Z vern $ - -@load ftp -@load anon - -# Definitions of constants. - -# Check if those commands carry any argument; anonymize non-empty -# argument. -const ftp_cmds_with_no_arg = { - "CDUP", "QUIT", "REIN", "PASV", "STOU", - "ABOR", "PWD", "SYST", "NOOP", - - "FEAT", "XPWD", -}; - -const ftp_cmds_with_file_arg = { - "APPE", "CWD", "DELE", "LIST", "MKD", - "NLST", "RMD", "RNFR", "RNTO", "RETR", - "STAT", "STOR", "SMNT", - # FTP extensions - "SIZE", "MDTM", - "MLSD", "MLST", - "XCWD", -}; - -# For following commands, we check if the argument conforms to the -# specification -- if so, it is safe to be left in the clear. -const ftp_cmds_with_safe_arg = { - "TYPE", "STRU", "MODE", "ALLO", "REST", - "HELP", - - "MACB", # MacBinary encoding -}; - -# ftp_other_cmds can be redefined in site/trace-specific ways. -const ftp_other_cmds = { - "LPRT", "OPTS", "CLNT", "RETP", - "EPSV", "XPWD", - "SOCK", # old FTP command (RFC 354) -} &redef; - -# Below defines patterns of arguments of FTP commands - -# The following patterns are case-insensitive -const ftp_safe_cmd_arg_pattern = - /TYPE (([AE]( [NTC])?)|I|(L [0-9]+))/ - | /STRU [FRP]/ - | /MODE [SBC]/ - | /ALLO [0-9]+([ \t]+R[ \t]+[0-9]+)?/ - | /REST [!-~]+/ - | /MACB (E|DISABLE|ENABLE)/ - | /SITE TRUTH ON/ - &redef; - -# The following list includes privacy-safe [cmd, arg] pairs and can be -# customized for particular traces -const ftp_safe_arg_list: set[string, string] = { -} &redef; - -# ftp_special_cmd_args offers an even more flexible way of customizing -# argument anonymization: for each [cmd, arg] pair in the table, the -# corresponding value will be the anonymized argument. -const ftp_special_cmd_args: table[string, string] of string = { -} &redef; - -# The following words are safe to be left in the clear as the argument -# of a HELP command. -const ftp_help_words = { - "USER", "PORT", "STOR", "MSAM", "RNTO", "NLST", "MKD", "CDUP", - "PASS", "PASV", "APPE", "MRSQ", "ABOR", "SITE", "XMKD", "XCUP", - "ACCT", "TYPE", "MLFL", "MRCP", "DELE", "SYST", "RMD", "STOU", - "SMNT", "STRU", "MAIL", "ALLO", "CWD", "STAT", "XRMD", "SIZE", - "REIN", "MODE", "MSND", "REST", "XCWD", "HELP", "PWD", "MDTM", - "QUIT", "RETR", "MSOM", "RNFR", "LIST", "NOOP", "XPWD", -} &redef; - -const ftp_port_pat = /[0-9]+([[:blank:]]*,[[:blank:]]*[0-9]+){5}/; - -# Pattern for the argument of EPRT command. -# TODO: the pattern works fot the common case but is not RFC2428-complete. -const ftp_eprt_pat = /\|1\|[0-9]{1,3}(\.[0-9]{1,3}){3}\|[0-9]{1,5}\|/; - -# IP addresses. -const ftp_ip_pat = /[0-9]{1,3}(\.[0-9]{1,3}){3}/; - -# Domain names (deficiency: domain suffices of countries). -const ftp_domain_name_pat = - /([\-0-9a-zA-Z]+\.)+(com|edu|net|org|gov|mil|uk|fr|nl|es|jp|it)/; - -# File names (printable characters). -const ftp_file_name_pat = /[[:print:]]+/; - -# File names that can be left in the clear. -const ftp_public_files = - /\// | /\.\./ # "/" and ".." - | /(\/etc\/|master\.)?(passwd|shadow|s?pwd\.db)/ # ftp_hot_files - | /\/(etc|usr\/bin|bin|sbin|kernel)(\/)?/ - | /\.rhosts/ | /\.forward/ # ftp_hot_guest_files -&redef; - -const ftp_sensitive_files = - /.*(etc\/|master\.)?(passwd|shadow|s?pwd\.db)/ # ftp_hot_files - | /\/(etc|usr\/bin|bin|sbin|kernel)\/.*/ - | /.*\.rhosts/ | /.*\.forward/ # ftp_hot_guest_files -&redef; - -# Public servers. -const ftp_public_servers: set[addr] = {} &redef; - -# Whether we keep all file names (valid or invalid) for public servers. -const ftp_keep_all_files_for_public_servers = F &redef; - -# Public files. -const ftp_known_public_files: set[addr, string] = {} &redef; - -# Hidden file/directory. -const ftp_hidden_file = /.*\/\.[^.\/].*/; -const ftp_public_hidden_file = /0/ &redef; - -# Options for file commands (LIST, NLST) that can be left in the clear. -const ftp_known_option = /-[[:alpha:]]{1,5}[ ]*/; - -const ftp_known_site_cmd = { - "UMASK", "GROUP", "INDEX", "GROUPS", - "IDLE", "GPASS", "EXEC", "CHECKMETHOD", - "CHMOD", "NEWER", "ALIAS", "CHECKSUM", - "HELP", "MINFO", "CDPATH", - - "TRUTH", "UTIME", -} &redef; - -const ftp_sensitive_ids: set[string] = { - "backdoor", "bomb", "diag", "gdm", "issadmin", "msql", "netfrack", - "netphrack", "own", "r00t", "root", "ruut", "smtp", "sundiag", "sync", - "sys", "sysadm", "sysdiag", "sysop", "sysoper", "system", "toor", "tour", - "y0uar3ownd", -}; - -redef anonymize_ip_addr = T; -redef rewriting_ftp_trace = T; - -global ftp_anon_log = open_log_file("ftp-anon") &redef; - -# Anonymized arguments, indexed by the anonymization seed. -global anonymized_args: table[string] of string; - -# Arguments left in the clear, indexed by the argument and the context. -global ftp_arg_left_in_the_clear: set[string, string]; - -# Valid files on public servers. -global ftp_valid_public_files: set[addr, string]; - -type ftp_cmd_arg_anon_result: record { - anonymized: bool; - cmd: string; - arg: string; -}; - - -# Whether anonymize_trace_specific_cmd_arg is defined: -const trace_specific_cmd_arg_anonymization = F &redef; - -# This function is to be defined in a trace-specific script. By -# default, use ftp-anonymizer-trace.bro. - -global anonymize_trace_specific_cmd_arg: - function(session: ftp_session_info, cmd: string, arg: string): - ftp_cmd_arg_anon_result; - - -# Anonymize FTP replies by message patterns. -const process_ftp_reply_by_message_pattern = F &redef; -global anonymize_ftp_reply_by_msg_pattern: - function(code: count, act_msg: string, - cmd_arg: ftp_cmd_arg, session: ftp_session_info): string; - - -# Anonymize an argument *completely* with a hash value of the string, -# and log the anonymization. -function anonymize_arg(typ: string, session: ftp_session_info, cmd: string, arg: string, seed: string): string - { - if ( arg == "" ) - return ""; # an empty argument is safe - - local arg_seed = string_cat(typ, seed, arg); - - if ( arg_seed in anonymized_args ) - return anonymized_args[arg_seed]; - - local a = anonymize_string(arg_seed); - anonymized_args[arg_seed] = a; - - print ftp_anon_log, - fmt("anonymize_arg: (%s) {%s} %s \"%s\" to \"%s\" in [%s]", - typ, seed, cmd, - to_string_literal(arg), to_string_literal(a), - id_string(session$connection_id)); - return a; - } - -# This function is called whenever an argument is to be left in the -# clear. It logs the action if it hasn't occurred before. -function leave_in_the_clear(msg: string, session: ftp_session_info, - arg: string, context: string): string - { - if ( [arg, context] !in ftp_arg_left_in_the_clear ) - { - add ftp_arg_left_in_the_clear[arg, context]; - print ftp_anon_log, fmt("leave_in_the_clear: (%s) \"%s\" [%s] in [%s]", - msg, to_string_literal(arg), context, - id_string(session$connection_id)); - } - return arg; - } - - -# Sometimes the argument of a file command contains an option string -# before the file name, such as in 'LIST -l /xyz/', the following -# function identifies such option strings and separate the argument -# accordingly. - -type separate_option_str_result: record { - opt_str: string; - file_name: string; -}; - -function separate_option_str(file_name: string): separate_option_str_result - { - local ret: separate_option_str_result; - if ( file_name == /-[[:alpha:]]+( .*)?/ ) - { - local parts = split_all(file_name, /-[[:alpha:]]+[ ]*/); - ret$opt_str = string_cat(parts[1], parts[2]); - parts[1] = ""; parts[2] = ""; - ret$file_name = cat_string_array(parts); - return ret; - } - else - return [$opt_str = "", $file_name = file_name]; - } - - -# Anonymize a user id -type login_status_type: enum { - LOGIN_PENDING, - LOGIN_SUCCESSFUL, - LOGIN_FAILED, - LOGIN_UNKNOWN, -}; - -function anonymize_user_id(session: ftp_session_info, id: string, login_status: login_status_type, msg: string): string - { - if ( id in ftp_guest_ids ) - { - leave_in_the_clear("guest_id", session, id, msg); - return id; - } - - else if ( id in ftp_sensitive_ids && login_status == LOGIN_FAILED ) - { - leave_in_the_clear("sensitive_id", session, id, msg); - return id; - } - - else - return anonymize_arg("user_name", session, "USER", id, cat(session$connection_id$resp_h, login_status)); - } - -# Anonymize a file name argument. -function anonymize_file_name_arg(session: ftp_session_info, cmd: string, arg: string, valid_file_name: bool): string - { - local file_name = arg; - local opt_str = ""; - if ( cmd == /LIST|NLST/ ) - { - # Separate the option from file name if there is one - - local ret = separate_option_str(file_name); - if ( ret$opt_str != "" ) - { - opt_str = ret$opt_str; - - # Shall we anonymize the option string? - if ( opt_str != ftp_known_option ) - { - # Anonymize the option conservatively - print ftp_anon_log, fmt("option_anonymized: \"%s\" from (%s %s)", - to_string_literal(opt_str), cmd, file_name); - opt_str = "-