Documentation updates for HTTP & IRC scripts.

Closes #733
This commit is contained in:
Seth Hall 2012-01-08 02:22:52 -05:00
parent 48ed922e06
commit 3be1222532
17 changed files with 160 additions and 111 deletions

View file

@ -8,29 +8,24 @@
module HTTP;
export {
## Pattern of file mime types to extract from HTTP entity bodies.
## Pattern of file mime types to extract from HTTP response entity bodies.
const extract_file_types = /NO_DEFAULT/ &redef;
## The on-disk prefix for files to be extracted from HTTP entity bodies.
const extraction_prefix = "http-item" &redef;
redef record Info += {
## This field can be set per-connection to determine if the entity body
## will be extracted. It must be set to T on or before the first
## entity_body_data event.
extracting_file: bool &default=F;
## This is the holder for the file handle as the file is being written
## to disk.
## On-disk file where the response body was extracted to.
extraction_file: file &log &optional;
};
redef record State += {
entity_bodies: count &default=0;
## Indicates if the response body is to be extracted or not. Must be
## set before or by the first :bro:id:`http_entity_data` event for the
## content.
extract_file: bool &default=F;
};
}
event http_entity_data(c: connection, is_orig: bool, length: count, data: string) &priority=5
event http_entity_data(c: connection, is_orig: bool, length: count, data: string) &priority=-5
{
# Client body extraction is not currently supported in this script.
if ( is_orig )
@ -41,8 +36,12 @@ event http_entity_data(c: connection, is_orig: bool, length: count, data: string
if ( c$http?$mime_type &&
extract_file_types in c$http$mime_type )
{
c$http$extracting_file = T;
local suffix = fmt("%s_%d.dat", is_orig ? "orig" : "resp", ++c$http_state$entity_bodies);
c$http$extract_file = T;
}
if ( c$http$extract_file )
{
local suffix = fmt("%s_%d.dat", is_orig ? "orig" : "resp", c$http_state$current_response);
local fname = generate_extraction_filename(extraction_prefix, c, suffix);
c$http$extraction_file = open(fname);
@ -50,12 +49,12 @@ event http_entity_data(c: connection, is_orig: bool, length: count, data: string
}
}
if ( c$http$extracting_file )
if ( c$http?$extraction_file )
print c$http$extraction_file, data;
}
event http_end_entity(c: connection, is_orig: bool)
{
if ( c$http$extracting_file )
if ( c$http?$extraction_file )
close(c$http$extraction_file);
}

View file

@ -11,7 +11,8 @@ export {
};
redef record Info += {
## The MD5 sum for a file transferred over HTTP will be stored here.
## MD5 sum for a file transferred over HTTP calculated from the
## response body.
md5: string &log &optional;
## This value can be set per-transfer to determine per request
@ -19,8 +20,8 @@ export {
## set to T at the time of or before the first chunk of body data.
calc_md5: bool &default=F;
## This boolean value indicates if an MD5 sum is currently being
## calculated for the current file transfer.
## Indicates if an MD5 sum is being calculated for the current
## request/response pair.
calculating_md5: bool &default=F;
};

View file

@ -1,5 +1,4 @@
##! This script is involved in the identification of file types in HTTP
##! response bodies.
##! Identification of file types in HTTP response bodies with file content sniffing.
@load base/frameworks/signatures
@load base/frameworks/notice
@ -15,27 +14,23 @@ module HTTP;
export {
redef enum Notice::Type += {
# This notice is thrown when the file extension doesn't
# seem to match the file contents.
## Indicates when the file extension doesn't seem to match the file contents.
Incorrect_File_Type,
};
redef record Info += {
## This will record the mime_type identified.
## Mime type of response body identified by content sniffing.
mime_type: string &log &optional;
## This indicates that no data of the current file transfer has been
## Indicates that no data of the current file transfer has been
## seen yet. After the first :bro:id:`http_entity_data` event, it
## will be set to T.
## will be set to F.
first_chunk: bool &default=T;
};
redef enum Tags += {
IDENTIFIED_FILE
};
# Create regexes that *should* in be in the urls for specifics mime types.
# Notices are thrown if the pattern doesn't match the url for the file type.
## Mapping between mime types and regular expressions for URLs
## The :bro:enum:`HTTP::Incorrect_File_Type` notice is generated if the pattern
## doesn't match the mime type that was discovered.
const mime_types_extensions: table[string] of pattern = {
["application/x-dosexec"] = /\.([eE][xX][eE]|[dD][lL][lL])/,
} &redef;

View file

@ -1,3 +1,7 @@
##! Implements base functionality for HTTP analysis. The logging model is
##! to log request/response pairs and all relevant metadata together in
##! a single record.
@load base/utils/numbers
@load base/utils/files
@ -8,6 +12,7 @@ export {
## Indicate a type of attack or compromise in the record to be logged.
type Tags: enum {
## Placeholder.
EMPTY
};
@ -15,64 +20,69 @@ export {
const default_capture_password = F &redef;
type Info: record {
## Timestamp for when the request happened.
ts: time &log;
uid: string &log;
id: conn_id &log;
## This represents the pipelined depth into the connection of this
## Represents the pipelined depth into the connection of this
## request/response transaction.
trans_depth: count &log;
## The verb used in the HTTP request (GET, POST, HEAD, etc.).
## Verb used in the HTTP request (GET, POST, HEAD, etc.).
method: string &log &optional;
## The value of the HOST header.
## Value of the HOST header.
host: string &log &optional;
## The URI used in the request.
## URI used in the request.
uri: string &log &optional;
## The value of the "referer" header. The comment is deliberately
## Value of the "referer" header. The comment is deliberately
## misspelled like the standard declares, but the name used here is
## "referrer" spelled correctly.
referrer: string &log &optional;
## The value of the User-Agent header from the client.
## Value of the User-Agent header from the client.
user_agent: string &log &optional;
## The actual uncompressed content size of the data transferred from
## Actual uncompressed content size of the data transferred from
## the client.
request_body_len: count &log &default=0;
## The actual uncompressed content size of the data transferred from
## Actual uncompressed content size of the data transferred from
## the server.
response_body_len: count &log &default=0;
## The status code returned by the server.
## Status code returned by the server.
status_code: count &log &optional;
## The status message returned by the server.
## Status message returned by the server.
status_msg: string &log &optional;
## The last 1xx informational reply code returned by the server.
## Last seen 1xx informational reply code returned by the server.
info_code: count &log &optional;
## The last 1xx informational reply message returned by the server.
## Last seen 1xx informational reply message returned by the server.
info_msg: string &log &optional;
## The filename given in the Content-Disposition header
## sent by the server.
## Filename given in the Content-Disposition header sent by the server.
filename: string &log &optional;
## This is a set of indicators of various attributes discovered and
## A set of indicators of various attributes discovered and
## related to a particular request/response pair.
tags: set[Tags] &log;
## The username if basic-auth is performed for the request.
## Username if basic-auth is performed for the request.
username: string &log &optional;
## The password if basic-auth is performed for the request.
## Password if basic-auth is performed for the request.
password: string &log &optional;
## This determines if the password will be captured for this request.
## Determines if the password will be captured for this request.
capture_password: bool &default=default_capture_password;
## All of the headers that may indicate if the request was proxied.
proxied: set[string] &log &optional;
};
## Structure to maintain state for an HTTP connection with multiple
## requests and responses.
type State: record {
## Pending requests.
pending: table[count] of Info;
current_response: count &default=0;
## Current request in the pending queue.
current_request: count &default=0;
## Current response in the pending queue.
current_response: count &default=0;
};
## The list of HTTP headers typically used to indicate a proxied request.
## A list of HTTP headers typically used to indicate proxied requests.
const proxy_headers: set[string] = {
"FORWARDED",
"X-FORWARDED-FOR",
@ -83,6 +93,8 @@ export {
"PROXY-CONNECTION",
} &redef;
## Event that can be handled to access the HTTP record as it is sent on
## to the logging framework.
global log_http: event(rec: Info);
}

View file

@ -5,8 +5,31 @@
module HTTP;
export {
## Given a string containing a series of key-value pairs separated by "=",
## this function can be used to parse out all of the key names.
##
## data: The raw data, such as a URL or cookie value.
##
## kv_splitter: A regular expression representing the separator between
## key-value pairs.
##
## Returns: A vector of strings containing the keys.
global extract_keys: function(data: string, kv_splitter: pattern): string_vec;
## Creates a URL from an :bro:type:`HTTP::Info` record. This should handle
## edge cases such as proxied requests appropriately.
##
## rec: An :bro:type:`HTTP::Info` record.
##
## Returns: A URL, not prefixed by "http://".
global build_url: function(rec: Info): string;
## Creates a URL from an :bro:type:`HTTP::Info` record. This should handle
## edge cases such as proxied requests appropriately.
##
## rec: An :bro:type:`HTTP::Info` record.
##
## Returns: A URL prefixed with "http://".
global build_url_http: function(rec: Info): string;
}

View file

@ -5,8 +5,9 @@
##! but that connection will actually be between B and C which could be
##! analyzed on a different worker.
##!
##! Example line from IRC server indicating that the DCC SEND is about to start:
##! PRIVMSG my_nick :^ADCC SEND whateverfile.zip 3640061780 1026 41709^A
# Example line from IRC server indicating that the DCC SEND is about to start:
# PRIVMSG my_nick :^ADCC SEND whateverfile.zip 3640061780 1026 41709^A
@load ./main
@load base/utils/files
@ -14,23 +15,24 @@
module IRC;
export {
redef enum Tag += { EXTRACTED_FILE };
## Pattern of file mime types to extract from IRC DCC file transfers.
const extract_file_types = /NO_DEFAULT/ &redef;
## The on-disk prefix for files to be extracted from IRC DCC file transfers.
## On-disk prefix for files to be extracted from IRC DCC file transfers.
const extraction_prefix = "irc-dcc-item" &redef;
redef record Info += {
## DCC filename requested.
dcc_file_name: string &log &optional;
## Size of the DCC transfer as indicated by the sender.
dcc_file_size: count &log &optional;
## Sniffed mime type of the file.
dcc_mime_type: string &log &optional;
## The file handle for the file to be extracted
extraction_file: file &log &optional;
## A boolean to indicate if the current file transfer should be extraced.
## A boolean to indicate if the current file transfer should be extracted.
extract_file: bool &default=F;
## The count of the number of file that have been extracted during the session.
@ -54,8 +56,10 @@ event file_transferred(c: connection, prefix: string, descr: string,
if ( extract_file_types == irc$dcc_mime_type )
{
irc$extract_file = T;
add irc$tags[EXTRACTED_FILE];
}
if ( irc$extract_file )
{
local suffix = fmt("%d.dat", ++irc$num_extracted_files);
local fname = generate_extraction_filename(extraction_prefix, c, suffix);
irc$extraction_file = open(fname);
@ -76,7 +80,7 @@ event file_transferred(c: connection, prefix: string, descr: string,
Log::write(IRC::LOG, irc);
irc$command = tmp;
if ( irc$extract_file && irc?$extraction_file )
if ( irc?$extraction_file )
set_contents_file(id, CONTENTS_RESP, irc$extraction_file);
# Delete these values in case another DCC transfer

View file

@ -1,36 +1,38 @@
##! This is the script that implements the core IRC analysis support. It only
##! logs a very limited subset of the IRC protocol by default. The points
##! that it logs at are NICK commands, USER commands, and JOIN commands. It
##! log various bits of meta data as indicated in the :bro:type:`IRC::Info`
##! record along with the command at the command arguments.
##! Implements the core IRC analysis support. The logging model is to log
##! IRC commands along with the associated response and some additional
##! metadata about the connection if it's available.
module IRC;
export {
redef enum Log::ID += { LOG };
type Tag: enum {
EMPTY
};
type Info: record {
## Timestamp when the command was seen.
ts: time &log;
uid: string &log;
id: conn_id &log;
## Nick name given for the connection.
nick: string &log &optional;
## User name given for the connection.
user: string &log &optional;
channels: set[string] &log &optional;
## Command given by the client.
command: string &log &optional;
## Value for the command given by the client.
value: string &log &optional;
## Any additional data for the command.
addl: string &log &optional;
tags: set[Tag] &log;
};
## Event that can be handled to access the IRC record as it is sent on
## to the logging framework.
global irc_log: event(rec: Info);
}
redef record connection += {
## IRC session information.
irc: Info &optional;
};

View file

@ -1,8 +1,8 @@
##! This script takes MD5 sums of files transferred over HTTP and checks them with
##! Team Cymru's Malware Hash Registry (http://www.team-cymru.org/Services/MHR/).
##! Detect file downloads over HTTP that have MD5 sums matching files in Team
##! Cymru's Malware Hash Registry (http://www.team-cymru.org/Services/MHR/).
##! By default, not all file transfers will have MD5 sums calculated. Read the
##! documentation for the :doc:base/protocols/http/file-hash.bro script to see how to
##! configure which transfers will have hashes calculated.
##! documentation for the :doc:base/protocols/http/file-hash.bro script to see
##! how to configure which transfers will have hashes calculated.
@load base/frameworks/notice
@load base/protocols/http

View file

@ -1,4 +1,4 @@
##! Intelligence based HTTP detections.
##! Intelligence based HTTP detections. Not yet working!
@load base/protocols/http/main
@load base/protocols/http/utils

View file

@ -16,7 +16,9 @@ export {
};
redef enum Metrics::ID += {
## Metric to track SQL injection attackers.
SQLI_ATTACKER,
## Metrics to track SQL injection victims.
SQLI_VICTIM,
};
@ -30,7 +32,7 @@ export {
COOKIE_SQLI,
};
## This defines the threshold that determines if an SQL injection attack
## Defines the threshold that determines if an SQL injection attack
## is ongoing based on the number of requests that appear to be SQL
## injection attacks.
const sqli_requests_threshold = 50 &redef;
@ -40,7 +42,7 @@ export {
## At the end of each interval the counter is reset.
const sqli_requests_interval = 5min &redef;
## This regular expression is used to match URI based SQL injections
## Regular expression is used to match URI based SQL injections.
const match_sql_injection_uri =
/[\?&][^[:blank:]\x00-\x37\|]+?=[\-[:alnum:]%]+([[:blank:]\x00-\x37]|\/\*.*?\*\/)*['"]?([[:blank:]\x00-\x37]|\/\*.*?\*\/|\)?;)+.*?([hH][aA][vV][iI][nN][gG]|[uU][nN][iI][oO][nN]|[eE][xX][eE][cC]|[sS][eE][lL][eE][cC][tT]|[dD][eE][lL][eE][tT][eE]|[dD][rR][oO][pP]|[dD][eE][cC][lL][aA][rR][eE]|[cC][rR][eE][aA][tT][eE]|[iI][nN][sS][eE][rR][tT])([[:blank:]\x00-\x37]|\/\*.*?\*\/)+/
| /[\?&][^[:blank:]\x00-\x37\|]+?=[\-0-9%]+([[:blank:]\x00-\x37]|\/\*.*?\*\/)*['"]?([[:blank:]\x00-\x37]|\/\*.*?\*\/|\)?;)+([xX]?[oO][rR]|[nN]?[aA][nN][dD])([[:blank:]\x00-\x37]|\/\*.*?\*\/)+['"]?(([^a-zA-Z&]+)?=|[eE][xX][iI][sS][tT][sS])/

View file

@ -1,3 +1,5 @@
##! Detect and log web applications through the software framework.
@load base/frameworks/signatures
@load base/frameworks/software
@load base/protocols/http
@ -10,10 +12,12 @@ redef Signatures::ignored_ids += /^webapp-/;
export {
redef enum Software::Type += {
## Identifier for web applications in the software framework.
WEB_APPLICATION,
};
redef record Software::Info += {
## Most root URL where the software was discovered.
url: string &optional &log;
};
}

View file

@ -1,5 +1,5 @@
##! This script take advantage of a few ways that installed plugin information
##! leaks from web browsers.
##! Detect browser plugins as they leak through requests to Omniture
##! advertising servers.
@load base/protocols/http
@load base/frameworks/software
@ -13,6 +13,7 @@ export {
};
redef enum Software::Type += {
## Identifier for browser plugins in the software framework.
BROWSER_PLUGIN
};
}

View file

@ -6,8 +6,11 @@ module HTTP;
export {
redef enum Software::Type += {
## Identifier for web servers in the software framework.
SERVER,
## Identifier for app servers in the software framework.
APPSERVER,
## Identifier for web browsers in the software framework.
BROWSER,
};

View file

@ -1,4 +1,4 @@
##! This script extracts and logs variables from cookies sent by clients
##! Extracts and logs variables names from cookies sent by clients.
@load base/protocols/http/main
@load base/protocols/http/utils
@ -6,6 +6,7 @@
module HTTP;
redef record Info += {
## Variable names extracted from all cookies.
cookie_vars: vector of string &optional &log;
};

View file

@ -1,10 +1,12 @@
##! This script extracts and logs variables from the requested URI
##! Extracts and log variables from the requested URI in the default HTTP
##! logging stream.
@load base/protocols/http
module HTTP;
redef record Info += {
## Variable names from the URI.
uri_vars: vector of string &optional &log;
};

View file

@ -3,9 +3,9 @@
#empty_field (empty)
#unset_field -
#path irc
#fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p nick user channels command value addl tags dcc_file_name dcc_file_size extraction_file
#types time string addr port addr port string string table[string] string string string table[enum] string count file
1311189164.119437 UWkUyAuUGXf 192.168.1.77 57640 66.198.80.67 6667 - - - NICK bloed - (empty) - - -
1311189164.119437 UWkUyAuUGXf 192.168.1.77 57640 66.198.80.67 6667 bloed - - USER sdkfje sdkfje Montreal.QC.CA.Undernet.org dkdkrwq (empty) - - -
1311189174.474127 UWkUyAuUGXf 192.168.1.77 57640 66.198.80.67 6667 bloed sdkfje - JOIN #easymovies (empty) (empty) - - -
1311189316.326025 UWkUyAuUGXf 192.168.1.77 57640 66.198.80.67 6667 bloed sdkfje - DCC #easymovies (empty) (empty) ladyvampress-default(2011-07-07)-OS.zip 42208 -
#fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p nick user command value addl dcc_file_name dcc_file_size extraction_file
#types time string addr port addr port string string string string string string count file
1311189164.119437 UWkUyAuUGXf 192.168.1.77 57640 66.198.80.67 6667 - - NICK bloed - - - -
1311189164.119437 UWkUyAuUGXf 192.168.1.77 57640 66.198.80.67 6667 bloed - USER sdkfje sdkfje Montreal.QC.CA.Undernet.org dkdkrwq - - -
1311189174.474127 UWkUyAuUGXf 192.168.1.77 57640 66.198.80.67 6667 bloed sdkfje JOIN #easymovies (empty) - - -
1311189316.326025 UWkUyAuUGXf 192.168.1.77 57640 66.198.80.67 6667 bloed sdkfje DCC #easymovies (empty) ladyvampress-default(2011-07-07)-OS.zip 42208 -

View file

@ -3,9 +3,9 @@
#empty_field (empty)
#unset_field -
#path irc
#fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p nick user channels command value addl tags dcc_file_name dcc_file_size dcc_mime_type extraction_file
#types time string addr port addr port string string table[string] string string string table[enum] string count string file
1311189164.119437 UWkUyAuUGXf 192.168.1.77 57640 66.198.80.67 6667 - - - NICK bloed - (empty) - - - -
1311189164.119437 UWkUyAuUGXf 192.168.1.77 57640 66.198.80.67 6667 bloed - - USER sdkfje sdkfje Montreal.QC.CA.Undernet.org dkdkrwq (empty) - - - -
1311189174.474127 UWkUyAuUGXf 192.168.1.77 57640 66.198.80.67 6667 bloed sdkfje - JOIN #easymovies (empty) (empty) - - - -
1311189316.326025 UWkUyAuUGXf 192.168.1.77 57640 66.198.80.67 6667 bloed sdkfje - DCC #easymovies (empty) IRC::EXTRACTED_FILE ladyvampress-default(2011-07-07)-OS.zip 42208 FAKE_MIME irc-dcc-item_192.168.1.77:57655-209.197.168.151:1024_1.dat
#fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p nick user command value addl dcc_file_name dcc_file_size dcc_mime_type extraction_file
#types time string addr port addr port string string string string string string count string file
1311189164.119437 UWkUyAuUGXf 192.168.1.77 57640 66.198.80.67 6667 - - NICK bloed - - - - -
1311189164.119437 UWkUyAuUGXf 192.168.1.77 57640 66.198.80.67 6667 bloed - USER sdkfje sdkfje Montreal.QC.CA.Undernet.org dkdkrwq - - - -
1311189174.474127 UWkUyAuUGXf 192.168.1.77 57640 66.198.80.67 6667 bloed sdkfje JOIN #easymovies (empty) - - - -
1311189316.326025 UWkUyAuUGXf 192.168.1.77 57640 66.198.80.67 6667 bloed sdkfje DCC #easymovies (empty) ladyvampress-default(2011-07-07)-OS.zip 42208 FAKE_MIME irc-dcc-item_192.168.1.77:57655-209.197.168.151:1024_1.dat