SSH&FTP Documentation updates.

Closes #732

scripts/base/protocols/ftp/file-extract.bro

@@ -1,4 +1,4 @@
-##! File extraction for FTP.
+##! File extraction support for FTP.
 
 @load ./main
 @load base/utils/files
@@ -6,7 +6,7 @@
 module FTP;
 
 export {
-	## Pattern of file mime types to extract from FTP entity bodies.
+	## Pattern of file mime types to extract from FTP transfers.
 	const extract_file_types = /NO_DEFAULT/ &redef;
 
 	## The on-disk prefix for files to be extracted from FTP-data transfers.
@@ -14,10 +14,15 @@ export {
 }
 
 redef record Info += {
-	## The file handle for the file to be extracted
+	## On disk file where it was extracted to.
 	extraction_file:       file &log &optional;
 	
+	## Indicates if the current command/response pair should attempt to 
+	## extract the file if a file was transferred.
 	extract_file:          bool &default=F;
+	
+	## Internal tracking of the total number of files extracted during this 
+	## session.
 	num_extracted_files:   count &default=0;
 };
 
@@ -33,7 +38,6 @@ event file_transferred(c: connection, prefix: string, descr: string,
 	if ( extract_file_types in s$mime_type )
 		{
 		s$extract_file = T;
-		add s$tags["extracted_file"];
 		++s$num_extracted_files;
 		}
 	}

scripts/base/protocols/ftp/main.bro

@@ -1,11 +1,7 @@
 ##! The logging this script does is primarily focused on logging FTP commands
 ##! along with metadata.  For example, if files are transferred, the argument
 ##! will take on the full path that the client is at along with the requested 
-##! file name.  
+##! file name.
-##! 
-##! TODO:
-##!
-##! * Handle encrypted sessions correctly (get an example?)
 
 @load ./utils-commands
 @load base/utils/paths
@@ -14,38 +10,64 @@
 module FTP;
 
 export {
+	## The FTP protocol logging stream identifier.
 	redef enum Log::ID += { LOG };
-
+	
+	## List of commands that should have their command/response pairs logged.
+	const logged_commands = {
+		"APPE", "DELE", "RETR", "STOR", "STOU", "ACCT"
+	} &redef;
+	
 	## This setting changes if passwords used in FTP sessions are captured or not.
 	const default_capture_password = F &redef;
 	
+	## User IDs that can be considered "anonymous".
+	const guest_ids = { "anonymous", "ftp", "guest" } &redef;
+	
 	type Info: record {
+		## Time when the command was sent.
 		ts:               time        &log;
 		uid:              string      &log;
 		id:               conn_id     &log;
+		## User name for the current FTP session.
 		user:             string      &log &default="<unknown>";
+		## Password for the current FTP session if captured.
 		password:         string      &log &optional;
+		## Command given by the client.
 		command:          string      &log &optional;
+		## Argument for the command if one is given.
 		arg:              string      &log &optional;
-		                              
+		
+		## Libmagic "sniffed" file type if the command indicates a file transfer.
 		mime_type:        string      &log &optional;
+		## Libmagic "sniffed" file description if the command indicates a file transfer.
 		mime_desc:        string      &log &optional;
+		## Size of the file if the command indicates a file transfer.
 		file_size:        count       &log &optional;
+		
+		## Reply code from the server in response to the command.
 		reply_code:       count       &log &optional;
+		## Reply message from the server in response to the command.
 		reply_msg:        string      &log &optional;
+		## Arbitrary tags that may indicate a particular attribute of this command.
 		tags:             set[string] &log &default=set();
 		
-		## By setting the CWD to '/.', we can indicate that unless something
+		## Current working directory that this session is in.  By making
+		## the default value '/.', we can indicate that unless something
 		## more concrete is discovered that the existing but unknown
 		## directory is ok to use.
 		cwd:                string  &default="/.";
+		
+		## Command that is currently waiting for a response.
 		cmdarg:             CmdArg  &optional;
+		## Queue for commands that have been sent but not yet responded to 
+		## are tracked here.
 		pending_commands:   PendingCmds;
 		
-		## This indicates if the session is in active or passive mode.
+		## Indicates if the session is in active or passive mode.
 		passive:            bool &default=F;
 		
-		## This determines if the password will be captured for this request.
+		## Determines if the password will be captured for this request.
 		capture_password:   bool &default=default_capture_password;
 	};
 
@@ -56,22 +78,12 @@ export {
 		y: count;
 		z: count;
 	};
-
-	# TODO: add this back in some form.  raise a notice again?
-	#const excessive_filename_len = 250 &redef;
-	#const excessive_filename_trunc_len = 32 &redef;
-
-	## These are user IDs that can be considered "anonymous".
-	const guest_ids = { "anonymous", "ftp", "guest" } &redef;
 	
-	## The list of commands that should have their command/response pairs logged.
+	## Parse FTP reply codes into the three constituent single digit values.
-	const logged_commands = {
-		"APPE", "DELE", "RETR", "STOR", "STOU", "ACCT"
-	} &redef;
-	
-	## This function splits FTP reply codes into the three constituent 
 	global parse_ftp_reply_code: function(code: count): ReplyCode;
-
+	
+	## Event that can be handled to access the :bro:type:`FTP::Info`
+	## record as it is sent on to the logging framework.
 	global log_ftp: event(rec: Info);
 }
 

scripts/base/protocols/ftp/utils-commands.bro

@@ -2,14 +2,22 @@ module FTP;
 
 export {
 	type CmdArg: record {
+		## Time when the command was sent.
 		ts:   time;
+		## Command.
 		cmd:  string &default="<unknown>";
+		## Argument for the command if one was given.
 		arg:  string &default="";
+		## Counter to track how many commands have been executed.
 		seq:  count &default=0;
 	};
-
+	
+	## Structure for tracking pending commands in the event that the client
+	## sends a large number of commands before the server has a chance to 
+	## reply.
 	type PendingCmds: table[count] of CmdArg;
-
+	
+	## Possible response codes for a wide variety of FTP commands.
 	const cmd_reply_code: set[string, count] = {
 		# According to RFC 959
 		["<init>", [120, 220, 421]],

scripts/base/protocols/ssh/main.bro

@@ -14,15 +14,17 @@
 module SSH;
 
 export {
+	## The SSH protocol logging stream identifier.
 	redef enum Log::ID += { LOG };
 	
 	redef enum Notice::Type += { 
-		## This indicates that a heuristically detected "successful" SSH 
+		## Indicates that a heuristically detected "successful" SSH 
 		## authentication occurred.
 		Login 
 	};
 
 	type Info: record {
+		## Time when the SSH connection began.
 		ts:              time         &log;
 		uid:             string       &log;
 		id:              conn_id      &log;
@@ -34,11 +36,11 @@ export {
 		## would be set for the opposite situation.
 		# TODO: handle local-local and remote-remote better.
 		direction:       Direction    &log &optional;
-		## The software string given by the client.
+		## Software string given by the client.
 		client:          string       &log &optional;
-		## The software string given by the server.
+		## Software string given by the server.
 		server:          string       &log &optional;
-		## The amount of data returned from the server.  This is currently
+		## Amount of data returned from the server.  This is currently
 		## the only measure of the success heuristic and it is logged to 
 		## assist analysts looking at the logs to make their own determination
 		## about the success on a case-by-case basis.
@@ -48,8 +50,8 @@ export {
 		done:            bool         &default=F;
 	};
 	
-	## The size in bytes at which the SSH connection is presumed to be
+	## The size in bytes of data sent by the server at which the SSH 
-	## successful.
+	## connection is presumed to be successful.
 	const authentication_data_size = 5500 &redef;
 	
 	## If true, we tell the event engine to not look at further data
@@ -58,14 +60,16 @@ export {
 	## kinds of analyses (e.g., tracking connection size).
 	const skip_processing_after_detection = F &redef;
 	
-	## This event is generated when the heuristic thinks that a login
+	## Event that is generated when the heuristic thinks that a login
 	## was successful.
 	global heuristic_successful_login: event(c: connection);
 	
-	## This event is generated when the heuristic thinks that a login
+	## Event that is generated when the heuristic thinks that a login
 	## failed.
 	global heuristic_failed_login: event(c: connection);
 	
+	## Event that can be handled to access the :bro:type:`SSH::Info`
+	## record as it is sent on to the logging framework.
 	global log_ssh: event(rec: Info);
 }
 

scripts/policy/protocols/ftp/detect.bro

@@ -7,7 +7,7 @@ module FTP;
 
 export {
 	redef enum Notice::Type += {
-		## This indicates that a successful response to a "SITE EXEC" 
+		## Indicates that a successful response to a "SITE EXEC" 
 		## command/arg pair was seen.
 		Site_Exec_Success,
 	};

scripts/policy/protocols/ftp/software.bro

@@ -12,8 +12,10 @@ module FTP;
 
 export {
 	redef enum Software::Type += {
-		FTP_CLIENT,
+		## Identifier for FTP clients in the software framework.
-		FTP_SERVER,
+		CLIENT,
+		## Not currently implemented.
+		SERVER,
 	};
 }
 
@@ -21,7 +23,7 @@ event ftp_request(c: connection, command: string, arg: string) &priority=4
 	{
 	if ( command == "CLNT" )
 		{
-		local si = Software::parse(arg, c$id$orig_h, FTP_CLIENT);
+		local si = Software::parse(arg, c$id$orig_h, CLIENT);
 		Software::found(c$id, si);
 		}
 	}

scripts/policy/protocols/ssh/detect-bruteforcing.bro

@@ -1,3 +1,5 @@
+##! Detect hosts which are doing password guessing attacks and/or password
+##! bruteforcing over SSH.
 
 @load base/protocols/ssh
 @load base/frameworks/metrics
@@ -13,13 +15,13 @@ export {
 		## determined failed logins.
 		Password_Guessing,
 		## Indicates that a host previously identified as a "password guesser"
-		## has now had a heuristically successful login attempt.
+		## has now had a heuristically successful login attempt.  This is not
+		## currently implemented.
 		Login_By_Password_Guesser,
 	};
 	
 	redef enum Metrics::ID  += {
-		## This metric is to measure failed logins with the hope of detecting
+		## Metric is to measure failed logins.
-		## bruteforcing hosts.
 		FAILED_LOGIN,
 	};
 
@@ -37,7 +39,7 @@ export {
 	## client subnets and the yield value represents server subnets.
 	const ignore_guessers: table[subnet] of subnet &redef;
 
-	## Keeps track of hosts identified as guessing passwords.
+	## Tracks hosts identified as guessing passwords.
 	global password_guessers: set[addr] 
 		&read_expire=guessing_timeout+1hr &synchronized &redef;
 }

scripts/policy/protocols/ssh/geo-data.bro

@@ -1,5 +1,4 @@
-##! This implements all of the additional information and geodata detections 
+##! Geodata based detections for SSH analysis.
-##! for SSH analysis.
 
 @load base/frameworks/notice
 @load base/protocols/ssh
@@ -19,8 +18,8 @@ export {
 		remote_location: geo_location &log &optional;
 	};
 	
-	## The set of countries for which you'd like to throw notices upon 
+	## The set of countries for which you'd like to generate notices upon 
-	## successful login
+	## successful login.
 	const watched_countries: set[string] = {"RO"} &redef;
 }
 

scripts/policy/protocols/ssh/interesting-hostnames.bro

@@ -10,7 +10,7 @@ module SSH;
 
 export {
 	redef enum Notice::Type += {
-		## Generated if a login originates or responds with a host and the 
+		## Generated if a login originates or responds with a host where the 
 		## reverse hostname lookup resolves to a name matched by the 
 		## :bro:id:`SSH::interesting_hostnames` regular expression.
 		Interesting_Hostname_Login,

scripts/policy/protocols/ssh/software.bro

@@ -1,4 +1,4 @@
-##! This script extracts SSH client and server information from SSH 
+##! Extracts SSH client and server information from SSH 
 ##! connections and forwards it to the software framework.
 
 @load base/frameworks/software
@@ -7,7 +7,9 @@ module SSH;
 
 export {
 	redef enum Software::Type += {
+		## Identifier for SSH clients in the software framework.
 		SERVER,
+		## Identifier for SSH servers in the software framework.
 		CLIENT,
 	};
 }