Merge remote-tracking branch 'origin/master' into topic/johanna/spicy-tls

* origin/master: (139 commits)
  Given the -C flag, set script-layer ignore_checksums to true.
  Add btest for "-C" flag vs the script-layer ignore_checksums global.
  Update doc submodule [nomail] [skip ci]
  Remove references to bro_broker in broker/Manager.h
  cmake: Fixup BRO_PLUGIN_INSTALL_PATH references
  testing/external: Bump hashes for community_id addition
  NEWS: Add entry for Community ID
  policy: Import zeek-community-id scripts into protocols/conn frameworks/notice
  Add irc_dcc_send_ack event and fix missing fields
  Fix install directory for plugins
  Update doc submodule [nomail] [skip ci]
  Add community_id_v1() based on corelight/zeek-community-id
  Update NEWS to cover cluster framework changes.
  Add cluster_started restart tests.
  Add basic cluster_started tests.
  Add cluster_started and node_fully_connected events.
  Add hook into cluster connection setup.
  Add broadcast_topics set.
  Generalize Cluster::worker_count.
  Edit pass over the current 6.0 NEWS entries. [nomail] [skip ci]
  ...

scripts/base/protocols/ftp/main.zeek

@@ -316,12 +316,58 @@ event ftp_request(c: connection, command: string, arg: string) &priority=5
 event ftp_reply(c: connection, code: count, msg: string, cont_resp: bool) &priority=5
 	{
 	set_ftp_session(c);
+
+	# Skip matching up intermediate reply lines (that do not have a
+	# valid status code) with pending commands. Because they may not
+	# have a proper status code, there's little point setting whatever
+	# their reply_code and reply_msg are on the command.
+	#
+	# There's a quirk: Some FTP servers return(ed?) replies like the
+	# following, violating the multi-line reply protocol:
+	#
+	#  c: STOR intermol.ps
+	#  s: 150 Opening ASCII mode data connection for 'intermol.ps'.
+	#  s: 230- WARNING! 4 bare linefeeds received in ASCII mode
+	#  s:    File may not have transferred correctly.
+	#  s: 226 Transfer complete.
+	#
+	# This is a multiline response started with 230-, but never finalized
+	# with the same status code. It should have been completed with
+	# "230 <some final message>", but instead was completed with "226 ...".
+	# This confuses our parser, returning cont_resp = T for all following
+	# server messages. This caused a regression as the current command wasn't
+	# updated for logging.
+	#
+	# The regex below is a best effort to keep existing behavior
+	# in face of such traffic. It matches on messages that look
+	# like valid status codes (starting with 3 digits followed by
+	# at least 10 ASCII characters).
+	#
+	# There's the following in RFC 959, so in the future we could push
+	# the detection/logic down into the parser instead of here.
+	#
+	#   If an intermediary line begins with a 3-digit number, the Server
+	#   must pad the front to avoid confusion.
+	#
+	if ( cont_resp && code == 0 && c$ftp?$reply_code )
+		{
+		if ( /^[1-9][0-9]{2} [[:print:]]{10}.*/ !in msg )
+			return;
+		else
+			{
+			# This might be worth a weird, but not sure it's
+			# worth it and how trigger happy it could be.
+			# Reporter::conn_weird("FTP_intermediate_line_with_reply_code", c, msg, "FTP");
+			}
+		}
+
 	c$ftp$cmdarg = get_pending_cmd(c$ftp$pending_commands, code, msg);
 	c$ftp$reply_code = code;
 	c$ftp$reply_msg = msg;
 
-	# TODO: figure out what to do with continued FTP response (not used much)
-	if ( cont_resp ) return;
+	# Do not parse out information from any but the first reply line.
+	if ( cont_resp )
+		return;
 
 	# TODO: do some sort of generic clear text login processing here.
 	local response_xyz = parse_ftp_reply_code(code);

scripts/base/protocols/irc/dcc-send.zeek

@@ -97,7 +97,7 @@ function log_dcc(f: fa_file)
 		}
 	}
 
-event file_new(f: fa_file) &priority=-5
+event file_sniff(f: fa_file, meta: fa_metadata) &priority=-5
 	{
 	if ( f$source == "IRC_DATA" )
 		log_dcc(f);

scripts/base/protocols/smtp/main.zeek

@@ -2,6 +2,7 @@
 @load base/utils/directions-and-hosts
 @load base/utils/email
 @load base/protocols/conn/removal-hooks
+@load base/frameworks/notice/weird
 
 module SMTP;
 
@@ -75,6 +76,11 @@ export {
 		messages_transferred:     count     &default=0;
 
 		pending_messages:         set[Info] &optional;
+
+		trans_mail_from_seen:     bool      &default=F;
+		trans_rcpt_to_seen:       bool      &default=F;
+		invalid_transactions:     count     &default=0;
+		analyzer_id:              count     &optional;
 	};
 
 	## Direction to capture the full "Received from" path.
@@ -91,6 +97,16 @@ export {
 
 	## SMTP finalization hook.  Remaining SMTP info may get logged when it's called.
 	global finalize_smtp: Conn::RemovalHook;
+
+	## When seeing a RCPT TO or DATA command, validate that it has been
+	## preceded by a MAIL FROM or RCPT TO command, respectively, else
+	## log a weird and possibly disable the SMTP analyzer upon too
+	## many invalid transactions.
+	option mail_transaction_validation = T;
+
+	## Disable the SMTP analyzer when that many invalid transactions
+	## have been observed in an SMTP session.
+	option max_invalid_mail_transactions = 25;
 }
 
 redef record connection += {
@@ -151,6 +167,22 @@ function set_smtp_session(c: connection)
 		c$smtp = new_smtp_log(c);
 	}
 
+function mail_transaction_invalid(c: connection, addl: string)
+	{
+	Reporter::conn_weird("smtp_mail_transaction_invalid", c, addl, "SMTP");
+
+	++c$smtp_state$invalid_transactions;
+
+	if ( max_invalid_mail_transactions > 0
+	     && c$smtp_state$invalid_transactions > max_invalid_mail_transactions
+	     && c$smtp_state?$analyzer_id )
+		{
+		Reporter::conn_weird("smtp_excessive_invalid_mail_transactions", c, "", "SMTP");
+		if ( disable_analyzer(c$id, c$smtp_state$analyzer_id) )
+			delete c$smtp_state$analyzer_id;
+		}
+	}
+
 function smtp_message(c: connection)
 	{
 	if ( c$smtp$has_client_activity )
@@ -160,6 +192,15 @@ function smtp_message(c: connection)
 		}
 	}
 
+event analyzer_confirmation_info(atype: AllAnalyzers::Tag, info: AnalyzerConfirmationInfo)
+	{
+	if ( atype != Analyzer::ANALYZER_SMTP )
+		return;
+
+	set_smtp_session(info$c);
+	info$c$smtp_state$analyzer_id = info$aid;
+	}
+
 event smtp_request(c: connection, is_orig: bool, command: string, arg: string) &priority=5
 	{
 	set_smtp_session(c);
@@ -184,6 +225,13 @@ event smtp_request(c: connection, is_orig: bool, command: string, arg: string) &
 			}
 
 		c$smtp$has_client_activity = T;
+		c$smtp_state$trans_rcpt_to_seen = T;
+
+		if ( mail_transaction_validation )
+			{
+			if ( ! c$smtp_state$trans_mail_from_seen )
+				mail_transaction_invalid(c, "rcpt to missing mail from");
+			}
 		}
 
 	else if ( upper_command == "MAIL" && /^[fF][rR][oO][mM]:/ in arg )
@@ -195,6 +243,23 @@ event smtp_request(c: connection, is_orig: bool, command: string, arg: string) &
 		if ( mailfrom != "" )
 			c$smtp$mailfrom = mailfrom;
 		c$smtp$has_client_activity = T;
+
+		c$smtp_state$trans_mail_from_seen = T;
+		c$smtp_state$trans_rcpt_to_seen = F;  # Reset state on MAIL FROM
+		}
+	else if ( upper_command == "DATA" )
+		{
+		if ( mail_transaction_validation )
+			{
+			if ( ! c$smtp_state$trans_rcpt_to_seen )  # mail from checked in rctp to
+				mail_transaction_invalid(c, "data missing rcpt to");
+			}
+		}
+	else if ( upper_command == "." )
+		{
+		# Reset state when we're seeing a .
+		c$smtp_state$trans_mail_from_seen = F;
+		c$smtp_state$trans_rcpt_to_seen = F;
 		}
 	}