postgresql: Initial parser implementation

This adds a protocol parser for the PostgreSQL protocol and a new
postgresql.log similar to the existing mysql.log.

This should be considered preliminary and hopefully during 7.1 and 7.2
with feedback from the community, we can improve on the events and logs.
Even if most PostgreSQL communication is encrypted in the real-world, this
will minimally allow monitoring of the SSLRequest and hand off further
analysis to the SSL analyzer.

This originates from github.com/awelzel/spicy-postgresql, with lots of
polishing happening in the past two days.

scripts/base/protocols/postgresql/__load__.zeek

@@ -0,0 +1,6 @@
+@if ( have_spicy_analyzers() )
+@load ./consts
+@load ./spicy-events
+@load ./main
+@load-sigs ./dpd
+@endif

scripts/base/protocols/postgresql/consts.zeek

@@ -0,0 +1,37 @@
+module PostgreSQL;
+
+export {
+	# https://www.postgresql.org/docs/current/protocol-error-fields.html
+	global error_ids: table[string] of string = {
+		["S"] = "SeverityLocalized",
+		["V"] = "Severity",  # non-localized
+		["C"] = "Code",
+		["M"] = "Message",
+		["D"] = "Detail",
+		["H"] = "Hint",
+		["P"] = "Position",
+		["p"] = "InternalPosition",
+		["q"] = "InternalQuery",
+		["W"] = "Where",
+		["s"] = "Schema",
+		["t"] = "Table",
+		["c"] = "Column",
+		["d"] = "Data",
+		["n"] = "Constraint",
+		["F"] = "File",
+		["L"] = "Line",
+		["R"] = "Routine",
+	} &default=function(c: string): string { return fmt("UnknownErrorId%s", c); } &redef;
+
+	global auth_ids: table[count] of string = {
+		[2] = "KerberosV5",
+		[3] = "CleartextPassword",
+		[5] = "MD5Password",
+		[7] = "GSSAPI",
+		[8] = "GSSAPIContinue",
+		[9] = "SSPI",
+		[10] = "SASL",
+		[11] = "SASLContinue",
+		[12] = "SASLFinal",
+	} &default=function(id: count): string { return fmt("UnknownAuthId%s", id); } &redef;
+}

scripts/base/protocols/postgresql/dpd.sig

@@ -0,0 +1,29 @@
+# Enable the analyzer if we see the SSLRequest message and a S|N reply from the server.
+signature dpd_postgresql_client_sslrequest {
+  ip-proto == tcp
+  payload /^\x00\x00\x00\x08\x04\xd2\x16\x2f/
+}
+
+signature dpd_postgresql_server_ssl_confirm {
+  requires-reverse-signature dpd_postgresql_client_sslrequest
+  payload /^[SN]/
+  enable "PostgreSQL"
+}
+
+signature dpd_postgresql_client_startup_3_x {
+  ip-proto == tcp
+  # 4 byte length, then protocol version major, minor (16bit each),
+  # then expect the "user\x00" parameter to follow. Not sure about
+  # other versions, but we likely wouldn't properly parse them anyway.
+  payload /^....\x00\x03\x00.{0,256}user\x00/
+}
+
+signature dpd_postgresql_server_any_response {
+  requires-reverse-signature dpd_postgresql_client_startup_3_x
+
+  # One byte printable message type 4 bytes length. Assumes the first
+  # server message is not larger 64k(2^16) so match on \x00\x00 after
+  # the first byte.
+  payload /^[a-zA-Z0-9]\x00\x00../
+  enable "PostgreSQL"
+}

scripts/base/protocols/postgresql/main.zeek

@@ -0,0 +1,245 @@
+##! Implements base functionality for PostgreSQL analysis.
+
+@load ./consts
+@load ./spicy-events
+
+@load base/protocols/conn/removal-hooks
+
+module PostgreSQL;
+
+export {
+	## Log stream identifier.
+	redef enum Log::ID += { LOG };
+
+	type Version: record {
+		major: count;
+		minor: count;
+	};
+
+	## Record type containing the column fields of the PostgreSQL log.
+	type Info: record {
+		## Timestamp for when the activity happened.
+		ts: time &log;
+		## Unique ID for the connection.
+		uid: string &log;
+		## The connection's 4-tuple of endpoint addresses/ports.
+		id: conn_id &log;
+
+		## The user as found in the StartupMessage.
+		user: string &optional &log;
+		## The database as found in the StartupMessage.
+		database: string &optional &log;
+		## The application name as found in the StartupMessage.
+		application_name: string &optional &log;
+
+		# The command or message from the frontend.
+		frontend: string &optional &log;
+		# Arguments for the command.
+		frontend_arg: string &optional &log;
+		# The reply from the backend.
+		backend: string &optional &log;
+		# Arguments for the reply from the backend.
+		backend_arg: string &optional &log;
+
+		# Whether the login/query was successful.
+		success: bool &optional &log;
+
+		# The number of rows returned or affectd.
+		rows: count &optional &log;
+	};
+
+	type State: record {
+		version: Version &optional;
+		user: string &optional;
+		database: string &optional;
+		application_name: string &optional;
+		rows: count &default=0;
+		errors: vector of string;
+	};
+
+	## Default hook into PostgreSQL logging.
+	global log_postgresql: event(rec: Info);
+
+	global finalize_postgresql: Conn::RemovalHook;
+
+	global ports: set[port] = { 5432/tcp } &redef;
+}
+
+redef record connection += {
+	postgresql: Info &optional;
+	postgresql_state: State &optional;
+};
+
+redef likely_server_ports += { ports };
+
+event zeek_init() {
+	Analyzer::register_for_ports(Analyzer::ANALYZER_POSTGRESQL, ports);
+
+	Log::create_stream(PostgreSQL::LOG, [$columns=Info, $ev=log_postgresql, $path="postgresql"]);
+}
+
+hook set_session(c: connection) {
+	if ( ! c?$postgresql )
+		c$postgresql = Info($ts=network_time(), $uid=c$uid, $id=c$id);
+
+	if ( ! c?$postgresql_state ) {
+		c$postgresql_state = State();
+		Conn::register_removal_hook(c, finalize_postgresql);
+	}
+}
+
+function emit_log(c: connection) {
+	if ( ! c?$postgresql )
+		return;
+
+	if ( c$postgresql_state?$user )
+		c$postgresql$user = c$postgresql_state$user;
+
+	if ( c$postgresql_state?$database )
+		c$postgresql$database = c$postgresql_state$database;
+
+	if ( c$postgresql_state?$application_name )
+		c$postgresql$application_name = c$postgresql_state$application_name;
+
+	Log::write(PostgreSQL::LOG, c$postgresql);
+	delete c$postgresql;
+}
+
+event PostgreSQL::ssl_request(c: connection) {
+	hook set_session(c);
+
+	c$postgresql$frontend = "ssl_request";
+}
+
+event PostgreSQL::ssl_reply(c: connection, b: string) {
+	hook set_session(c);
+
+	c$postgresql$backend = "ssl_reply";
+	c$postgresql$backend_arg = b;
+	c$postgresql$success = b == "S";
+
+	emit_log(c);
+}
+
+event PostgreSQL::startup_parameter(c: connection, name: string, value: string) {
+	hook set_session(c);
+
+	if ( name == "user" ) {
+		c$postgresql_state$user = value;
+	} else if ( name == "database" ) {
+		c$postgresql_state$database = value;
+	} else if ( name== "application_name" ) {
+		c$postgresql_state$application_name = value;
+	}
+}
+
+event PostgreSQL::startup_message(c: connection, major: count, minor: count) {
+	hook set_session(c);
+
+	c$postgresql_state$version = Version($major=major, $minor=minor);
+	c$postgresql$frontend = "startup";
+}
+
+event PostgreSQL::error_response_identified_field(c: connection, code: string, value: string) {
+	hook set_session(c);
+
+	local errors = c$postgresql_state$errors;
+	errors += fmt("%s=%s", error_ids[code], value);
+}
+
+event PostgreSQL::notice_response_identified_field(c: connection, code: string, value: string) {
+	hook set_session(c);
+
+	local notice = fmt("%s=%s", error_ids[code], value);
+	if ( c$postgresql?$backend_arg )
+		c$postgresql$backend_arg += "," + notice;
+	else
+		c$postgresql$backend_arg = notice;
+}
+
+event PostgreSQL::error_response(c: connection) {
+	hook set_session(c);
+
+	if ( c$postgresql?$backend )
+		c$postgresql$backend += ",error";
+	else
+		c$postgresql$backend = "error";
+
+	local errors = join_string_vec(c$postgresql_state$errors, ",");
+	c$postgresql_state$errors = vector();
+
+	if ( c$postgresql?$backend_arg )
+		c$postgresql$backend_arg += "," + errors;
+	else
+		c$postgresql$backend_arg = errors;
+
+	c$postgresql$success = F;
+
+	emit_log(c);
+}
+
+event PostgreSQL::authentication_request(c: connection, identifier: count, data: string) {
+	hook set_session(c);
+
+	if ( c$postgresql?$backend && ! ends_with(c$postgresql$backend, "auth") )
+		c$postgresql$backend += ",auth_request";
+	else
+		c$postgresql$backend = "auth_request";
+
+	if ( c$postgresql?$backend_arg )
+		c$postgresql$backend_arg += "," + auth_ids[identifier];
+	else
+		c$postgresql$backend_arg = auth_ids[identifier];
+}
+
+event PostgreSQL::authentication_ok(c: connection) {
+	hook set_session(c);
+
+	c$postgresql$backend = "auth_ok";
+	c$postgresql$success = T;
+
+	emit_log(c);
+}
+
+event PostgreSQL::terminate(c: connection) {
+	if ( c?$postgresql )
+		emit_log(c);
+
+	hook set_session(c);
+	c$postgresql$frontend = "terminate";
+	emit_log(c);
+}
+
+event PostgreSQL::simple_query(c: connection, query: string) {
+	if ( c?$postgresql )
+		emit_log(c);
+
+	hook set_session(c);
+
+	c$postgresql$frontend = "simple_query";
+	c$postgresql$frontend_arg = query;
+	c$postgresql_state$rows = 0;
+}
+
+event PostgreSQL::data_row(c: connection, column_values: count) {
+	hook set_session(c);
+
+	++c$postgresql_state$rows;
+}
+
+event PostgreSQL::ready_for_query(c: connection, transaction_status: string) {
+	# Log a query (if there was one).
+	if ( ! c?$postgresql )
+		return;
+
+	# If no one said otherwise, the last action was successful.
+	if ( ! c$postgresql?$success )
+		c$postgresql$success = transaction_status == "I" || transaction_status == "T";
+
+	c$postgresql$rows = c$postgresql_state$rows;
+	emit_log(c);
+}
+
+hook finalize_postgresql(c: connection) &priority=-5 {
+	emit_log(c);
+}

scripts/base/protocols/postgresql/spicy-events.zeek

@@ -0,0 +1,147 @@
+##! Events generated by the PostgreSQL analyzer.
+
+## Event generated for frontend SSLRequest messages.
+##
+## c: The connection.
+global PostgreSQL::ssl_request: event(c: connection);
+
+## Event generated for backend SSL reply.
+##
+## c: The connection.
+##
+## data: The server's reply: S for secure, N for unencrypted.
+global PostgreSQL::ssl_reply: event(c: connection, data: string);
+
+## Event generated for backend authentication requests.
+##
+## c: The connection.
+##
+## identifier: The identifier in the request.
+##
+## data: The request data, if any.
+##
+## .. zeek:see:: PostgreSQL::authentication_response
+## .. zeek:see:: PostgreSQL::authentication_ok
+global PostgreSQL::authentication_request: event(c: connection, identifier: count, data: string);
+
+## Event generated for backend authentication requests indicating successful
+## authentication.
+##
+## c: The connection.
+##
+## .. zeek:see:: PostgreSQL::authentication_request
+## .. zeek:see:: PostgreSQL::authentication_response
+global PostgreSQL::authentication_ok: event(c: connection);
+
+## Event generated for frontend authentication responses.
+##
+## c: The connection.
+##
+## data: The response data, if any.
+##
+## .. zeek:see:: PostgreSQL::authentication_request
+## .. zeek:see:: PostgreSQL::authentication_ok
+global PostgreSQL::authentication_response: event(c: connection, data: string);
+
+
+## Event generated for every parameter in a StartupMessage.
+##
+## c: The connection.
+##
+## name: The name of the parameter.
+##
+## value: The value of the parameter.
+global PostgreSQL::startup_parameter: event(c: connection, name: string, value: string);
+
+## Event generated for a StartupMessage.
+##
+## c: The connection.
+##
+## major: The major protocol version.
+##
+## minor: The minor protocol version.
+global PostgreSQL::startup_message: event(c: connection, major: count, minor: count);
+
+## Event generated for every backed ReadyForQuery message.
+##
+## c: The connection.
+##
+## transaction_status: I (idle), T (in transaction block), E (error).
+global PostgreSQL::ready_for_query: event(c: connection, transaction_status: string);
+
+## Event generated for every frontend SimpleQuery message.
+##
+## c: The connection.
+##
+## query: The query string.
+global PostgreSQL::simple_query: event(c: connection, query: string);
+
+## Event generated for identified field within an ErrorResponse.
+##
+## c: The connection.
+##
+## code: The code (https://www.postgresql.org/docs/current/protocol-error-fields.html)
+##
+## value: The field value.
+##
+## .. zeek:see:: PostgreSQL::error_response
+global PostgreSQL::error_response_identified_field: event(c: connection, code: string, value: string);
+
+## Event generated for a ErrorResponse.
+##
+## c: The connection.
+##
+## .. zeek:see:: PostgreSQL::error_response_identified_field
+global PostgreSQL::error_response: event(c: connection);
+
+## Event generated for identified field within a NoticeResponse.
+##
+## c: The connection.
+##
+## code: The code (https://www.postgresql.org/docs/current/protocol-error-fields.html)
+##
+## value: The field value.
+##
+## .. zeek:see:: PostgreSQL::notice_response
+global PostgreSQL::notice_response_identified_field: event(c: connection, code: string, value: string);
+
+## Event generated for a NoticeResponse.
+##
+## c: The connection.
+##
+## .. zeek:see:: PostgreSQL::notice_response_identified_field
+global PostgreSQL::notice_response: event(c: connection);
+
+## Event generated for every backend DataRow message.
+##
+## c: The connection.
+##
+## column_values: The number of columns in this row.
+global PostgreSQL::data_row: event(c: connection, column_values: count);
+
+## Event generated for backend runtime parameter status reports.
+##
+## c: The connection.
+##
+## name: The name of the runtime parameter.
+##
+## value: The current value of the parameter.
+##
+global PostgreSQL::parameter_status: event(c: connection, name: string, value: string);
+
+## Generated for a BackendKeyData message for cancellation.
+##
+## c: The connection.
+##
+## process_id: The process ID of the backend.
+##
+## secret_key: The secret key of the backend.
+global PostgreSQL::backend_key_data: event(c: connection, process_id: count, secret_key: count);
+
+## Event generated For a frontend Terminate message.
+##
+## c: The connection.
+global PostgreSQL::terminate: event(c: connection);
+
+## Event generated for not implemented messages.
+global PostgreSQL::not_implemented: event(c: connection, is_orig: bool, typ: string, chunk: string);