HTTP: Support HTTP::upgrade_content_type_analyzers

This allows to select the Upgrade analyzer based on the tuple
of [Upgrade, Content-Type] as motivated by the Docker API trace
that sets a very specific Content-Type in the HTTP response.

Closes #4068

scripts/base/init-bare.zeek

@@ -452,6 +452,15 @@ export {
 
 module HTTP;
 export {
+	## Lookup table for Upgrade analyzers indexed by the value of the
+	## Upgrade header as well as the value of the Content-Type header.
+	## First, a case sensitive lookup of both values is done using the
+	## server's Upgrade and Content-Type header. If no match is found,
+	## the all lower-case values are used. If there's still no match,
+	## Zeek continues the lookup in :zeek:see:`HTTP::upgrade_analyzers`,
+	## considering just the server's Upgrade header.
+	const upgrade_content_type_analyzers: table[string, string] of Analyzer::Tag &redef;
+
 	## Lookup table for Upgrade analyzers. First, a case sensitive lookup
 	## is done using the server's Upgrade header. If no match is found,
 	## the all lower-case value is used. If there's still no match Zeek

src/analyzer/protocol/http/HTTP.cc

@@ -17,6 +17,10 @@
 #include "zeek/analyzer/protocol/mime/MIME.h"
 #include "zeek/file_analysis/Manager.h"
 
+namespace zeek {
+std::string obj_desc_short(const Obj* o);
+}
+
 namespace zeek::analyzer::http {
 
 const bool DEBUG_http = false;
@@ -1365,6 +1369,63 @@ void HTTP_Analyzer::ReplyMade(bool interrupted, const char* msg) {
         reply_state = EXPECT_REPLY_LINE;
 }
 
+// Look for an analyzer tag in the HTTP::upgrade_content_type_analyzers or HTTP::upgrade_analyzer tables.
+static EnumValPtr LookupUpgradeAnalyzer(const std::string& proto, const std::string& content_type) {
+    static const auto& upgrade_analyzers_content_type = id::find_val<TableVal>("HTTP::upgrade_content_type_analyzers");
+    static const auto& lv_type = upgrade_analyzers_content_type->GetType<TableType>()->GetIndices();
+    static const auto& upgrade_analyzers = id::find_val<TableVal>("HTTP::upgrade_analyzers");
+
+    ValPtr proto_val = make_intrusive<StringVal>(proto);
+    ValPtr proto_val_lower;
+    ValPtr result;
+
+    // If we have a content_type header and there's entries in the corresponding
+    // table, lookup the case-sensitive [Upgrade, Content-Type] version first. If
+    // there's no match, use the lower-case version next.
+    if ( ! content_type.empty() && upgrade_analyzers_content_type->Size() > 0 ) {
+        auto make_key = [](const auto& p, const auto c) -> ListValPtr {
+            return zeek::make_intrusive<ListVal>(lv_type, std::vector{p, c});
+        };
+
+        ValPtr content_type_val = make_intrusive<StringVal>(content_type);
+        auto lv_key = make_key(proto_val, content_type_val);
+
+        result = upgrade_analyzers_content_type->Find(lv_key);
+        if ( ! result ) {
+            // No match. Lower the values and try again.
+            proto_val_lower = zeek::make_intrusive<zeek::StringVal>(util::strtolower(proto));
+            ValPtr content_type_val_lower = zeek::make_intrusive<zeek::StringVal>(util::strtolower(content_type));
+            lv_key = make_key(proto_val_lower, content_type_val_lower);
+            result = upgrade_analyzers_content_type->Find(lv_key);
+        }
+
+        if ( result ) {
+            DBG_LOG(DBG_ANALYZER, "Found %s in table HTTP::upgrade_analyzers_content_type for key %s",
+                    zeek::obj_desc_short(result.get()).c_str(), zeek::obj_desc_short(lv_key.get()).c_str());
+
+            return cast_intrusive<EnumVal>(result);
+        }
+    }
+
+    // We would have returned if upgrade_content_type_analyzers had a matching
+    // entry. Continue with the HTTP::upgrade_analyzer table now, using just
+    // the Upgrade protocol.
+    result = upgrade_analyzers->Find(proto_val);
+    if ( ! result ) {
+        // No match, try the lower-case value, possibly re-using the
+        // string done earlier.
+        proto_val = proto_val_lower ? proto_val_lower : zeek::make_intrusive<zeek::StringVal>(util::strtolower(proto));
+        result = upgrade_analyzers->Find(proto_val);
+    }
+
+    if ( result ) {
+        DBG_LOG(DBG_ANALYZER, "Found %s in HTTP::upgrade_analyzers for protocol value %s",
+                zeek::obj_desc_short(result.get()).c_str(), proto_val->AsStringVal()->CheckString());
+    }
+
+    return cast_intrusive<EnumVal>(result);
+}
+
 void HTTP_Analyzer::HTTP_Upgrade() {
     // Upgraded connection that switches immediately - e.g. websocket.
 
@@ -1383,23 +1444,7 @@ void HTTP_Analyzer::HTTP_Upgrade() {
         content_line_resp->SetPlainDelivery(remaining_in_content_line);
     }
 
-    // Lookup an analyzer tag in the HTTP::upgrade_analyzer table.
-    static const auto& upgrade_analyzers = id::find_val<TableVal>("HTTP::upgrade_analyzers");
-
-    auto upgrade_protocol_val = make_intrusive<StringVal>(upgrade_protocol);
-    auto v = upgrade_analyzers->Find(upgrade_protocol_val);
-    if ( ! v ) {
-        // If not found, try the all lower version, too.
-        auto lower_upgrade_protocol = util::strtolower(upgrade_protocol);
-        upgrade_protocol_val = make_intrusive<StringVal>(lower_upgrade_protocol);
-        v = upgrade_analyzers->Find(upgrade_protocol_val);
-    }
-
-    if ( v ) {
-        auto analyzer_tag_val = cast_intrusive<EnumVal>(v);
-        DBG_LOG(DBG_ANALYZER, "Found %s in HTTP::upgrade_analyzers for %s",
-                analyzer_tag_val->GetType<EnumType>()->Lookup(analyzer_tag_val->AsEnum()),
-                upgrade_protocol_val->CheckString());
+    if ( auto analyzer_tag_val = LookupUpgradeAnalyzer(upgrade_protocol, server_content_type) ) {
         auto analyzer_tag = analyzer_mgr->GetComponentTag(analyzer_tag_val.get());
         auto* analyzer = analyzer_mgr->InstantiateAnalyzer(std::move(analyzer_tag), Conn());
         if ( analyzer ) {
@@ -1422,8 +1467,9 @@ void HTTP_Analyzer::HTTP_Upgrade() {
         }
     }
     else {
-        DBG_LOG(DBG_ANALYZER, "No mapping for %s in HTTP::upgrade_analyzers, using PIA instead",
-                upgrade_protocol.c_str());
+        DBG_LOG(DBG_ANALYZER,
+                "No mapping for %s, %s in upgrade_analyzers_content_type or upgrade_analyzers, using PIA instead",
+                upgrade_protocol.c_str(), server_content_type.c_str());
         pia = new analyzer::pia::PIA_TCP(Conn());
         if ( AddChildAnalyzer(pia) ) {
             pia->FirstPacket(true, TransportProto::TRANSPORT_TCP);
@@ -1547,6 +1593,9 @@ void HTTP_Analyzer::HTTP_Header(bool is_orig, analyzer::mime::MIME_Header* h) {
     if ( ! is_orig && analyzer::mime::istrequal(h->get_name(), "upgrade") )
         upgrade_protocol.assign(h->get_value_token().data, h->get_value_token().length);
 
+    if ( ! is_orig && analyzer::mime::istrequal(h->get_name(), "content-type") )
+        server_content_type.assign(h->get_value().data, h->get_value().length);
+
     if ( http_header ) {
         zeek::detail::Rule::PatternType rule =
             is_orig ? zeek::detail::Rule::HTTP_REQUEST_HEADER : zeek::detail::Rule::HTTP_REPLY_HEADER;

src/analyzer/protocol/http/HTTP.h

@@ -251,6 +251,8 @@ protected:
     // set to the protocol string when encountering an "upgrade" header
     // in a reply.
     std::string upgrade_protocol;
+    // set to the server's "content type" header
+    std::string server_content_type;
 
     StringValPtr request_method;
 

testing/btest/Baseline/scripts.base.protocols.http.upgrade-to-tcp-content-type-2/out

@@ -0,0 +1,3 @@
+### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63.
+ClEkJM2Vm5giqnMf4h, Connection upgraded to tcp
+analyzer_violation_info, Analyzer::ANALYZER_HTTP, not a http reply line, ClEkJM2Vm5giqnMf4h, 13

testing/btest/Baseline/scripts.base.protocols.http.upgrade-to-tcp-content-type-3/out

@@ -0,0 +1,5 @@
+### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63.
+ClEkJM2Vm5giqnMf4h, Connection upgraded to tcp
+ClEkJM2Vm5giqnMf4h, stream_deliver, / # 
+ClEkJM2Vm5giqnMf4h, stream_deliver, \x0d/ # \x1b[J
+ClEkJM2Vm5giqnMf4h, stream_deliver, l

testing/btest/Baseline/scripts.base.protocols.http.upgrade-to-tcp-content-type-4/out

@@ -0,0 +1,5 @@
+### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63.
+ClEkJM2Vm5giqnMf4h, Connection upgraded to tcp
+ClEkJM2Vm5giqnMf4h, stream_deliver, / # 
+ClEkJM2Vm5giqnMf4h, stream_deliver, \x0d/ # \x1b[J
+ClEkJM2Vm5giqnMf4h, stream_deliver, l

testing/btest/Baseline/scripts.base.protocols.http.upgrade-to-tcp-content-type/out

@@ -0,0 +1,5 @@
+### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63.
+ClEkJM2Vm5giqnMf4h, Connection upgraded to tcp
+ClEkJM2Vm5giqnMf4h, stream_deliver, / # 
+ClEkJM2Vm5giqnMf4h, stream_deliver, \x0d/ # \x1b[J
+ClEkJM2Vm5giqnMf4h, stream_deliver, l

testing/btest/scripts/base/protocols/http/upgrade-to-tcp-content-type.zeek

@@ -0,0 +1,68 @@
+# @TEST-EXEC: zeek -b -C -r $TRACES/http/docker-http-upgrade.pcap ./common.zeek %INPUT >out
+# @TEST-EXEC: btest-diff out
+
+@load ./common.zeek
+redef HTTP::upgrade_content_type_analyzers += {
+	["tcp", "application/vnd.docker.raw-stream"] = Analyzer::ANALYZER_STREAM_EVENT,
+};
+
+# @TEST-START-NEXT
+# triggers a HTTP violation because upgrade_analyzers_content_type is
+# preferred and it's using HTTP and that's not the right analyzer.
+redef HTTP::upgrade_content_type_analyzers += {
+	["tcp", "application/vnd.docker.raw-stream"] = Analyzer::ANALYZER_HTTP,
+};
+
+redef HTTP::upgrade_analyzers += {
+	["tcp"] =  Analyzer::ANALYZER_STREAM_EVENT,
+};
+
+# @TEST-START-NEXT
+# triggers no violation because upgrade_analyzers_content_type is
+# preferred - the reverse of the above test.
+redef HTTP::upgrade_content_type_analyzers += {
+	["tcp", "application/vnd.docker.raw-stream"] = Analyzer::ANALYZER_STREAM_EVENT,
+};
+
+redef HTTP::upgrade_analyzers += {
+	["tcp"] =  Analyzer::ANALYZER_HTTP,
+};
+
+# @TEST-START-NEXT
+# this falls back to upgrade_analyzers and enables the stream event analyzer
+# as the content type does not match: cooked in table, raw in trace.
+redef HTTP::upgrade_content_type_analyzers += {
+	["tcp", "application/vnd.docker.cooked-stream"] = Analyzer::ANALYZER_HTTP,
+
+	# And nope, this does not work!
+	["tcp", "application/*"] = Analyzer::ANALYZER_HTTP,
+};
+
+redef HTTP::upgrade_analyzers += {
+	["tcp"] =  Analyzer::ANALYZER_STREAM_EVENT,
+};
+
+# @TEST-START-FILE common.zeek
+@load base/protocols/http
+
+event http_connection_upgrade(c: connection, protocol: string)
+	{
+	print c$uid, fmt("Connection upgraded to %s", protocol);
+	}
+
+global deliveries = 0;
+
+event stream_deliver(c: connection, is_orig: bool, data: string)
+	{
+	++deliveries;
+	print c$uid, "stream_deliver", data[:32];
+
+	if ( deliveries == 3 )
+		disable_analyzer(c$id, current_analyzer());
+	}
+
+event analyzer_violation_info(tag: AllAnalyzers::Tag, info: AnalyzerViolationInfo)
+	{
+	print "analyzer_violation_info", tag, info$reason, info$c$uid, info$aid;
+	}
+# @TEST-END-FILE