diff --git a/CHANGES b/CHANGES index d097e91568..0a465527bf 100644 --- a/CHANGES +++ b/CHANGES @@ -1,4 +1,11 @@ +3.1.0-dev.18 | 2019-08-09 10:43:28 -0700 + + * GH-419: improve multi-protocol logging in known_services.log (Mauro Palumbo) + + Previously, when multiple protocols were detected on a given addr/port + pair, not all protocols were always logged. + 3.1.0-dev.7 | 2019-08-09 09:56:06 -0700 * Remove empty services from known_services.log (Mauro Palumbo) diff --git a/NEWS b/NEWS index 7951f8bde4..b6c072e168 100644 --- a/NEWS +++ b/NEWS @@ -3,6 +3,25 @@ This document summarizes the most important changes in the current Zeek release. For an exhaustive list of changes, see the ``CHANGES`` file (note that submodules, such as Broker, come with their own ``CHANGES``.) +Zeek 3.1.0 +========== + +New Functionality +----------------- + +Changed Functionality +--------------------- + +- The key type of ``Known::service_store`` has changed to + ``Known::AddrPortServTriplet`` and ``Known::services`` is now a table + instead of just a set. + +Removed Functionality +--------------------- + +Deprecated Functionality +------------------------ + Zeek 3.0.0 ========== diff --git a/VERSION b/VERSION index 46e9218943..440640dcef 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -3.1.0-dev.7 +3.1.0-dev.18 diff --git a/doc b/doc index 1cd9867ca6..56019a5297 160000 --- a/doc +++ b/doc @@ -1 +1 @@ -Subproject commit 1cd9867ca6c8ab73db1f2b665ec69332fe88734c +Subproject commit 56019a5297cd5d167e0febc6b0b33a32a9dbe96e diff --git a/scripts/policy/protocols/conn/known-services.zeek b/scripts/policy/protocols/conn/known-services.zeek index 7feea978d2..1fe40e1cbc 100644 --- a/scripts/policy/protocols/conn/known-services.zeek +++ b/scripts/policy/protocols/conn/known-services.zeek @@ -37,13 +37,14 @@ export { ## See :zeek:type:`Host` for possible choices. option service_tracking = LOCAL_HOSTS; - type AddrPortPair: record { + type AddrPortServTriplet: record { host: addr; p: port; + serv: string; }; ## Holds the set of all known services. Keys in the store are - ## :zeek:type:`Known::AddrPortPair` and their associated value is + ## :zeek:type:`Known::AddrPortServTriplet` and their associated value is ## always the boolean value of "true". global service_store: Cluster::StoreInfo; @@ -62,11 +63,11 @@ export { ## of duplicates, but can also be inspected by other scripts for ## different purposes. ## - ## In cluster operation, this set is uniformly distributed across + ## In cluster operation, this table is uniformly distributed across ## proxy nodes. ## - ## This set is automatically populated and shouldn't be directly modified. - global services: set[addr, port] &create_expire=1day; + ## This table is automatically populated and shouldn't be directly modified. + global services: table[addr, port] of set[string] &create_expire=1day; ## Event that can be handled to access the :zeek:type:`Known::ServicesInfo` ## record as it is sent on to the logging framework. @@ -79,6 +80,20 @@ redef record connection += { known_services_done: bool &default=F; }; +# Check if the triplet (host,port_num,service) is already in Known::services +function check(info: ServicesInfo) : bool + { + if ( [info$host, info$port_num] !in Known::services ) + return F; + + for ( s in info$service ) + { + if ( s !in Known::services[info$host, info$port_num] ) + return F; + } + + return T; + } event zeek_init() { @@ -89,28 +104,34 @@ event zeek_init() } event service_info_commit(info: ServicesInfo) - { if ( ! Known::use_service_store ) return; - local key = AddrPortPair($host = info$host, $p = info$port_num); + local tempservs = info$service; - when ( local r = Broker::put_unique(Known::service_store$store, key, - T, Known::service_store_expiry) ) + for ( s in tempservs ) { - if ( r$status == Broker::SUCCESS ) + local key = AddrPortServTriplet($host = info$host, $p = info$port_num, $serv = s); + + when ( local r = Broker::put_unique(Known::service_store$store, key, + T, Known::service_store_expiry) ) { - if ( r$result as bool ) - Log::write(Known::SERVICES_LOG, info); + if ( r$status == Broker::SUCCESS ) + { + if ( r$result as bool ) { + info$service = set(s); # log one service at the time if multiservice + Log::write(Known::SERVICES_LOG, info); + } + } + else + Reporter::error(fmt("%s: data store put_unique failure", + Known::service_store_name)); + } + timeout Known::service_store_timeout + { + Log::write(Known::SERVICES_LOG, info); } - else - Reporter::error(fmt("%s: data store put_unique failure", - Known::service_store_name)); - } - timeout Known::service_store_timeout - { - Log::write(Known::SERVICES_LOG, info); } } @@ -119,14 +140,32 @@ event known_service_add(info: ServicesInfo) if ( Known::use_service_store ) return; - if ( [info$host, info$port_num] in Known::services ) + if ( check(info) ) return; - add Known::services[info$host, info$port_num]; + if ( [info$host, info$port_num] !in Known::services ) + Known::services[info$host, info$port_num] = set(); + + # service to log can be a subset of info$service if some were already seen + local info_to_log: ServicesInfo; + info_to_log$ts = info$ts; + info_to_log$host = info$host; + info_to_log$port_num = info$port_num; + info_to_log$port_proto = info$port_proto; + info_to_log$service = set(); + + for ( s in info$service ) + { + if ( s !in Known::services[info$host, info$port_num] ) + { + add Known::services[info$host, info$port_num][s]; + add info_to_log$service[s]; + } + } @if ( ! Cluster::is_enabled() || Cluster::local_node_type() == Cluster::PROXY ) - Log::write(Known::SERVICES_LOG, info); + Log::write(Known::SERVICES_LOG, info_to_log); @endif } @@ -139,7 +178,7 @@ event Cluster::node_up(name: string, id: string) return; # Drop local suppression cache on workers to force HRW key repartitioning. - Known::services = set(); + Known::services = table(); } event Cluster::node_down(name: string, id: string) @@ -151,7 +190,7 @@ event Cluster::node_down(name: string, id: string) return; # Drop local suppression cache on workers to force HRW key repartitioning. - Known::services = set(); + Known::services = table(); } event service_info_commit(info: ServicesInfo) @@ -159,7 +198,7 @@ event service_info_commit(info: ServicesInfo) if ( Known::use_service_store ) return; - if ( [info$host, info$port_num] in Known::services ) + if ( check(info) ) return; local key = cat(info$host, info$port_num); @@ -186,10 +225,16 @@ function known_services_done(c: connection) return; } + # Drop services starting with "-" (confirmed-but-then-violated protocol) + local tempservs: set[string]; + for (s in c$service) + if ( s[0] != "-" ) + add tempservs[s]; + local info = ServicesInfo($ts = network_time(), $host = id$resp_h, $port_num = id$resp_p, $port_proto = get_port_transport_proto(id$resp_p), - $service = c$service); + $service = tempservs); # If no protocol was detected, wait a short time before attempting to log # in case a protocol is detected on another connection. diff --git a/testing/btest/Baseline/scripts.policy.protocols.conn.known-services-multi/known_services.log b/testing/btest/Baseline/scripts.policy.protocols.conn.known-services-multi/known_services.log new file mode 100644 index 0000000000..6a4f11cfab --- /dev/null +++ b/testing/btest/Baseline/scripts.policy.protocols.conn.known-services-multi/known_services.log @@ -0,0 +1,11 @@ +#separator \x09 +#set_separator , +#empty_field (empty) +#unset_field - +#path known_services +#open 2019-08-09-17-38-17 +#fields ts host port_num port_proto service +#types time addr port enum set[string] +1455718916.856316 192.168.2.230 443 tcp SSH +1455718922.796688 192.168.2.230 443 tcp SSL +#close 2019-08-09-17-38-18 diff --git a/testing/btest/Traces/ssl-and-ssh-using-sslh.trace b/testing/btest/Traces/ssl-and-ssh-using-sslh.trace new file mode 100644 index 0000000000..6a608a2168 Binary files /dev/null and b/testing/btest/Traces/ssl-and-ssh-using-sslh.trace differ diff --git a/testing/btest/scripts/policy/protocols/conn/known-services-multi.zeek b/testing/btest/scripts/policy/protocols/conn/known-services-multi.zeek new file mode 100644 index 0000000000..649dcf03a2 --- /dev/null +++ b/testing/btest/scripts/policy/protocols/conn/known-services-multi.zeek @@ -0,0 +1,7 @@ +# A test case for when more than a single service is detected for a given +# (addr, port) pair. + +# @TEST-EXEC: zeek -C -r $TRACES/ssl-and-ssh-using-sslh.trace %INPUT "Known::service_tracking = ALL_HOSTS" +# @TEST-EXEC: btest-diff known_services.log + +@load protocols/conn/known-services diff --git a/testing/external/commit-hash.zeek-testing b/testing/external/commit-hash.zeek-testing index 4c877406cd..e929575c73 100644 --- a/testing/external/commit-hash.zeek-testing +++ b/testing/external/commit-hash.zeek-testing @@ -1 +1 @@ -5e5a5e8dbb94215a7ca1def810f4bbe0322bc72e +dc6a8f1de9f3b298406051282abcaa6e8f198695 diff --git a/testing/external/commit-hash.zeek-testing-private b/testing/external/commit-hash.zeek-testing-private index 5208dd575f..206a369bb1 100644 --- a/testing/external/commit-hash.zeek-testing-private +++ b/testing/external/commit-hash.zeek-testing-private @@ -1 +1 @@ -b7cf5aa8224fb39baf7497d187f48165fad050da +fdfcdffd464fd2114be03feacfd075d73a8b1ef9