Merge remote-tracking branch 'origin/master' into topic/dnthayer/doc-fixes-for-2.6

CHANGES

@@ -1,4 +1,36 @@
 
+2.5-792 | 2018-08-01 12:15:31 -0500
+
+  * fix NTLM NegotiateFlags field offsets (Jeffrey Bencteux)
+
+2.5-790 | 2018-08-01 11:25:27 -0500
+
+  * Fix --with-binpac configure option (Jon Siwek, Corelight)
+
+  * Update CAF-finding logic (Jon Siwek, Corelight)
+
+2.5-787 | 2018-07-31 16:50:55 -0500
+
+  * Add Cisco FabricPath support (Damani Wade, Corelight)
+
+  * Replace GeoIP Legacy DB support with MaxMind DB support
+    (Jonathan Perkins, Corelight)
+
+2.5-782 | 2018-07-31 11:53:22 +0200
+
+  * Update install instructions for OpenSSL 1.1 compat (Jon Siwek, Corelight)
+
+  * Remove requestorName parameter of ocsp_request event
+
+    This field isn't publicly available via the OpenSSL 1.1 API, not used
+    in the base scripts, and has no example in the test suit, so removing
+    it is simpler than trying to support manually parsing it out of the
+    raw data. (Jon Siwek, Corelight)
+
+  * Adjust x509 unit tests to work around OpenSSL 1.0 vs. 1.1 differences (Jon Siwek, Corelight)
+
+  * Fixes for OpenSSL 1.1 support (Jon Siwek, Corelight)
+
 2.5-775 | 2018-07-24 16:39:34 -0500
 
   * Add broker/binpac/caf dirs to bro-config script (Jon Siwek, Corelight)

CMakeLists.txt

@@ -95,14 +95,15 @@ FindRequiredPackage(OpenSSL)
 FindRequiredPackage(BIND)
 FindRequiredPackage(ZLIB)
 
-find_package(CAF COMPONENTS core io)
+find_package(CAF COMPONENTS core io openssl)
 if (CAF_FOUND)
+    # e.g. if not using embedded CAF, then need to know where to look
+    # for CAF headers since that may differ from where Broker headers
+    # are found (and including a Broker header may pull in CAF headers).
 	include_directories(BEFORE ${CAF_INCLUDE_DIRS})
-else ()
-	list(APPEND MISSING_PREREQ_DESCS CAF)
 endif ()
 
-if (NOT BinPAC_ROOT_DIR AND
+if (NOT BINPAC_EXE_PATH AND
     EXISTS ${CMAKE_CURRENT_SOURCE_DIR}/aux/binpac/CMakeLists.txt)
     add_subdirectory(aux/binpac)
 endif ()
@@ -140,11 +141,11 @@ include_directories(BEFORE
 # Optional Dependencies
 
 set(USE_GEOIP false)
-find_package(LibGeoIP)
+find_package(LibMMDB)
-if (LIBGEOIP_FOUND)
+if (LibMMDB_FOUND)
     set(USE_GEOIP true)
-    include_directories(BEFORE ${LibGeoIP_INCLUDE_DIR})
+    include_directories(BEFORE ${LibMMDB_INCLUDE_DIR})
-    list(APPEND OPTLIBS ${LibGeoIP_LIBRARY})
+    list(APPEND OPTLIBS ${LibMMDB_LIBRARY})
 endif ()
 
 set(USE_KRB5 false)
@@ -224,6 +225,10 @@ include(CheckNameserCompat)
 include(GetArchitecture)
 include(RequireCXX11)
 
+if ( (OPENSSL_VERSION VERSION_EQUAL "1.1.0") OR (OPENSSL_VERSION VERSION_GREATER "1.1.0") )
+  set(BRO_HAVE_OPENSSL_1_1 true CACHE INTERNAL "" FORCE)
+endif()
+
 # Tell the plugin code that we're building as part of the main tree.
 set(BRO_PLUGIN_INTERNAL_BUILD true CACHE INTERNAL "" FORCE)
 
@@ -323,7 +328,7 @@ message(
     "\nBroctl:            ${INSTALL_BROCTL}"
     "\nAux. Tools:        ${INSTALL_AUX_TOOLS}"
     "\n"
-    "\nGeoIP:             ${USE_GEOIP}"
+    "\nlibmaxminddb:      ${USE_GEOIP}"
     "\nKerberos:          ${USE_KRB5}"
     "\ngperftools found:  ${HAVE_PERFTOOLS}"
     "\n        tcmalloc:  ${USE_PERFTOOLS_TCMALLOC}"

NEWS

@@ -281,6 +281,8 @@ New Functionality
 
 - Bro now supports PPPoE over QinQ.
 
+- Bro now supports OpenSSL 1.1.
+
 Changed Functionality
 ---------------------
 
@@ -360,6 +362,15 @@ Changed Functionality
 - The string_to_pattern() built-in (and the now-deprecated merge_pattern()
   built-in) is no longer restricted to only be called at initialization time.
 
+- GeoIP Legacy Database support has been replaced with GeoIP2 MaxMind DB
+  format support.
+
+  - This updates the "lookup_location" and "lookup_asn" BIFs to use
+    libmaxminddb.  The motivation for this is that MaxMind is discontinuing
+    GeoLite Legacy databases: no updates after April 1, 2018, no downloads
+    after January 2, 2019.  It's also noted that all GeoIP Legacy databases
+    may be discontinued as they are superseded by GeoIP2.
+
 Removed Functionality
 ---------------------
 
@@ -372,6 +383,8 @@ Removed Functionality
 - BroControl: The option 'IPv6Comm' and 'ZoneID' options are no longer
   available (though Broker should be able to handle IPv6 automatically).
 
+- The "ocsp_request" event no longer has "requestorName" parameter.
+
 Deprecated Functionality
 ------------------------
 

VERSION

@@ -1 +1 @@
-2.5-775
+2.5-792

aux/bifcl

@@ -1 +1 @@
-Subproject commit 7fffb6bcf1ea78331ca2fc0e7d16642ec2463ca0
+Subproject commit f648ad79b20baba4f80259d059044ae78d56d7c4

aux/binpac

@@ -1 +1 @@
-Subproject commit 4a06b4cf3ce2112426c306cd358753793167bbeb
+Subproject commit 7e6b47ee90c5b6ab80b0e6f93d5cf835fd86ce4e

aux/bro-aux

@@ -1 +1 @@
-Subproject commit c633a852ec0bb983e74c0f47b73d03a30b7f1259
+Subproject commit 1b27ec4c24bb13443f1a5e8f4249ff4e20e06dd1

aux/broccoli

@@ -1 +1 @@
-Subproject commit a4a859a7a87363167239205bd07fedced0c584a3
+Subproject commit 80a4aa68927c2f60ece1200268106edc27f50338

aux/broctl

@@ -1 +1 @@
-Subproject commit 206fc8254a4049dfad420d5571704de4a06444bd
+Subproject commit d900149ef6e2f744599a9575b67fb7155953bd4a

aux/broker

@@ -1 +1 @@
-Subproject commit b5508f387bb6c24c95f23de479184869b9836769
+Subproject commit 488dc806d0f777dcdee04f3794f04121ded08b8e

bro-config.h.in

@@ -111,12 +111,6 @@
 /* Define if KRB5 is available */
 #cmakedefine USE_KRB5
 
-/* Whether the found GeoIP API supports IPv6 Country Edition */
-#cmakedefine HAVE_GEOIP_COUNTRY_EDITION_V6
-
-/* Whether the found GeoIP API supports IPv6 City Edition */
-#cmakedefine HAVE_GEOIP_CITY_EDITION_REV0_V6
-
 /* Use Google's perftools */
 #cmakedefine USE_PERFTOOLS_DEBUG
 

cmake

@@ -1 +1 @@
-Subproject commit c4c2d6eeb7609836255abb054b91c4039bb7e276
+Subproject commit ec66fd8fbd81fd8b25ced0214016f4c89604a15c

configure

@@ -72,7 +72,7 @@ Usage: $0 [OPTION]... [VAR=VALUE]...
                            (a required Broker dependency)
 
   Optional Packages in Non-Standard Locations:
-    --with-geoip=PATH      path to the libGeoIP install root
+    --with-geoip=PATH      path to the libmaxminddb install root
     --with-krb5=PATH       path to krb5 install root
     --with-perftools=PATH  path to Google Perftools install root
     --with-jemalloc=PATH   path to jemalloc install root
@@ -241,6 +241,7 @@ while [ $# -ne 0 ]; do
             ;;
         --with-binpac=*)
             append_cache_entry BINPAC_EXE_PATH     PATH $optarg
+            append_cache_entry BinPAC_ROOT_DIR     PATH "$(dirname $optarg)/.."
             ;;
         --with-bifcl=*)
             append_cache_entry BIFCL_EXE_PATH      PATH   $optarg
@@ -252,7 +253,7 @@ while [ $# -ne 0 ]; do
             append_cache_entry BISON_EXECUTABLE PATH $optarg
             ;;
         --with-geoip=*)
-            append_cache_entry LibGeoIP_ROOT_DIR PATH $optarg
+            append_cache_entry LibMMDB_ROOT_DIR PATH $optarg
             ;;
         --with-krb5=*)
             append_cache_entry LibKrb5_ROOT_DIR PATH $optarg

doc/frameworks/geoip.rst

@@ -8,11 +8,13 @@ GeoLocation
 .. rst-class:: opening
 
     During the process of creating policy scripts the need may arise
-    to find the geographic location for an IP address. Bro has support
+    to find the geographic location for an IP address. Bro had support
     for the `GeoIP library <http://www.maxmind.com/app/c>`__ at the
-    policy script level beginning with release 1.3 to account for this
+    policy script level from release 1.3 to 2.5.X to account for this
-    need.  To use this functionality, you need to first install the libGeoIP
+    need.  Starting with release 2.6 GeoIP support requires `libmaxminddb
-    software, and then install the GeoLite city database before building
+    <https://github.com/maxmind/libmaxminddb/releases>`__.
+    To use this functionality, you need to first install the libmaxminddb
+    software, and then install the GeoLite2 city database before building
     Bro.
 
 .. contents::
@@ -20,85 +22,91 @@ GeoLocation
 Install libGeoIP
 ----------------
 
-Before building Bro, you need to install libGeoIP.
+Before building Bro, you need to install libmaxminddb.
 
 * FreeBSD:
 
   .. console::
 
-      sudo pkg install GeoIP
+      sudo pkg install libmaxminddb
 
 * RPM/RedHat-based Linux:
 
   .. console::
 
-      sudo yum install GeoIP-devel
+      sudo yum install libmaxminddb-devel
 
 * DEB/Debian-based Linux:
 
   .. console::
 
-      sudo apt-get install libgeoip-dev
+      sudo apt-get install libmaxminddb-dev
 
 * Mac OS X:
 
   You need to install from your preferred package management system
   (e.g. MacPorts, Fink, or Homebrew).  The name of the package that you need
-  may be libgeoip, geoip, or geoip-dev, depending on which package management
+  may be libmaxminddb, maxminddb, or libmaxminddb-dev, depending on which
-  system you are using.
+  package management system you are using.
 
 
-GeoIPLite Database Installation
+GeoLite2-City Database Installation
--------------------------------
+-----------------------------------
 
-A country database for GeoIPLite is included when you do the C API
+Bro can use the city or country database.  The city database includes cities
-install, but for Bro, we are using the city database which includes
+and regions in addition to countries.
-cities and regions in addition to countries.
 
-`Download <http://www.maxmind.com/app/geolitecity>`__ the GeoLite city
+`Download <http://geolite.maxmind.com/download/geoip/database/GeoLite2-City.tar.gz>`__
-binary database:
+the GeoLite2 city binary database:
 
 .. console::
 
-    wget http://geolite.maxmind.com/download/geoip/database/GeoLiteCity.dat.gz
+    wget http://geolite.maxmind.com/download/geoip/database/GeoLite2-City.tar.gz
-    gunzip GeoLiteCity.dat.gz
+    tar zxf GeoLite2-City.tar.gz
 
-Next, the file needs to be renamed and put in the GeoIP database directory.
+Next, the file "GeoLite2-City_YYYYMMDD/GeoLite2-City.mmdb" needs to be renamed
-This directory should already exist and will vary depending on which platform
+and put in the GeoIP database directory.  This directory should already exist
-and package you are using.  For FreeBSD, use ``/usr/local/share/GeoIP``.  For
+and will vary depending on which platform and package you are using.  For
-Linux, use ``/usr/share/GeoIP`` or ``/var/lib/GeoIP`` (choose whichever one
+FreeBSD, use ``/usr/local/share/GeoIP``.  For Linux, use ``/usr/share/GeoIP``
-already exists).
+or ``/var/lib/GeoIP`` (choose whichever one already exists).
     
 .. console::
 
-    mv GeoLiteCity.dat <path_to_database_dir>/GeoIPCity.dat
+    mv <extracted subdir>/GeoLite2-City.mmdb <path_to_database_dir>/GeoLite2-City.mmdb
-
-Note that there is a separate database for IPv6 addresses, which can also
-be installed if you want GeoIP functionality for IPv6.
 
 Testing
 -------
 
 Before using the GeoIP functionality, it is a good idea to verify that
-everything is setup correctly.  After installing libGeoIP and the GeoIP city
+everything is setup correctly.  After installing libmaxminddb and the GeoIP
-database, and building Bro, you can quickly check if the GeoIP functionality
+city database, and building Bro, you can quickly check if the GeoIP
-works by running a command like this:
+functionality works by running a command like this:
 
 .. console::
 
     bro -e "print lookup_location(8.8.8.8);"
 
-If you see an error message similar to "Failed to open GeoIP City database",
+If you see an error message similar to "Failed to open GeoIP location
-then you may need to either rename or move your GeoIP city database file (the
+database", then you may need to either rename or move your GeoIP
-error message should give you the full pathname of the database file that
+location database file.  Bro looks for location database files in the
-Bro is looking for).
+following order by default:
+
+    /usr/share/GeoIP/GeoLite2-City.mmdb
+    /var/lib/GeoIP/GeoLite2-City.mmdb
+    /usr/local/share/GeoIP/GeoLite2-City.mmdb
+    /usr/local/var/GeoIP/GeoLite2-City.mmdb
+    /usr/share/GeoIP/GeoLite2-Country.mmdb
+    /var/lib/GeoIP/GeoLite2-Country.mmdb
+    /usr/local/share/GeoIP/GeoLite2-Country.mmdb
+    /usr/local/var/GeoIP/GeoLite2-Country.mmdb
 
 If you see an error message similar to "Bro was not configured for GeoIP
-support", then you need to rebuild Bro and make sure it is linked against
+support", then you either need to rebuild Bro and make sure it is linked
-libGeoIP.  Normally, if libGeoIP is installed correctly then it should
+against libmaxminddb or else set the :bro:see:`mmdb_dir`` value
-automatically be found when building Bro.  If this doesn't happen, then
+correctly.  Normally, if libmaxminddb is installed correctly then it
-you may need to specify the path to the libGeoIP installation
+should automatically be found when building Bro.  If this doesn't
-(e.g. ``./configure --with-geoip=<path>``).
+happen, then you may need to specify the path to the libmaxminddb
+installation (e.g. ``./configure --with-geoip=<path>``).
 
 Usage
 -----

doc/install/install.rst

@@ -54,18 +54,12 @@ To install the required dependencies, you can use:
 
      sudo yum install cmake make gcc gcc-c++ flex bison libpcap-devel openssl-devel python-devel swig zlib-devel
 
-  In order to build Bro on Fedora 26, install ``compat-openssl10-devel`` instead
-  of ``openssl-devel``.
-
 * DEB/Debian-based Linux:
 
   .. console::
 
      sudo apt-get install cmake make gcc g++ flex bison libpcap-dev libssl-dev python-dev swig zlib1g-dev
 
-  In order to build Bro on Debian 9 or Ubuntu 18.04, install ``libssl1.0-dev``
-  instead of ``libssl-dev``.
-
 * FreeBSD:
 
   Most required dependencies should come with a minimal FreeBSD install

scripts/base/init-bare.bro

@@ -872,6 +872,9 @@ type geo_location: record {
 	longitude: double &optional;	##< Longitude.
 } &log;
 
+## The directory containing MaxMind DB (*.mmdb) files to use for GeoIP support.
+const mmdb_dir: string = "" &redef;
+
 ## Computed entropy values. The record captures a number of measures that are
 ## computed in parallel. See `A Pseudorandom Number Sequence Test Program
 ## <http://www.fourmilab.ch/random>`_ for more information, Bro uses the same

src/analyzer/protocol/ntlm/ntlm-protocol.pac

@@ -171,7 +171,7 @@ type NTLM_Negotiate_Flags = record {
 	request_non_nt_session_key          : bool = (flags & 0x00400000) > 0;
 	negotiate_identify                  : bool = (flags & 0x00100000) > 0;
 
-	negotiate_extended_sessionsecurity  : bool = (flags & 0x00040000) > 0;
+	negotiate_extended_sessionsecurity  : bool = (flags & 0x00080000) > 0;
 	target_type_server                  : bool = (flags & 0x00020000) > 0;
 	target_type_domain                  : bool = (flags & 0x00010000) > 0;
 
@@ -179,14 +179,14 @@ type NTLM_Negotiate_Flags = record {
 	negotiate_oem_workstation_supplied  : bool = (flags & 0x00002000) > 0;
 	negotiate_oem_domain_supplied       : bool = (flags & 0x00001000) > 0;
 
-	negotiate_anonymous_connection      : bool = (flags & 0x00000400) > 0;
+	negotiate_anonymous_connection      : bool = (flags & 0x00000800) > 0;
-	negotiate_ntlm                      : bool = (flags & 0x00000100) > 0;
+	negotiate_ntlm                      : bool = (flags & 0x00000200) > 0;
 
 	negotiate_lm_key                    : bool = (flags & 0x00000080) > 0;
 	negotiate_datagram                  : bool = (flags & 0x00000040) > 0;
 	negotiate_seal                      : bool = (flags & 0x00000020) > 0;
+	negotiate_sign                      : bool = (flags & 0x00000010) > 0;
 
-	negotiate_sign                      : bool = (flags & 0x00000008) > 0;
 	request_target                      : bool = (flags & 0x00000004) > 0;
 	negotiate_oem                       : bool = (flags & 0x00000002) > 0;
 	negotiate_unicode                   : bool = (flags & 0x00000001) > 0;

src/bro.bif

@@ -3599,77 +3599,269 @@ function lookup_hostname%(host: string%) : addr_set
 %%{
 #ifdef USE_GEOIP
 extern "C" {
-#include <GeoIPCity.h>
+#include <maxminddb.h>
+#include <sys/stat.h>
+#include <sys/socket.h>
+#include <netinet/in.h>
+#include <netinet/ip.h>
 }
 
-static GeoIP* open_geoip_db(GeoIPDBTypes type)
+class MMDB {
+public:
+	MMDB(const char* filename);
+
+	~MMDB();
+
+	MMDB_lookup_result_s Lookup(const struct sockaddr* const sa);
+
+private:
+	MMDB_s mmdb;
+};
+
+MMDB::MMDB(const char *filename)
 	{
-	GeoIP* geoip = 0;
+	int status = MMDB_open(filename, MMDB_MODE_MMAP, &mmdb);
 
-	if ( GeoIP_db_avail(type) )
+	if ( MMDB_SUCCESS != status )
-		geoip = GeoIP_open_type(type, GEOIP_MEMORY_CACHE);
+		{
-
+		throw std::runtime_error(MMDB_strerror(status));
-	return geoip;
+		}
 	}
 
-static GeoIP* open_geoip_city_db()
+MMDB::~MMDB()
 	{
-	GeoIP* geoip = open_geoip_db(GEOIP_CITY_EDITION_REV0);
+	MMDB_close(&mmdb);
-
-	if ( ! geoip )
-		geoip = open_geoip_db(GEOIP_CITY_EDITION_REV1);
-
-	if ( ! geoip )
-		{
-		string rev0_path = GeoIPDBFileName[GEOIP_CITY_EDITION_REV0];
-		string rev1_path = GeoIPDBFileName[GEOIP_CITY_EDITION_REV1];
-		string db_path = rev0_path;
-
-		// Maybe in the future the revisions won't share a common default path.
-		if ( rev0_path != rev1_path )
-			db_path = rev0_path + " or " + rev1_path;
-
-		reporter->Info("Failed to open GeoIP City database: %s",
-		               db_path.c_str());
-		}
-
-	return geoip;
 	}
 
-static GeoIP* open_geoip_city_db_v6()
+MMDB_lookup_result_s MMDB::Lookup(const struct sockaddr* const sa)
 	{
-	GeoIP* geoip = 0;
+	int mmdb_error;
+	MMDB_lookup_result_s result = MMDB_lookup_sockaddr(&mmdb, sa, &mmdb_error);
 
-	// Both city edition revisions for IPv6 show up in libGeoIP 1.4.7.
+	if ( MMDB_SUCCESS != mmdb_error )
-#ifdef HAVE_GEOIP_CITY_EDITION_REV0_V6
-	geoip = open_geoip_db(GEOIP_CITY_EDITION_REV0_V6);
-
-	if ( ! geoip )
-		geoip = open_geoip_db(GEOIP_CITY_EDITION_REV1_V6);
-
-	if ( ! geoip )
 		{
-		string rev0_path = GeoIPDBFileName[GEOIP_CITY_EDITION_REV0_V6];
+		throw std::runtime_error(MMDB_strerror(mmdb_error));
-		string rev1_path = GeoIPDBFileName[GEOIP_CITY_EDITION_REV1_V6];
-		string db_path = rev0_path;
-
-		// Maybe in the future the revisions won't share a common default path.
-		if ( rev0_path != rev1_path )
-			db_path = rev0_path + " or " + rev1_path;
-
-		reporter->Info("Failed to open GeoIP Cityv6 database: %s",
-		               db_path.c_str());
 		}
-#endif
 
-	return geoip;
+	return result;
+	}
+
+std::unique_ptr<MMDB> mmdb_loc;
+std::unique_ptr<MMDB> mmdb_asn;
+
+static bool mmdb_open(const char* filename, bool asn)
+	{
+	struct stat buf;
+
+	if ( 0 != stat(filename, &buf) )
+		{
+		return false;
+		}
+
+	try
+		{
+		if ( asn )
+			{
+			mmdb_asn.reset(new MMDB(filename));
+			}
+		else
+			{
+			mmdb_loc.reset(new MMDB(filename));
+			}
+		}
+
+	catch ( const std::exception& e )
+		{
+		reporter->Info("Failed to open MaxMind DB: %s [%s]", filename,
+		               e.what());
+		return false;
+		}
+
+	return true;
+	}
+
+static bool mmdb_open_loc(const char* filename)
+	{
+	return mmdb_open(filename, false);
+	}
+
+static bool mmdb_open_asn(const char* filename)
+	{
+	return mmdb_open(filename, true);
+	}
+
+static bool mmdb_lookup(const IPAddr& addr, MMDB_lookup_result_s& result,
+                        bool asn)
+	{
+	struct sockaddr_storage ss = {0};
+
+	if ( IPv4 == addr.GetFamily() )
+		{
+		struct sockaddr_in* sa = (struct sockaddr_in*)&ss;
+		sa->sin_family = AF_INET;
+		addr.CopyIPv4(&sa->sin_addr);
+		}
+
+	else
+		{
+		struct sockaddr_in6* sa = (struct sockaddr_in6*)&ss;
+		sa->sin6_family = AF_INET6;
+		addr.CopyIPv6(&sa->sin6_addr);
+		}
+
+	try
+		{
+		result = asn ? mmdb_asn->Lookup((struct sockaddr*)&ss)
+		             : mmdb_loc->Lookup((struct sockaddr*)&ss);
+		}
+
+	catch ( const std::exception& e )
+		{
+		reporter->Info("MaxMind DB lookup location error [%s]",
+		               e.what());
+		return false;
+		}
+
+	return result.found_entry;
+	}
+
+static bool mmdb_lookup_loc(const IPAddr& addr, MMDB_lookup_result_s& result)
+	{
+	return mmdb_lookup(addr, result, false);
+	}
+
+static bool mmdb_lookup_asn(const IPAddr& addr, MMDB_lookup_result_s& result)
+	{
+	return mmdb_lookup(addr, result, true);
+	}
+
+static Val* mmdb_getvalue(MMDB_entry_data_s* entry_data, int status,
+                          int data_type )
+	{
+	switch (status)
+		{
+		case MMDB_SUCCESS:
+			if ( entry_data->has_data )
+				{
+				switch (data_type)
+					{
+					case MMDB_DATA_TYPE_UTF8_STRING:
+						return new StringVal(entry_data->data_size,
+						                     entry_data->utf8_string);
+						break;
+
+					case MMDB_DATA_TYPE_DOUBLE:
+						return new Val(entry_data->double_value, TYPE_DOUBLE);
+						break;
+
+					case MMDB_DATA_TYPE_UINT32:
+						return new Val(entry_data->uint32, TYPE_COUNT);
+
+					default:
+						break;
+					}
+				}
+			break;
+
+		case MMDB_LOOKUP_PATH_DOES_NOT_MATCH_DATA_ERROR:
+			// key doesn't exist, nothing to do
+			break;
+
+		default:
+			reporter->Info("MaxMind DB error [%s]", MMDB_strerror(status));
+			break;
+		}
+
+	return nullptr;
+	}
+
+static bool mmdb_try_open_loc ()
+	{
+	// City database is always preferred over Country database.
+	auto mmdb_dir_val = global_scope()->Lookup("mmdb_dir")->ID_Val();
+	std::string mmdb_dir = mmdb_dir_val->AsString()->CheckString();
+
+	if ( ! mmdb_dir.empty() )
+		{
+		auto d = mmdb_dir + "/GeoLite2-City.mmdb";
+
+		if ( mmdb_open_loc(d.data()) )
+			return true;
+
+		d = mmdb_dir + "/GeoLite2-Country.mmdb";
+
+		if ( mmdb_open_loc(d.data()) )
+			return true;;
+		}
+
+	return mmdb_open_loc("/usr/share/GeoIP/GeoLite2-City.mmdb")
+	    || mmdb_open_loc("/var/lib/GeoIP/GeoLite2-City.mmdb")
+	    || mmdb_open_loc("/usr/local/share/GeoIP/GeoLite2-City.mmdb")
+	    || mmdb_open_loc("/usr/local/var/GeoIP/GeoLite2-City.mmdb")
+	    || mmdb_open_loc("/usr/share/GeoIP/GeoLite2-Country.mmdb")
+	    || mmdb_open_loc("/var/lib/GeoIP/GeoLite2-Country.mmdb")
+	    || mmdb_open_loc("/usr/local/share/GeoIP/GeoLite2-Country.mmdb")
+	    || mmdb_open_loc("/usr/local/var/GeoIP/GeoLite2-Country.mmdb");
+	}
+
+static bool mmdb_try_open_asn ()
+	{
+	auto mmdb_dir_val = global_scope()->Lookup("mmdb_dir")->ID_Val();
+	std::string mmdb_dir = mmdb_dir_val->AsString()->CheckString();
+
+	if ( ! mmdb_dir.empty() )
+		{
+		auto d = mmdb_dir + "/GeoLite2-ASN.mmdb";
+
+		if ( mmdb_open_asn(d.data()) )
+			return true;
+		}
+
+	return mmdb_open_asn("/usr/share/GeoIP/GeoLite2-ASN.mmdb")
+	    || mmdb_open_asn("/var/lib/GeoIP/GeoLite2-ASN.mmdb")
+	    || mmdb_open_asn("/usr/local/share/GeoIP/GeoLite2-ASN.mmdb")
+	    || mmdb_open_asn("/usr/local/var/GeoIP/GeoLite2-ASN.mmdb");
 	}
 
 #endif
 %%}
 
+## Initializes MMDB for later use of lookup_location.
+## Requires Bro to be built with ``libmaxminddb``.
+##
+## f: The filename of the MaxMind City or Country DB.
+##
+## Returns: A boolean indicating whether the db was successfully opened.
+##
+## .. bro:see:: lookup_asn
+function mmdb_open_location_db%(f: string%) : bool
+	%{
+#ifdef USE_GEOIP
+	return new Val(mmdb_open_loc(f->CheckString()), TYPE_BOOL);
+#else
+        return new Val(0, TYPE_BOOL);
+#endif
+	%}
+
+## Initializes MMDB for later use of lookup_asn.
+## Requires Bro to be built with ``libmaxminddb``.
+##
+## f: The filename of the MaxMind ASN DB.
+##
+## Returns: A boolean indicating whether the db was successfully opened.
+##
+## .. bro:see:: lookup_asn
+function mmdb_open_asn_db%(f: string%) : bool
+	%{
+#ifdef USE_GEOIP
+	return new Val(mmdb_open_asn(f->CheckString()), TYPE_BOOL);
+#else
+	return new Val(0, TYPE_BOOL);
+#endif
+	%}
+
 ## Performs a geo-lookup of an IP address.
-## Requires Bro to be built with ``libgeoip``.
+## Requires Bro to be built with ``libmaxminddb``.
 ##
 ## a: The IP address to lookup.
 ##
@@ -3681,104 +3873,52 @@ function lookup_location%(a: addr%) : geo_location
 	RecordVal* location = new RecordVal(geo_location);
 
 #ifdef USE_GEOIP
-	static bool geoip_initialized = false;
+	if ( ! mmdb_loc )
-	static GeoIP* geoip = 0;
-	static GeoIP* geoip_v6 = 0;
-	static bool have_city_db = false;
-	static bool have_cityv6_db = false;
-	GeoIPRecord* gir = 0;
-	const char* cc = 0;
-
-	if ( ! geoip_initialized )
 		{
-		geoip_initialized = true;
+		if ( ! mmdb_try_open_loc() )
-		geoip = open_geoip_city_db();
-
-		if ( ! geoip )
 			{
-			geoip = open_geoip_db(GEOIP_COUNTRY_EDITION);
+			builtin_error("Failed to open GeoIP location database");
-			string db_path = GeoIPDBFileName[GEOIP_COUNTRY_EDITION];
+			return location;
-
-			if ( ! geoip )
-				builtin_error(fmt("Failed fall back to GeoIP Country "
-				                  "database: %s",
-				                  GeoIPDBFileName[GEOIP_COUNTRY_EDITION]));
-			else
-				reporter->Info("Fell back to GeoIP Country database");
 			}
-		else
-			have_city_db = true;
-
-		geoip_v6 = open_geoip_city_db_v6();
-
-		if ( geoip_v6 )
-			have_cityv6_db = true;
-
-#ifdef HAVE_GEOIP_COUNTRY_EDITION_V6
-		if ( ! geoip_v6 )
-			{
-			geoip_v6 = open_geoip_db(GEOIP_COUNTRY_EDITION_V6);
-
-			if ( ! geoip_v6 )
-				reporter->Info("Failed to open GeoIPv6 Country database: %s",
-				               GeoIPDBFileName[GEOIP_COUNTRY_EDITION_V6]);
-			}
-#endif
-
-		if ( ! geoip_v6 )
-			builtin_error("Can't open GeoIPv6 City/Country database");
 		}
 
-#ifdef HAVE_GEOIP_COUNTRY_EDITION_V6
+	MMDB_lookup_result_s result;
-	if ( geoip_v6 && a->AsAddr().GetFamily() == IPv6 )
+
+	if ( mmdb_lookup_loc(a->AsAddr(), result) )
 		{
-		geoipv6_t ga;
+		MMDB_entry_data_s entry_data;
-		a->AsAddr().CopyIPv6(&ga);
+		int status;
-		if ( have_cityv6_db )
-			gir = GeoIP_record_by_ipnum_v6(geoip_v6, ga);
-		else
-			cc = GeoIP_country_code_by_ipnum_v6(geoip_v6, ga);
-		}
-	else
-#endif
 
-	if ( geoip && a->AsAddr().GetFamily() == IPv4 )
+		// Get Country ISO Code
-		{
+		status = MMDB_get_value(&result.entry, &entry_data,
-		const uint32* bytes;
+		                        "country", "iso_code", nullptr);
-		a->AsAddr().GetBytes(&bytes);
+		location->Assign(0, mmdb_getvalue(&entry_data, status,
-		if ( have_city_db )
+		                 MMDB_DATA_TYPE_UTF8_STRING));
-			gir = GeoIP_record_by_ipnum(geoip, ntohl(*bytes));
-		else
-			cc = GeoIP_country_code_by_ipnum(geoip, ntohl(*bytes));
-		}
 
-	if ( gir )
+		// Get Major Subdivision ISO Code
-		{
+		status = MMDB_get_value(&result.entry, &entry_data,
-		if ( gir->country_code )
+		                        "subdivisions", "0", "iso_code", nullptr);
-			location->Assign(0, new StringVal(gir->country_code));
+		location->Assign(1, mmdb_getvalue(&entry_data, status,
+		                 MMDB_DATA_TYPE_UTF8_STRING));
 
-		if ( gir->region )
+		// Get City English Name
-			location->Assign(1, new StringVal(gir->region));
+		status = MMDB_get_value(&result.entry, &entry_data,
+		                        "city", "names", "en", nullptr);
+		location->Assign(2, mmdb_getvalue(&entry_data, status,
+		                 MMDB_DATA_TYPE_UTF8_STRING));
 
-		if ( gir->city )
+		// Get Location Latitude
-			location->Assign(2, new StringVal(gir->city));
+		status = MMDB_get_value(&result.entry, &entry_data,
+		                        "location", "latitude", nullptr);
+		location->Assign(3, mmdb_getvalue(&entry_data, status,
+		                 MMDB_DATA_TYPE_DOUBLE));
 
-		if ( gir->latitude )
+		// Get Location Longitude
-			location->Assign(3, new Val(gir->latitude,
+		status = MMDB_get_value(&result.entry, &entry_data,
-							TYPE_DOUBLE));
+		                        "location", "longitude", nullptr);
+		location->Assign(4, mmdb_getvalue(&entry_data, status,
+		                 MMDB_DATA_TYPE_DOUBLE));
 
-		if ( gir->longitude )
-			location->Assign(4, new Val(gir->longitude,
-						TYPE_DOUBLE));
-
-		GeoIPRecord_delete(gir);
-
-		return location;
-		}
-
-	else if ( cc )
-		{
-		location->Assign(0, new StringVal(cc));
 		return location;
 		}
 
@@ -3792,64 +3932,45 @@ function lookup_location%(a: addr%) : geo_location
 		}
 #endif
 
-	// We can get here even if we have GeoIP support if we weren't
+	// We can get here even if we have MMDB support if we weren't
 	// able to initialize it or it didn't return any information for
 	// the address.
 
 	return location;
 	%}
 
-## Performs an AS lookup of an IP address.
+## Performs an ASN lookup of an IP address.
-## Requires Bro to be built with ``libgeoip``.
+## Requires Bro to be built with ``libmaxminddb``.
 ##
 ## a: The IP address to lookup.
 ##
-## Returns: The number of the AS that contains *a*.
+## Returns: The number of the ASN that contains *a*.
 ##
 ## .. bro:see:: lookup_location
 function lookup_asn%(a: addr%) : count
 	%{
 #ifdef USE_GEOIP
-	static GeoIP* geoip_asn = 0;
+	if ( ! mmdb_asn )
-	static bool geoip_asn_initialized = false;
-	char* gir = 0;
-
-	if ( ! geoip_asn_initialized )
 		{
-		geoip_asn_initialized = true;
+		if ( ! mmdb_try_open_asn() )
-		geoip_asn = open_geoip_db(GEOIP_ASNUM_EDITION);
-
-		if ( ! geoip_asn )
-			builtin_error(fmt("Can't open GeoIP ASNUM database: %s",
-			                  GeoIPDBFileName[GEOIP_ASNUM_EDITION]));
-		}
-
-	if ( geoip_asn )
-		{
-// IPv6 support showed up in 1.4.5.
-#ifdef HAVE_GEOIP_COUNTRY_EDITION_V6
-		if ( a->AsAddr().GetFamily() == IPv6 )
 			{
-			geoipv6_t ga;
+			builtin_error("No open GeoIP ASN database");
-			a->AsAddr().CopyIPv6(&ga);
+			return new Val(0, TYPE_COUNT);
-			gir = GeoIP_name_by_ipnum_v6(geoip_asn, ga);
-			}
-		else
-#endif
-
-		if ( a->AsAddr().GetFamily() == IPv4 )
-			{
-			const uint32* bytes;
-			a->AsAddr().GetBytes(&bytes);
-			gir = GeoIP_name_by_ipnum(geoip_asn, ntohl(*bytes));
 			}
 		}
 
-	if ( gir )
+	MMDB_lookup_result_s result;
+
+	if ( mmdb_lookup_asn(a->AsAddr(), result) )
 		{
-		// Move the pointer +2 so we don't return
+		MMDB_entry_data_s entry_data;
-		// the first two characters: "AS".
+		int status;
-		return new Val(atoi(gir+2), TYPE_COUNT);
+
+		// Get Autonomous System Number
+		status = MMDB_get_value(&result.entry, &entry_data,
+		                        "autonomous_system_number", nullptr);
+		Val* asn = mmdb_getvalue(&entry_data, status, MMDB_DATA_TYPE_UINT32);
+		return asn == nullptr ? new Val(0, TYPE_COUNT) : asn;
 		}
 
 #else // not USE_GEOIP

src/file_analysis/analyzer/x509/OCSP.cc

@@ -21,7 +21,7 @@
 // helper function of sk_X509_value to avoid namespace problem
 // sk_X509_value(X,Y) = > SKM_sk_value(X509,X,Y)
 // X509 => file_analysis::X509
-X509 *helper_sk_X509_value(STACK_OF(X509) *certs, int i)
+X509* helper_sk_X509_value(const STACK_OF(X509)* certs, int i)
 	{
 	return sk_X509_value(certs, i);
 	}
@@ -42,38 +42,84 @@ static Val* get_ocsp_type(RecordVal* args, const char* name)
 	return rval;
 	}
 
-static void OCSP_RESPID_bio(OCSP_RESPID *resp_id, BIO* bio)
+static bool OCSP_RESPID_bio(OCSP_BASICRESP* basic_resp, BIO* bio)
 	{
-	if (resp_id->type == V_OCSP_RESPID_NAME)
+#if ( OPENSSL_VERSION_NUMBER < 0x10100000L )
-		X509_NAME_print_ex(bio, resp_id->value.byName, 0, XN_FLAG_ONELINE);
+	ASN1_OCTET_STRING* key  = nullptr;
-	else if (resp_id->type == V_OCSP_RESPID_KEY)
+	X509_NAME*         name = nullptr;
-		i2a_ASN1_STRING(bio, resp_id->value.byKey, V_ASN1_OCTET_STRING);
+
+	if ( ! basic_resp->tbsResponseData )
+		return false;
+
+	auto resp_id = basic_resp->tbsResponseData->responderId;
+
+	if ( resp_id->type == V_OCSP_RESPID_NAME )
+		name = resp_id->value.byName;
+	else if ( resp_id->type == V_OCSP_RESPID_KEY )
+		key = resp_id->value.byKey;
+	else
+		return false;
+#else
+	const ASN1_OCTET_STRING* key  = nullptr;
+	const X509_NAME*         name = nullptr;
+
+	if ( ! OCSP_resp_get0_id(basic_resp, &key, &name) )
+		return false;
+#endif
+
+	if ( name )
+		X509_NAME_print_ex(bio, name, 0, XN_FLAG_ONELINE);
+	else
+		i2a_ASN1_STRING(bio, key, V_ASN1_OCTET_STRING);
+
+	return true;
 	}
 
-void ocsp_add_cert_id(OCSP_CERTID *cert_id, val_list* vl, BIO* bio)
+bool ocsp_add_cert_id(const OCSP_CERTID* cert_id, val_list* vl, BIO* bio)
 	{
+	ASN1_OBJECT*       hash_alg         = nullptr;
+	ASN1_OCTET_STRING* issuer_name_hash = nullptr;
+	ASN1_OCTET_STRING* issuer_key_hash  = nullptr;
+	ASN1_INTEGER*      serial_number    = nullptr;
+
+	auto res = OCSP_id_get0_info(&issuer_name_hash, &hash_alg,
+	                             &issuer_key_hash, &serial_number,
+	                             const_cast<OCSP_CERTID*>(cert_id));
+
+	if ( ! res )
+		{
+		reporter->Weird("OpenSSL failed to get OCSP_CERTID info");
+		vl->append(new StringVal(""));
+		vl->append(new StringVal(""));
+		vl->append(new StringVal(""));
+		vl->append(new StringVal(""));
+		return false;
+		}
+
 	char buf[OCSP_STRING_BUF_SIZE];
 	memset(buf, 0, sizeof(buf));
 
-	i2a_ASN1_OBJECT(bio, cert_id->hashAlgorithm->algorithm);
+	i2a_ASN1_OBJECT(bio, hash_alg);
 	int len = BIO_read(bio, buf, sizeof(buf));
 	vl->append(new StringVal(len, buf));
 	BIO_reset(bio);
 
-	i2a_ASN1_STRING(bio, cert_id->issuerNameHash, V_ASN1_OCTET_STRING);
+	i2a_ASN1_STRING(bio, issuer_name_hash, V_ASN1_OCTET_STRING);
 	len = BIO_read(bio, buf, sizeof(buf));
 	vl->append(new StringVal(len, buf));
 	BIO_reset(bio);
 
-	i2a_ASN1_STRING(bio, cert_id->issuerKeyHash, V_ASN1_OCTET_STRING);
+	i2a_ASN1_STRING(bio, issuer_key_hash, V_ASN1_OCTET_STRING);
 	len = BIO_read(bio, buf, sizeof(buf));
 	vl->append(new StringVal(len, buf));
 	BIO_reset(bio);
 
-	i2a_ASN1_INTEGER(bio, cert_id->serialNumber);
+	i2a_ASN1_INTEGER(bio, serial_number);
 	len = BIO_read(bio, buf, sizeof(buf));
 	vl->append(new StringVal(len, buf));
 	BIO_reset(bio);
+
+	return true;
 	}
 
 file_analysis::Analyzer* OCSP::InstantiateRequest(RecordVal* args, File* file)
@@ -124,6 +170,7 @@ bool file_analysis::OCSP::EndOfFile()
 	else
 		{
 		OCSP_RESPONSE *resp = d2i_OCSP_RESPONSE(NULL, &ocsp_char, ocsp_data.size());
+
 		if (!resp)
 			{
 			reporter->Weird(fmt("OPENSSL Could not parse OCSP response (fuid %s)", GetFile()->GetID().c_str()));
@@ -138,28 +185,255 @@ bool file_analysis::OCSP::EndOfFile()
 	return true;
 }
 
-void file_analysis::OCSP::ParseRequest(OCSP_REQUEST *req, const char* fid)
+#if ( OPENSSL_VERSION_NUMBER >= 0x10100000L )
+// Re-encode and then parse out ASN1 structures to get at what we need...
+/*-  BasicOCSPResponse       ::= SEQUENCE {
+ *      tbsResponseData      ResponseData,
+ *      signatureAlgorithm   AlgorithmIdentifier,
+ *      signature            BIT STRING,
+ *      certs                [0] EXPLICIT SEQUENCE OF Certificate OPTIONAL }
+typedef struct ocsp_basic_response_st {
+    OCSP_RESPDATA *tbsResponseData;
+    X509_ALGOR *signatureAlgorithm;
+    ASN1_BIT_STRING *signature;
+    STACK_OF(X509) *certs;
+} OCSP_BASICRESP;
+*/
+static StringVal* parse_basic_resp_sig_alg(OCSP_BASICRESP* basic_resp,
+                                           BIO* bio, char* buf, size_t buf_len)
 	{
-	OCSP_REQINFO *inf     = req->tbsRequest;
+	int der_basic_resp_len = 0;
+	unsigned char* der_basic_resp_dat = nullptr;
 
+	der_basic_resp_len = i2d_OCSP_BASICRESP(basic_resp, &der_basic_resp_dat);
+
+	if ( der_basic_resp_len <= 0 )
+		return new StringVal("");
+
+	const unsigned char* const_der_basic_resp_dat = der_basic_resp_dat;
+
+	auto bseq = d2i_ASN1_SEQUENCE_ANY(nullptr, &const_der_basic_resp_dat,
+	                                  der_basic_resp_len);
+
+	if ( ! bseq )
+		{
+		OPENSSL_free(der_basic_resp_dat);
+		return new StringVal("");
+		}
+
+	if ( sk_ASN1_TYPE_num(bseq) < 3 )
+		{
+		sk_ASN1_TYPE_free(bseq);
+		OPENSSL_free(der_basic_resp_dat);
+		return new StringVal("");
+		}
+
+	auto constexpr sig_alg_idx = 1u;
+	auto aseq_type = sk_ASN1_TYPE_value(bseq, sig_alg_idx);
+
+	if ( ASN1_TYPE_get(aseq_type) != V_ASN1_SEQUENCE )
+		{
+		sk_ASN1_TYPE_free(bseq);
+		OPENSSL_free(der_basic_resp_dat);
+		return new StringVal("");
+		}
+
+	auto aseq_str = aseq_type->value.asn1_string;
+	auto aseq_len = ASN1_STRING_length(aseq_str);
+	auto aseq_dat = ASN1_STRING_get0_data(aseq_str);
+
+	auto aseq = d2i_ASN1_SEQUENCE_ANY(nullptr, &aseq_dat, aseq_len);
+
+	if ( ! aseq )
+		{
+		sk_ASN1_TYPE_free(bseq);
+		OPENSSL_free(der_basic_resp_dat);
+		return new StringVal("");
+		}
+
+	if ( sk_ASN1_TYPE_num(aseq) < 1 )
+		{
+		sk_ASN1_TYPE_free(aseq);
+		sk_ASN1_TYPE_free(bseq);
+		OPENSSL_free(der_basic_resp_dat);
+		return new StringVal("");
+		}
+
+	auto constexpr alg_obj_idx = 0u;
+	auto alg_obj_type = sk_ASN1_TYPE_value(aseq, alg_obj_idx);
+
+	if ( ASN1_TYPE_get(alg_obj_type) != V_ASN1_OBJECT )
+		{
+		sk_ASN1_TYPE_free(aseq);
+		sk_ASN1_TYPE_free(bseq);
+		OPENSSL_free(der_basic_resp_dat);
+		return new StringVal("");
+		}
+
+	auto alg_obj = alg_obj_type->value.object;
+	i2a_ASN1_OBJECT(bio, alg_obj);
+	auto alg_len = BIO_read(bio, buf, buf_len);
+	auto rval = new StringVal(alg_len, buf);
+	BIO_reset(bio);
+
+	sk_ASN1_TYPE_free(aseq);
+	sk_ASN1_TYPE_free(bseq);
+	OPENSSL_free(der_basic_resp_dat);
+	return rval;
+	}
+
+static Val* parse_basic_resp_data_version(OCSP_BASICRESP* basic_resp)
+	{
+	int der_basic_resp_len = 0;
+	unsigned char* der_basic_resp_dat = nullptr;
+
+	der_basic_resp_len = i2d_OCSP_BASICRESP(basic_resp, &der_basic_resp_dat);
+
+	if ( der_basic_resp_len <= 0 )
+		return new Val(-1, TYPE_COUNT);
+
+	const unsigned char* const_der_basic_resp_dat = der_basic_resp_dat;
+
+	auto bseq = d2i_ASN1_SEQUENCE_ANY(nullptr, &const_der_basic_resp_dat,
+	                                  der_basic_resp_len);
+
+	if ( ! bseq )
+		{
+		OPENSSL_free(der_basic_resp_dat);
+		return new Val(-1, TYPE_COUNT);
+		}
+
+	if ( sk_ASN1_TYPE_num(bseq) < 3 )
+		{
+		sk_ASN1_TYPE_free(bseq);
+		OPENSSL_free(der_basic_resp_dat);
+		return new Val(-1, TYPE_COUNT);
+		}
+
+	auto constexpr resp_data_idx = 0u;
+	auto dseq_type = sk_ASN1_TYPE_value(bseq, resp_data_idx);
+
+	if ( ASN1_TYPE_get(dseq_type) != V_ASN1_SEQUENCE )
+		{
+		sk_ASN1_TYPE_free(bseq);
+		OPENSSL_free(der_basic_resp_dat);
+		return new Val(-1, TYPE_COUNT);
+		}
+
+	auto dseq_str = dseq_type->value.asn1_string;
+	auto dseq_len = ASN1_STRING_length(dseq_str);
+	auto dseq_dat = ASN1_STRING_get0_data(dseq_str);
+
+	auto dseq = d2i_ASN1_SEQUENCE_ANY(nullptr, &dseq_dat, dseq_len);
+
+	if ( ! dseq )
+		{
+		sk_ASN1_TYPE_free(bseq);
+		OPENSSL_free(der_basic_resp_dat);
+		return new StringVal("");
+		}
+
+	if ( sk_ASN1_TYPE_num(dseq) < 1 )
+		{
+		sk_ASN1_TYPE_free(dseq);
+		sk_ASN1_TYPE_free(bseq);
+		OPENSSL_free(der_basic_resp_dat);
+		return new StringVal("");
+		}
+
+/*-  ResponseData ::= SEQUENCE {
+ *      version              [0] EXPLICIT Version DEFAULT v1,
+ *      responderID              ResponderID,
+ *      producedAt               GeneralizedTime,
+ *      responses                SEQUENCE OF SingleResponse,
+ *      responseExtensions   [1] EXPLICIT Extensions OPTIONAL }
+ */
+
+	auto constexpr version_idx = 0u;
+	auto version_type = sk_ASN1_TYPE_value(dseq, version_idx);
+
+	if ( ASN1_TYPE_get(version_type) != V_ASN1_INTEGER )
+		{
+		sk_ASN1_TYPE_free(dseq);
+		sk_ASN1_TYPE_free(bseq);
+		OPENSSL_free(der_basic_resp_dat);
+		// Not present, use default value.
+		return new Val(0, TYPE_COUNT);
+		}
+
+	uint64_t asn1_int = ASN1_INTEGER_get(version_type->value.integer);
+	sk_ASN1_TYPE_free(dseq);
+	sk_ASN1_TYPE_free(bseq);
+	OPENSSL_free(der_basic_resp_dat);
+	return new Val(asn1_int, TYPE_COUNT);
+	}
+
+static uint64 parse_request_version(OCSP_REQUEST* req)
+	{
+	int der_req_len = 0;
+	unsigned char* der_req_dat = nullptr;
+	der_req_len = i2d_OCSP_REQUEST(req, &der_req_dat);
+	const unsigned char* const_der_req_dat = der_req_dat;
+
+	if ( ! der_req_dat )
+		return -1;
+
+	auto rseq = d2i_ASN1_SEQUENCE_ANY(nullptr, &const_der_req_dat,
+	                                  der_req_len);
+
+	if ( ! rseq )
+		{
+		OPENSSL_free(der_req_dat);
+		return -1;
+		}
+
+	if ( sk_ASN1_TYPE_num(rseq) < 1 )
+		{
+		sk_ASN1_TYPE_free(rseq);
+		OPENSSL_free(der_req_dat);
+		return -1;
+		}
+
+	auto constexpr version_idx = 0u;
+	auto version_type = sk_ASN1_TYPE_value(rseq, version_idx);
+
+	if ( ASN1_TYPE_get(version_type) != V_ASN1_INTEGER )
+		{
+		sk_ASN1_TYPE_free(rseq);
+		OPENSSL_free(der_req_dat);
+		// Not present, use default value.
+		return 0;
+		}
+
+	uint64_t asn1_int = ASN1_INTEGER_get(version_type->value.integer);
+	sk_ASN1_TYPE_free(rseq);
+	OPENSSL_free(der_req_dat);
+	return asn1_int;
+	}
+#endif
+
+void file_analysis::OCSP::ParseRequest(OCSP_REQUEST* req, const char* fid)
+	{
 	char buf[OCSP_STRING_BUF_SIZE]; // we need a buffer for some of the openssl functions
 	memset(buf, 0, sizeof(buf));
 
 	// build up our response as we go along...
 	val_list* vl = new val_list();
 	vl->append(GetFile()->GetVal()->Ref());
-	vl->append(new Val((uint64)ASN1_INTEGER_get(inf->version), TYPE_COUNT));
-	BIO *bio = BIO_new(BIO_s_mem());
 
-	if (inf->requestorName != NULL)
+	uint64 version = 0;
-		{
+
-		GENERAL_NAME_print(bio, inf->requestorName);
+#if ( OPENSSL_VERSION_NUMBER < 0x10100000L )
-		int len = BIO_read(bio, buf, sizeof(buf));
+	if ( req->tbsRequest->version )
-		vl->append(new StringVal(len, buf));
+		version = (uint64)ASN1_INTEGER_get(req->tbsRequest->version);
-		BIO_reset(bio);
+#else
-		}
+	version = parse_request_version(req);
-	else
+	// TODO: try to parse out general name ?
-		vl->append(new StringVal(0, ""));
+#endif
+
+	vl->append(new Val(version, TYPE_COUNT));
+
+	BIO *bio = BIO_new(BIO_s_mem());
 
 	mgr.QueueEvent(ocsp_request, vl);
 
@@ -182,10 +456,12 @@ void file_analysis::OCSP::ParseRequest(OCSP_REQUEST *req, const char* fid)
 void file_analysis::OCSP::ParseResponse(OCSP_RESPVal *resp_val, const char* fid)
 	{
 	OCSP_RESPONSE   *resp       = resp_val->GetResp();
-	OCSP_RESPBYTES  *resp_bytes = resp->responseBytes;
+	//OCSP_RESPBYTES  *resp_bytes = resp->responseBytes;
 	OCSP_BASICRESP  *basic_resp = nullptr;
 	OCSP_RESPDATA   *resp_data  = nullptr;
 	OCSP_RESPID     *resp_id    = nullptr;
+	const ASN1_GENERALIZEDTIME* produced_at = nullptr;
+	const STACK_OF(X509)* certs = nullptr;
 
 	int resp_count, num_ext = 0;
 	VectorVal *certs_vector = nullptr;
@@ -203,11 +479,11 @@ void file_analysis::OCSP::ParseResponse(OCSP_RESPVal *resp_val, const char* fid)
 	mgr.QueueEvent(ocsp_response_status, vl);
 	vl = nullptr;
 
-	if (!resp_bytes)
+	//if (!resp_bytes)
-		{
+	//	{
-		Unref(status_val);
+	//	Unref(status_val);
-		return;
+	//	return;
-		}
+	//	}
 
 	BIO *bio = BIO_new(BIO_s_mem());
 	//i2a_ASN1_OBJECT(bio, resp_bytes->responseType);
@@ -219,31 +495,53 @@ void file_analysis::OCSP::ParseResponse(OCSP_RESPVal *resp_val, const char* fid)
 	if ( !basic_resp )
 		goto clean_up;
 
+#if ( OPENSSL_VERSION_NUMBER < 0x10100000L )
 	resp_data = basic_resp->tbsResponseData;
 	if ( !resp_data )
 		goto clean_up;
+#endif
 
 	vl = new val_list();
 	vl->append(GetFile()->GetVal()->Ref());
 	vl->append(resp_val->Ref());
 	vl->append(status_val);
+
+#if ( OPENSSL_VERSION_NUMBER < 0x10100000L )
 	vl->append(new Val((uint64)ASN1_INTEGER_get(resp_data->version), TYPE_COUNT));
+#else
+	vl->append(parse_basic_resp_data_version(basic_resp));
+#endif
 
 	// responderID
-	resp_id = resp_data->responderId;
+	if ( OCSP_RESPID_bio(basic_resp, bio) )
-	OCSP_RESPID_bio(resp_id, bio);
+		{
-	len = BIO_read(bio, buf, sizeof(buf));
+		len = BIO_read(bio, buf, sizeof(buf));
-	vl->append(new StringVal(len, buf));
+		vl->append(new StringVal(len, buf));
-	BIO_reset(bio);
+		BIO_reset(bio);
+		}
+	else
+		{
+		reporter->Weird("OpenSSL failed to get OCSP responder id");
+		vl->append(new StringVal(""));
+		}
 
 	// producedAt
-	vl->append(new Val(GetTimeFromAsn1(resp_data->producedAt, fid, reporter), TYPE_TIME));
+#if ( OPENSSL_VERSION_NUMBER < 0x10100000L )
+	produced_at = resp_data->producedAt;
+#else
+	produced_at = OCSP_resp_get0_produced_at(basic_resp);
+#endif
+
+	vl->append(new Val(GetTimeFromAsn1(produced_at, fid, reporter), TYPE_TIME));
 
 	// responses
-	resp_count = sk_OCSP_SINGLERESP_num(resp_data->responses);
+
+	resp_count = OCSP_resp_count(basic_resp);
+
 	for ( int i=0; i<resp_count; i++ )
 		{
-		OCSP_SINGLERESP *single_resp = sk_OCSP_SINGLERESP_value(resp_data->responses, i);
+		OCSP_SINGLERESP* single_resp = OCSP_resp_get0(basic_resp, i);
+
 		if ( !single_resp )
 			continue;
 
@@ -251,24 +549,41 @@ void file_analysis::OCSP::ParseResponse(OCSP_RESPVal *resp_val, const char* fid)
 		rvl->append(GetFile()->GetVal()->Ref());
 
 		// cert id
-		OCSP_CERTID *cert_id = single_resp->certId;
+		const OCSP_CERTID* cert_id = nullptr;
+
+#if ( OPENSSL_VERSION_NUMBER < 0x10100000L )
+		cert_id = single_resp->certId;
+#else
+		cert_id = OCSP_SINGLERESP_get0_id(single_resp);
+#endif
+
 		ocsp_add_cert_id(cert_id, rvl, bio);
 		BIO_reset(bio);
 
 		// certStatus
-		OCSP_CERTSTATUS *cert_status = single_resp->certStatus;
+		int status = V_OCSP_CERTSTATUS_UNKNOWN;
-		const char* cert_status_str = OCSP_cert_status_str(cert_status->type);
+		int reason = OCSP_REVOKED_STATUS_NOSTATUS;
+		ASN1_GENERALIZEDTIME* revoke_time = nullptr;
+		ASN1_GENERALIZEDTIME* this_update = nullptr;
+		ASN1_GENERALIZEDTIME* next_update = nullptr;
+
+		if ( ! OCSP_resp_find_status(basic_resp,
+		                             const_cast<OCSP_CERTID*>(cert_id),
+		                             &status, &reason, &revoke_time,
+		                             &this_update, &next_update) )
+			reporter->Weird("OpenSSL failed to find status of OCSP response");
+
+		const char* cert_status_str = OCSP_cert_status_str(status);
 		rvl->append(new StringVal(strlen(cert_status_str), cert_status_str));
 
 		// revocation time and reason if revoked
-		if ( cert_status->type == V_OCSP_CERTSTATUS_REVOKED )
+		if ( status == V_OCSP_CERTSTATUS_REVOKED )
 			{
-			OCSP_REVOKEDINFO *revoked_info = cert_status->value.revoked;
+			rvl->append(new Val(GetTimeFromAsn1(revoke_time, fid, reporter), TYPE_TIME));
-			rvl->append(new Val(GetTimeFromAsn1(revoked_info->revocationTime, fid, reporter), TYPE_TIME));
 
-			if ( revoked_info->revocationReason )
+			if ( reason != OCSP_REVOKED_STATUS_NOSTATUS )
 				{
-				const char* revoke_reason = OCSP_crl_reason_str(ASN1_ENUMERATED_get(revoked_info->revocationReason));
+				const char* revoke_reason = OCSP_crl_reason_str(reason);
 				rvl->append(new StringVal(strlen(revoke_reason), revoke_reason));
 				}
 			else
@@ -280,9 +595,13 @@ void file_analysis::OCSP::ParseResponse(OCSP_RESPVal *resp_val, const char* fid)
 			rvl->append(new StringVal(0, ""));
 			}
 
-		rvl->append(new Val(GetTimeFromAsn1(single_resp->thisUpdate, fid, reporter), TYPE_TIME));
+		if ( this_update )
-		if ( single_resp->nextUpdate )
+			rvl->append(new Val(GetTimeFromAsn1(this_update, fid, reporter), TYPE_TIME));
-			rvl->append(new Val(GetTimeFromAsn1(single_resp->nextUpdate, fid, reporter), TYPE_TIME));
+		else
+			rvl->append(new Val(0, TYPE_TIME));
+
+		if ( next_update )
+			rvl->append(new Val(GetTimeFromAsn1(next_update, fid, reporter), TYPE_TIME));
 		else
 			rvl->append(new Val(0, TYPE_TIME));
 
@@ -299,10 +618,14 @@ void file_analysis::OCSP::ParseResponse(OCSP_RESPVal *resp_val, const char* fid)
 			}
 		}
 
+#if ( OPENSSL_VERSION_NUMBER < 0x10100000L )
 	i2a_ASN1_OBJECT(bio, basic_resp->signatureAlgorithm->algorithm);
 	len = BIO_read(bio, buf, sizeof(buf));
 	vl->append(new StringVal(len, buf));
 	BIO_reset(bio);
+#else
+	vl->append(parse_basic_resp_sig_alg(basic_resp, bio, buf, sizeof(buf)));
+#endif
 
 	//i2a_ASN1_OBJECT(bio, basic_resp->signature);
 	//len = BIO_read(bio, buf, sizeof(buf));
@@ -311,13 +634,20 @@ void file_analysis::OCSP::ParseResponse(OCSP_RESPVal *resp_val, const char* fid)
 
 	certs_vector = new VectorVal(internal_type("x509_opaque_vector")->AsVectorType());
 	vl->append(certs_vector);
-	if ( basic_resp->certs )
+
+#if ( OPENSSL_VERSION_NUMBER < 0x10100000L )
+	certs = basic_resp->certs;
+#else
+	certs = OCSP_resp_get0_certs(basic_resp);
+#endif
+
+	if ( certs )
 		{
-		int num_certs = sk_X509_num(basic_resp->certs);
+		int num_certs = sk_X509_num(certs);
 		for ( int i=0; i<num_certs; i++ )
 			{
-			::X509 *this_cert = X509_dup(helper_sk_X509_value(basic_resp->certs, i));
+			::X509 *this_cert = X509_dup(helper_sk_X509_value(certs, i));
-			//::X509 *this_cert = X509_dup(sk_X509_value(basic_resp->certs, i));
+			//::X509 *this_cert = X509_dup(sk_X509_value(certs, i));
 			if (this_cert)
 				certs_vector->Assign(i, new file_analysis::X509Val(this_cert));
 			else

src/file_analysis/analyzer/x509/OCSP.h

@@ -29,8 +29,8 @@ protected:
 	OCSP(RecordVal* args, File* file, bool request);
 
 private:
-	void ParseResponse(OCSP_RESPVal *, const char* fid = 0);
+	void ParseResponse(OCSP_RESPVal*, const char* fid = 0);
-	void ParseRequest(OCSP_REQUEST *, const char* fid = 0);
+	void ParseRequest(OCSP_REQUEST*, const char* fid = 0);
 	void ParseExtensionsSpecific(X509_EXTENSION* ex, bool, ASN1_OBJECT*, const char*) override;
 
 	std::string ocsp_data;

src/file_analysis/analyzer/x509/X509.cc

@@ -290,7 +290,11 @@ void file_analysis::X509::ParseSAN(X509_EXTENSION* ext)
 				continue;
 				}
 
+#if ( OPENSSL_VERSION_NUMBER < 0x10100000L )
 			const char* name = (const char*) ASN1_STRING_data(gen->d.ia5);
+#else
+			const char* name = (const char*) ASN1_STRING_get0_data(gen->d.ia5);
+#endif
 			StringVal* bs = new StringVal(name);
 
 			switch ( gen->type )

src/file_analysis/analyzer/x509/functions.bif

@@ -109,21 +109,36 @@ STACK_OF(X509)* x509_get_untrusted_stack(VectorVal* certs_vec)
 
 // We need this function to be able to identify the signer certificate of an
 // OCSP request out of a list of possible certificates.
-X509* x509_get_ocsp_signer(STACK_OF(X509) *certs, OCSP_RESPID *rid)
+X509* x509_get_ocsp_signer(const STACK_OF(X509)* certs,
+                           OCSP_BASICRESP* basic_resp)
 	{
-	// We support two lookup types - either by response id or by key.
+	const ASN1_OCTET_STRING* key  = nullptr;
-	if ( rid->type == V_OCSP_RESPID_NAME )
+	const X509_NAME*         name = nullptr;
-		return X509_find_by_subject(certs, rid->value.byName);
 
-	// There only should be name and type - but let's be sure...
+#if ( OPENSSL_VERSION_NUMBER < 0x10100000L )
-	if ( rid->type != V_OCSP_RESPID_KEY )
+	OCSP_RESPID* resp_id = basic_resp->tbsResponseData->responderId;
+
+	if ( resp_id->type == V_OCSP_RESPID_NAME )
+		name = resp_id->value.byName;
+	else if ( resp_id->type == V_OCSP_RESPID_KEY )
+		key = resp_id->value.byKey;
+	else
 		return 0;
+#else
+	if ( ! OCSP_resp_get0_id(basic_resp, &key, &name) )
+		return 0;
+#endif
+
+	if ( name )
+		return X509_find_by_subject(const_cast<STACK_OF(X509)*>(certs),
+									const_cast<X509_NAME*>(name));
 
 	// Just like OpenSSL, we just support SHA-1 lookups and bail out otherwhise.
-	if ( rid->value.byKey->length != SHA_DIGEST_LENGTH )
+	if ( key->length != SHA_DIGEST_LENGTH )
 		return 0;
 
-	unsigned char* key_hash = rid->value.byKey->data;
+	unsigned char* key_hash = key->data;
+
 	for ( int i = 0; i < sk_X509_num(certs); ++i )
 		{
 		unsigned char digest[SHA_DIGEST_LENGTH];
@@ -328,15 +343,19 @@ function x509_ocsp_verify%(certs: x509_opaque_vector, ocsp_reply: string, root_c
 
 	// Because we actually want to be able to give nice error messages that show why we were
 	// not able to verify the OCSP response - do our own verification logic first.
-	signer = x509_get_ocsp_signer(basic->certs, basic->tbsResponseData->responderId);
+#if ( OPENSSL_VERSION_NUMBER < 0x10100000L )
+	signer = x509_get_ocsp_signer(basic->certs, basic);
+#else
+	signer = x509_get_ocsp_signer(OCSP_resp_get0_certs(basic), basic);
+#endif
 
 	/*
 	Do this perhaps - OpenSSL also cannot do it, so I do not really feel bad about it.
 	Needs a different lookup because the root store is no stack of X509 certs
 
-	if ( !s igner )
+	if ( ! signer )
 		// if we did not find it in the certificates that were sent, search in the root store
-		signer = x509_get_ocsp_signer(ocsp_certs, basic->tbsResponseData->responderId);
+		signer = x509_get_ocsp_signer(ocsp_certs, basic);
 	*/
 
 	if ( ! signer )

src/file_analysis/analyzer/x509/ocsp_events.bif

@@ -7,13 +7,10 @@
 ##
 ## req: version: the version of the OCSP request. Typically 0 (Version 1).
 ##
-## requestorName: name of the OCSP requestor. This attribute is optional; if
-##                it is not set, an empty string is returned here.
-##
 ## .. bro:see:: ocsp_request_certificate ocsp_response_status
 ##              ocsp_response_bytes ocsp_response_certificate ocsp_extension
 ##              x509_ocsp_ext_signed_certificate_timestamp
-event ocsp_request%(f: fa_file, version: count, requestorName: string%);
+event ocsp_request%(f: fa_file, version: count%);
 
 ## Event that is raised when encountering an OCSP request for a certificate,
 ## e.g. in an HTTP connection. See :rfc:`6960` for more details.

src/iosource/Packet.cc

@@ -140,6 +140,20 @@ void Packet::ProcessLayer2()
 
 	case DLT_EN10MB:
 		{
+		// Skip past Cisco FabricPath to encapsulated ethernet frame.
+		if ( pdata[12] == 0x89 && pdata[13] == 0x03 )
+			{
+			auto constexpr cfplen = 16;
+
+			if ( pdata + cfplen + GetLinkHeaderSize(link_type) >= end_of_data )
+				{
+				Weird("truncated_link_header_cfp");
+				return;
+				}
+
+			pdata += cfplen;
+			}
+
 		// Get protocol being carried from the ethernet frame.
 		int protocol = (pdata[12] << 8) + pdata[13];
 

testing/btest/Baseline/bifs.x509_verify/stdout-openssl-1.1

@@ -0,0 +1,6 @@
+Validation result: certificate has expired
+Validation result: ok
+Resulting chain:
+Fingerprint: 70829f77ff4b6e908324a3f4e1940fce6c489098, Subject: CN=www.tobu-estate.com,OU=Terms of use at www.verisign.com/rpa (c)05,O=TOBU RAILWAY Co.\,Ltd.,L=Sumida-ku,ST=Tokyo,C=JP
+Fingerprint: 5deb8f339e264c19f6686f5f8f32b54a4c46b476, Subject: CN=VeriSign Class 3 Secure Server CA - G3,OU=Terms of use at https://www.verisign.com/rpa (c)10,OU=VeriSign Trust Network,O=VeriSign\, Inc.,C=US
+Fingerprint: 4eb6d578499b1ccf5f581ead56be3d9b6744a5e5, Subject: CN=VeriSign Class 3 Public Primary Certification Authority - G5,OU=(c) 2006 VeriSign\, Inc. - For authorized use only,OU=VeriSign Trust Network,O=VeriSign\, Inc.,C=US

testing/btest/Baseline/core.cisco-fabric-path/conn.log

@@ -0,0 +1,41 @@
+#separator \x09
+#set_separator	,
+#empty_field	(empty)
+#unset_field	-
+#path	conn
+#open	2018-07-09-14-17-29
+#fields	ts	uid	id.orig_h	id.orig_p	id.resp_h	id.resp_p	proto	service	duration	orig_bytes	resp_bytes	conn_state	local_orig	local_resp	missed_bytes	history	orig_pkts	orig_ip_bytes	resp_pkts	resp_ip_bytes	tunnel_parents
+#types	time	string	addr	port	addr	port	enum	string	interval	count	count	string	bool	bool	count	string	count	count	count	count	set[string]
+1529347003.860008	C7fIlMZDuRiqjpYbb	1.1.1.6	57005	2.2.2.2	48879	tcp	-	0.001018	0	0	S0	-	-	0	S	2	80	0	0	-
+1529347003.861732	CykQaM33ztNt0csB9a	1.1.1.4	57005	2.2.2.2	48879	tcp	-	0.000928	0	0	S0	-	-	0	S	2	80	0	0	-
+1529347003.863372	CtxTCR2Yer0FR1tIBg	1.1.1.14	57005	2.2.2.2	48879	tcp	-	0.000928	0	0	S0	-	-	0	S	2	80	0	0	-
+1529347003.865002	CpmdRlaUoJLN3uIRa	1.1.1.12	57005	2.2.2.2	48879	tcp	-	0.000926	0	0	S0	-	-	0	S	2	80	0	0	-
+1529347003.866648	C1Xkzz2MaGtLrc1Tla	1.1.1.0	57005	2.2.2.2	48879	tcp	-	0.001042	0	0	S0	-	-	0	S	2	80	0	0	-
+1529347003.868394	CqlVyW1YwZ15RhTBc4	1.1.1.2	57005	2.2.2.2	48879	tcp	-	0.000920	0	0	S0	-	-	0	S	2	80	0	0	-
+1529347003.870014	CLNN1k2QMum1aexUK7	1.1.1.8	57005	2.2.2.2	48879	tcp	-	0.000930	0	0	S0	-	-	0	S	2	80	0	0	-
+1529347003.871649	CBA8792iHmnhPLksKa	1.1.1.10	57005	2.2.2.2	48879	tcp	-	0.000928	0	0	S0	-	-	0	S	2	80	0	0	-
+1529347003.873385	CGLPPc35OzDQij1XX8	1234::e	57005	5678::	48879	tcp	-	0.001139	0	0	S0	-	-	0	S	2	120	0	0	-
+1529347003.875322	CiyBAq1bBLNaTiTAc	1234::c	57005	5678::	48879	tcp	-	0.001027	0	0	S0	-	-	0	S	2	120	0	0	-
+1529347003.877182	CFSwNi4CNGxcuffo49	1234::6	57005	5678::	48879	tcp	-	0.001055	0	0	S0	-	-	0	S	2	120	0	0	-
+1529347003.879034	Cipfzj1BEnhejw8cGf	1234::4	57005	5678::	48879	tcp	-	0.001018	0	0	S0	-	-	0	S	2	120	0	0	-
+1529347003.881330	CV5WJ42jPYbNW9JNWf	1234::8	57005	5678::	48879	tcp	-	0.001029	0	0	S0	-	-	0	S	2	120	0	0	-
+1529347003.883152	CPhDKt12KQPUVbQz06	1234::a	57005	5678::	48879	tcp	-	0.001005	0	0	S0	-	-	0	S	2	120	0	0	-
+1529347003.884945	CAnFrb2Cvxr5T7quOc	1234::	57005	5678::	48879	tcp	-	0.001005	0	0	S0	-	-	0	S	2	120	0	0	-
+1529347003.886751	C8rquZ3DjgNW06JGLl	1234::2	57005	5678::	48879	tcp	-	0.001120	0	0	S0	-	-	0	S	2	120	0	0	-
+1529347003.851951	CFLRIC3zaTU1loLGxh	1234::4	57005	5678::	48879	udp	-	0.000905	0	0	S0	-	-	0	D	2	96	0	0	-
+1529347003.855232	Ck51lg1bScffFj34Ri	1234::a	57005	5678::	48879	udp	-	0.000894	0	0	S0	-	-	0	D	2	96	0	0	-
+1529347003.839636	CtPZjS20MLrsMUOJi2	1.1.1.12	57005	2.2.2.2	48879	udp	-	0.000847	0	0	S0	-	-	0	D	2	56	0	0	-
+1529347003.858393	CNnMIj2QSd84NKf7U3	1234::2	57005	5678::	48879	udp	-	0.000902	0	0	S0	-	-	0	D	2	96	0	0	-
+1529347003.842649	CmES5u32sYpV7JYN	1.1.1.2	57005	2.2.2.2	48879	udp	-	0.000830	0	0	S0	-	-	0	D	2	56	0	0	-
+1529347003.850367	C0LAHyvtKSQHyJxIl	1234::6	57005	5678::	48879	udp	-	0.000898	0	0	S0	-	-	0	D	2	96	0	0	-
+1529347003.848776	CwjjYJ2WqgTbAqiHl6	1234::c	57005	5678::	48879	udp	-	0.000902	0	0	S0	-	-	0	D	2	96	0	0	-
+1529347003.856801	C9mvWx3ezztgzcexV7	1234::	57005	5678::	48879	udp	-	0.000898	0	0	S0	-	-	0	D	2	96	0	0	-
+1529347003.841103	CUM0KZ3MLUfNB0cl11	1.1.1.0	57005	2.2.2.2	48879	udp	-	0.000926	0	0	S0	-	-	0	D	2	56	0	0	-
+1529347003.845524	C37jN32gN3y3AZzyf6	1.1.1.10	57005	2.2.2.2	48879	udp	-	0.000843	0	0	S0	-	-	0	D	2	56	0	0	-
+1529347003.847079	C3eiCBGOLw3VtHfOj	1234::e	57005	5678::	48879	udp	-	0.001014	0	0	S0	-	-	0	D	2	96	0	0	-
+1529347003.853544	C9rXSW3KSpTYvPrlI1	1234::8	57005	5678::	48879	udp	-	0.001010	0	0	S0	-	-	0	D	2	96	0	0	-
+1529347003.836659	ClEkJM2Vm5giqnMf4h	1.1.1.4	57005	2.2.2.2	48879	udp	-	0.000847	0	0	S0	-	-	0	D	2	56	0	0	-
+1529347003.838130	C4J4Th3PJpwUYZZ6gc	1.1.1.14	57005	2.2.2.2	48879	udp	-	0.000880	0	0	S0	-	-	0	D	2	56	0	0	-
+1529347003.844086	CP5puj4I8PtEU4qzYg	1.1.1.8	57005	2.2.2.2	48879	udp	-	0.000830	0	0	S0	-	-	0	D	2	56	0	0	-
+1529347003.834704	CHhAvVGS1DHFjwGM9	1.1.1.6	57005	2.2.2.2	48879	udp	-	0.001243	0	0	S0	-	-	0	D	2	56	0	0	-
+#close	2018-07-09-14-17-29

testing/btest/bifs/x509_verify.bro

@@ -1,5 +1,14 @@
 # @TEST-EXEC: bro -r $TRACES/tls/tls-expired-cert.trace %INPUT
-# @TEST-EXEC: btest-diff .stdout
+
+# This is a hack: the results of OpenSSL 1.1's vs 1.0's
+# X509_verify_cert() -> X509_STORE_CTX_get1_chain() calls
+# differ.  Word seems to be that OpenSSL 1.1's cert-chain-building
+# code is significantly different/rewritten so may be the reason...
+
+# @TEST-EXEC: cp .stdout stdout-openssl-1.0
+# @TEST-EXEC: cp .stdout stdout-openssl-1.1
+
+# @TEST-EXEC: grep -q "BRO_HAVE_OPENSSL_1_1" $BUILD/CMakeCache.txt && btest-diff stdout-openssl-1.1 || btest-diff stdout-openssl-1.0
 
 redef SSL::root_certs += {
 	["OU=Class 3 Public Primary Certification Authority,O=VeriSign\, Inc.,C=US"] = "\x30\x82\x02\x3C\x30\x82\x01\xA5\x02\x10\x70\xBA\xE4\x1D\x10\xD9\x29\x34\xB6\x38\xCA\x7B\x03\xCC\xBA\xBF\x30\x0D\x06\x09\x2A\x86\x48\x86\xF7\x0D\x01\x01\x02\x05\x00\x30\x5F\x31\x0B\x30\x09\x06\x03\x55\x04\x06\x13\x02\x55\x53\x31\x17\x30\x15\x06\x03\x55\x04\x0A\x13\x0E\x56\x65\x72\x69\x53\x69\x67\x6E\x2C\x20\x49\x6E\x63\x2E\x31\x37\x30\x35\x06\x03\x55\x04\x0B\x13\x2E\x43\x6C\x61\x73\x73\x20\x33\x20\x50\x75\x62\x6C\x69\x63\x20\x50\x72\x69\x6D\x61\x72\x79\x20\x43\x65\x72\x74\x69\x66\x69\x63\x61\x74\x69\x6F\x6E\x20\x41\x75\x74\x68\x6F\x72\x69\x74\x79\x30\x1E\x17\x0D\x39\x36\x30\x31\x32\x39\x30\x30\x30\x30\x30\x30\x5A\x17\x0D\x32\x38\x30\x38\x30\x31\x32\x33\x35\x39\x35\x39\x5A\x30\x5F\x31\x0B\x30\x09\x06\x03\x55\x04\x06\x13\x02\x55\x53\x31\x17\x30\x15\x06\x03\x55\x04\x0A\x13\x0E\x56\x65\x72\x69\x53\x69\x67\x6E\x2C\x20\x49\x6E\x63\x2E\x31\x37\x30\x35\x06\x03\x55\x04\x0B\x13\x2E\x43\x6C\x61\x73\x73\x20\x33\x20\x50\x75\x62\x6C\x69\x63\x20\x50\x72\x69\x6D\x61\x72\x79\x20\x43\x65\x72\x74\x69\x66\x69\x63\x61\x74\x69\x6F\x6E\x20\x41\x75\x74\x68\x6F\x72\x69\x74\x79\x30\x81\x9F\x30\x0D\x06\x09\x2A\x86\x48\x86\xF7\x0D\x01\x01\x01\x05\x00\x03\x81\x8D\x00\x30\x81\x89\x02\x81\x81\x00\xC9\x5C\x59\x9E\xF2\x1B\x8A\x01\x14\xB4\x10\xDF\x04\x40\xDB\xE3\x57\xAF\x6A\x45\x40\x8F\x84\x0C\x0B\xD1\x33\xD9\xD9\x11\xCF\xEE\x02\x58\x1F\x25\xF7\x2A\xA8\x44\x05\xAA\xEC\x03\x1F\x78\x7F\x9E\x93\xB9\x9A\x00\xAA\x23\x7D\xD6\xAC\x85\xA2\x63\x45\xC7\x72\x27\xCC\xF4\x4C\xC6\x75\x71\xD2\x39\xEF\x4F\x42\xF0\x75\xDF\x0A\x90\xC6\x8E\x20\x6F\x98\x0F\xF8\xAC\x23\x5F\x70\x29\x36\xA4\xC9\x86\xE7\xB1\x9A\x20\xCB\x53\xA5\x85\xE7\x3D\xBE\x7D\x9A\xFE\x24\x45\x33\xDC\x76\x15\xED\x0F\xA2\x71\x64\x4C\x65\x2E\x81\x68\x45\xA7\x02\x03\x01\x00\x01\x30\x0D\x06\x09\x2A\x86\x48\x86\xF7\x0D\x01\x01\x02\x05\x00\x03\x81\x81\x00\xBB\x4C\x12\x2B\xCF\x2C\x26\x00\x4F\x14\x13\xDD\xA6\xFB\xFC\x0A\x11\x84\x8C\xF3\x28\x1C\x67\x92\x2F\x7C\xB6\xC5\xFA\xDF\xF0\xE8\x95\xBC\x1D\x8F\x6C\x2C\xA8\x51\xCC\x73\xD8\xA4\xC0\x53\xF0\x4E\xD6\x26\xC0\x76\x01\x57\x81\x92\x5E\x21\xF1\xD1\xB1\xFF\xE7\xD0\x21\x58\xCD\x69\x17\xE3\x44\x1C\x9C\x19\x44\x39\x89\x5C\xDC\x9C\x00\x0F\x56\x8D\x02\x99\xED\xA2\x90\x45\x4C\xE4\xBB\x10\xA4\x3D\xF0\x32\x03\x0E\xF1\xCE\xF8\xE8\xC9\x51\x8C\xE6\x62\x9F\xE6\x9F\xC0\x7D\xB7\x72\x9C\xC9\x36\x3A\x6B\x9F\x4E\xA8\xFF\x64\x0D\x64"

testing/btest/core/cisco-fabric-path.bro

@@ -0,0 +1,2 @@
+# @TEST-EXEC: bro -C -r $TRACES/cisco-fabric-path.pcap
+# @TEST-EXEC: btest-diff conn.log

testing/btest/scripts/base/protocols/rdp/rdp-x509.bro

@@ -1,5 +1,5 @@
 # @TEST-EXEC: bro -r $TRACES/rdp/rdp-x509.pcap %INPUT
 # @TEST-EXEC: btest-diff rdp.log
-# @TEST-EXEC: btest-diff x509.log
+# @TEST-EXEC: TEST_DIFF_CANONIFIER="$SCRIPTS/diff-remove-timestamps | $SCRIPTS/diff-remove-x509-key-info" btest-diff x509.log
 
 @load base/protocols/rdp

testing/btest/scripts/base/protocols/ssl/ocsp-http-get.test

@@ -17,9 +17,9 @@ event ocsp_extension(f: fa_file, ext: X509::Extension, global_resp: bool)
 	print "extension: ", ext, global_resp;
 	}
 
-event ocsp_request(f: fa_file, version: count, requestorName: string)
+event ocsp_request(f: fa_file, version: count)
 	{
-	print "request", version, requestorName;
+	print "request", version, "";
 	}
 
 event ocsp_request_certificate(f: fa_file, hashAlgorithm: string, issuerNameHash: string, issuerKeyHash: string, serialNumber: string)

testing/btest/scripts/base/protocols/ssl/ocsp-request-only.test

@@ -16,9 +16,9 @@ event ocsp_extension(f: fa_file, ext: X509::Extension, global_resp: bool)
 	print "extension: ", ext, global_resp;
 	}
 
-event ocsp_request(f: fa_file, version: count, requestorName: string)
+event ocsp_request(f: fa_file, version: count)
 	{
-	print "request", version, requestorName;
+	print "request", version, "";
 	}
 
 event ocsp_request_certificate(f: fa_file, hashAlgorithm: string, issuerNameHash: string, issuerKeyHash: string, serialNumber: string)

testing/btest/scripts/base/protocols/ssl/ocsp-request-response.test

@@ -17,9 +17,9 @@ event ocsp_extension(f: fa_file, ext: X509::Extension, global_resp: bool)
 	print "extension: ", ext, global_resp;
 	}
 
-event ocsp_request(f: fa_file, version: count, requestorName: string)
+event ocsp_request(f: fa_file, version: count)
 	{
-	print "request", version, requestorName;
+	print "request", version, "";
 	}
 
 event ocsp_request_certificate(f: fa_file, hashAlgorithm: string, issuerNameHash: string, issuerKeyHash: string, serialNumber: string)

testing/btest/scripts/base/protocols/ssl/ocsp-response-only.test

@@ -17,9 +17,9 @@ event ocsp_extension(f: fa_file, ext: X509::Extension, global_resp: bool)
 	print "extension: ", ext, global_resp;
 	}
 
-event ocsp_request(f: fa_file, version: count, requestorName: string)
+event ocsp_request(f: fa_file, version: count)
 	{
-	print "request", version, requestorName;
+	print "request", version, "";
 	}
 
 event ocsp_request_certificate(f: fa_file, hashAlgorithm: string, issuerNameHash: string, issuerKeyHash: string, serialNumber: string)

testing/btest/scripts/base/protocols/ssl/ocsp-revoked.test

@@ -17,9 +17,9 @@ event ocsp_extension(f: fa_file, ext: X509::Extension, global_resp: bool)
 	print "extension: ", ext, global_resp;
 	}
 
-event ocsp_request(f: fa_file, version: count, requestorName: string)
+event ocsp_request(f: fa_file, version: count)
 	{
-	print "request", version, requestorName;
+	print "request", version, "";
 	}
 
 event ocsp_request_certificate(f: fa_file, hashAlgorithm: string, issuerNameHash: string, issuerKeyHash: string, serialNumber: string)

testing/scripts/diff-remove-x509-key-info

@@ -0,0 +1,55 @@
+#! /usr/bin/env bash
+#
+# A diff canonifier that removes all X.509 public key information
+# which, in the specific case of the RDP protocol's misuse of
+# md5WithRSAEncryption, seems that OpenSSL 1.0 is able to manually
+# workaround by setting to rsaEncryption, but OpenSSL 1.1 still fails
+# to extract the key, so the corresponding fields are always removed here.
+
+awk '
+BEGIN { FS="\t"; OFS="\t"; key_type_col = -1; key_length_col = -1; exponent_col = -1; curve_col = -1 }
+
+/^#/ {
+    if ( $1 == "#fields" )
+        {
+        for ( i = 2; i <= NF; ++i )
+            {
+            if ( $i == "certificate.key_type" )
+                key_type_col = i-1;
+            if ( $i == "certificate.key_length" )
+                key_length_col = i-1;
+            if ( $i == "certificate.exponent" )
+                exponent_col = i-1;
+            if ( $i == "certificate.curve" )
+                curve_col = i-1;
+            }
+        }
+
+    print;
+    next;
+}
+
+key_type_col > 0 {
+        # Mark it regardless of whether it is set.
+        $key_type_col = "x";
+}
+
+key_length_col > 0 {
+        # Mark it regardless of whether it is set.
+        $key_length_col = "x";
+}
+
+exponent_col > 0 {
+        # Mark it regardless of whether it is set.
+        $exponent_col = "x";
+}
+
+curve_col > 0 {
+        # Mark it regardless of whether it is set.
+        $curve_col = "x";
+}
+
+{
+    print;
+}
+'