diff --git a/CHANGES b/CHANGES index 6b5905ee3f..07e64b4a0f 100644 --- a/CHANGES +++ b/CHANGES @@ -1,4 +1,36 @@ +2.5-792 | 2018-08-01 12:15:31 -0500 + + * fix NTLM NegotiateFlags field offsets (Jeffrey Bencteux) + +2.5-790 | 2018-08-01 11:25:27 -0500 + + * Fix --with-binpac configure option (Jon Siwek, Corelight) + + * Update CAF-finding logic (Jon Siwek, Corelight) + +2.5-787 | 2018-07-31 16:50:55 -0500 + + * Add Cisco FabricPath support (Damani Wade, Corelight) + + * Replace GeoIP Legacy DB support with MaxMind DB support + (Jonathan Perkins, Corelight) + +2.5-782 | 2018-07-31 11:53:22 +0200 + + * Update install instructions for OpenSSL 1.1 compat (Jon Siwek, Corelight) + + * Remove requestorName parameter of ocsp_request event + + This field isn't publicly available via the OpenSSL 1.1 API, not used + in the base scripts, and has no example in the test suit, so removing + it is simpler than trying to support manually parsing it out of the + raw data. (Jon Siwek, Corelight) + + * Adjust x509 unit tests to work around OpenSSL 1.0 vs. 1.1 differences (Jon Siwek, Corelight) + + * Fixes for OpenSSL 1.1 support (Jon Siwek, Corelight) + 2.5-775 | 2018-07-24 16:39:34 -0500 * Add broker/binpac/caf dirs to bro-config script (Jon Siwek, Corelight) diff --git a/CMakeLists.txt b/CMakeLists.txt index 5f4f949a6f..452f2834cc 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -95,14 +95,15 @@ FindRequiredPackage(OpenSSL) FindRequiredPackage(BIND) FindRequiredPackage(ZLIB) -find_package(CAF COMPONENTS core io) +find_package(CAF COMPONENTS core io openssl) if (CAF_FOUND) + # e.g. if not using embedded CAF, then need to know where to look + # for CAF headers since that may differ from where Broker headers + # are found (and including a Broker header may pull in CAF headers). include_directories(BEFORE ${CAF_INCLUDE_DIRS}) -else () - list(APPEND MISSING_PREREQ_DESCS CAF) endif () -if (NOT BinPAC_ROOT_DIR AND +if (NOT BINPAC_EXE_PATH AND EXISTS ${CMAKE_CURRENT_SOURCE_DIR}/aux/binpac/CMakeLists.txt) add_subdirectory(aux/binpac) endif () @@ -140,11 +141,11 @@ include_directories(BEFORE # Optional Dependencies set(USE_GEOIP false) -find_package(LibGeoIP) -if (LIBGEOIP_FOUND) +find_package(LibMMDB) +if (LibMMDB_FOUND) set(USE_GEOIP true) - include_directories(BEFORE ${LibGeoIP_INCLUDE_DIR}) - list(APPEND OPTLIBS ${LibGeoIP_LIBRARY}) + include_directories(BEFORE ${LibMMDB_INCLUDE_DIR}) + list(APPEND OPTLIBS ${LibMMDB_LIBRARY}) endif () set(USE_KRB5 false) @@ -224,6 +225,10 @@ include(CheckNameserCompat) include(GetArchitecture) include(RequireCXX11) +if ( (OPENSSL_VERSION VERSION_EQUAL "1.1.0") OR (OPENSSL_VERSION VERSION_GREATER "1.1.0") ) + set(BRO_HAVE_OPENSSL_1_1 true CACHE INTERNAL "" FORCE) +endif() + # Tell the plugin code that we're building as part of the main tree. set(BRO_PLUGIN_INTERNAL_BUILD true CACHE INTERNAL "" FORCE) @@ -323,7 +328,7 @@ message( "\nBroctl: ${INSTALL_BROCTL}" "\nAux. Tools: ${INSTALL_AUX_TOOLS}" "\n" - "\nGeoIP: ${USE_GEOIP}" + "\nlibmaxminddb: ${USE_GEOIP}" "\nKerberos: ${USE_KRB5}" "\ngperftools found: ${HAVE_PERFTOOLS}" "\n tcmalloc: ${USE_PERFTOOLS_TCMALLOC}" diff --git a/NEWS b/NEWS index c8ec3d281f..5a2ad70c56 100644 --- a/NEWS +++ b/NEWS @@ -281,6 +281,8 @@ New Functionality - Bro now supports PPPoE over QinQ. +- Bro now supports OpenSSL 1.1. + Changed Functionality --------------------- @@ -360,6 +362,15 @@ Changed Functionality - The string_to_pattern() built-in (and the now-deprecated merge_pattern() built-in) is no longer restricted to only be called at initialization time. +- GeoIP Legacy Database support has been replaced with GeoIP2 MaxMind DB + format support. + + - This updates the "lookup_location" and "lookup_asn" BIFs to use + libmaxminddb. The motivation for this is that MaxMind is discontinuing + GeoLite Legacy databases: no updates after April 1, 2018, no downloads + after January 2, 2019. It's also noted that all GeoIP Legacy databases + may be discontinued as they are superseded by GeoIP2. + Removed Functionality --------------------- @@ -372,6 +383,8 @@ Removed Functionality - BroControl: The option 'IPv6Comm' and 'ZoneID' options are no longer available (though Broker should be able to handle IPv6 automatically). +- The "ocsp_request" event no longer has "requestorName" parameter. + Deprecated Functionality ------------------------ diff --git a/VERSION b/VERSION index 4ef74ff31b..67eaa8d910 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -2.5-775 +2.5-792 diff --git a/aux/bifcl b/aux/bifcl index 7fffb6bcf1..f648ad79b2 160000 --- a/aux/bifcl +++ b/aux/bifcl @@ -1 +1 @@ -Subproject commit 7fffb6bcf1ea78331ca2fc0e7d16642ec2463ca0 +Subproject commit f648ad79b20baba4f80259d059044ae78d56d7c4 diff --git a/aux/binpac b/aux/binpac index 4a06b4cf3c..7e6b47ee90 160000 --- a/aux/binpac +++ b/aux/binpac @@ -1 +1 @@ -Subproject commit 4a06b4cf3ce2112426c306cd358753793167bbeb +Subproject commit 7e6b47ee90c5b6ab80b0e6f93d5cf835fd86ce4e diff --git a/aux/bro-aux b/aux/bro-aux index c633a852ec..1b27ec4c24 160000 --- a/aux/bro-aux +++ b/aux/bro-aux @@ -1 +1 @@ -Subproject commit c633a852ec0bb983e74c0f47b73d03a30b7f1259 +Subproject commit 1b27ec4c24bb13443f1a5e8f4249ff4e20e06dd1 diff --git a/aux/broccoli b/aux/broccoli index a4a859a7a8..80a4aa6892 160000 --- a/aux/broccoli +++ b/aux/broccoli @@ -1 +1 @@ -Subproject commit a4a859a7a87363167239205bd07fedced0c584a3 +Subproject commit 80a4aa68927c2f60ece1200268106edc27f50338 diff --git a/aux/broctl b/aux/broctl index 206fc8254a..d900149ef6 160000 --- a/aux/broctl +++ b/aux/broctl @@ -1 +1 @@ -Subproject commit 206fc8254a4049dfad420d5571704de4a06444bd +Subproject commit d900149ef6e2f744599a9575b67fb7155953bd4a diff --git a/aux/broker b/aux/broker index b5508f387b..488dc806d0 160000 --- a/aux/broker +++ b/aux/broker @@ -1 +1 @@ -Subproject commit b5508f387bb6c24c95f23de479184869b9836769 +Subproject commit 488dc806d0f777dcdee04f3794f04121ded08b8e diff --git a/bro-config.h.in b/bro-config.h.in index 19ab863a3f..1b351e550f 100644 --- a/bro-config.h.in +++ b/bro-config.h.in @@ -111,12 +111,6 @@ /* Define if KRB5 is available */ #cmakedefine USE_KRB5 -/* Whether the found GeoIP API supports IPv6 Country Edition */ -#cmakedefine HAVE_GEOIP_COUNTRY_EDITION_V6 - -/* Whether the found GeoIP API supports IPv6 City Edition */ -#cmakedefine HAVE_GEOIP_CITY_EDITION_REV0_V6 - /* Use Google's perftools */ #cmakedefine USE_PERFTOOLS_DEBUG diff --git a/cmake b/cmake index c4c2d6eeb7..ec66fd8fbd 160000 --- a/cmake +++ b/cmake @@ -1 +1 @@ -Subproject commit c4c2d6eeb7609836255abb054b91c4039bb7e276 +Subproject commit ec66fd8fbd81fd8b25ced0214016f4c89604a15c diff --git a/configure b/configure index 830a70b60f..46aaeb3bbd 100755 --- a/configure +++ b/configure @@ -72,7 +72,7 @@ Usage: $0 [OPTION]... [VAR=VALUE]... (a required Broker dependency) Optional Packages in Non-Standard Locations: - --with-geoip=PATH path to the libGeoIP install root + --with-geoip=PATH path to the libmaxminddb install root --with-krb5=PATH path to krb5 install root --with-perftools=PATH path to Google Perftools install root --with-jemalloc=PATH path to jemalloc install root @@ -241,6 +241,7 @@ while [ $# -ne 0 ]; do ;; --with-binpac=*) append_cache_entry BINPAC_EXE_PATH PATH $optarg + append_cache_entry BinPAC_ROOT_DIR PATH "$(dirname $optarg)/.." ;; --with-bifcl=*) append_cache_entry BIFCL_EXE_PATH PATH $optarg @@ -252,7 +253,7 @@ while [ $# -ne 0 ]; do append_cache_entry BISON_EXECUTABLE PATH $optarg ;; --with-geoip=*) - append_cache_entry LibGeoIP_ROOT_DIR PATH $optarg + append_cache_entry LibMMDB_ROOT_DIR PATH $optarg ;; --with-krb5=*) append_cache_entry LibKrb5_ROOT_DIR PATH $optarg diff --git a/doc/frameworks/geoip.rst b/doc/frameworks/geoip.rst index d756f97589..d826aabff6 100644 --- a/doc/frameworks/geoip.rst +++ b/doc/frameworks/geoip.rst @@ -8,11 +8,13 @@ GeoLocation .. rst-class:: opening During the process of creating policy scripts the need may arise - to find the geographic location for an IP address. Bro has support + to find the geographic location for an IP address. Bro had support for the `GeoIP library `__ at the - policy script level beginning with release 1.3 to account for this - need. To use this functionality, you need to first install the libGeoIP - software, and then install the GeoLite city database before building + policy script level from release 1.3 to 2.5.X to account for this + need. Starting with release 2.6 GeoIP support requires `libmaxminddb + `__. + To use this functionality, you need to first install the libmaxminddb + software, and then install the GeoLite2 city database before building Bro. .. contents:: @@ -20,85 +22,91 @@ GeoLocation Install libGeoIP ---------------- -Before building Bro, you need to install libGeoIP. +Before building Bro, you need to install libmaxminddb. * FreeBSD: .. console:: - sudo pkg install GeoIP + sudo pkg install libmaxminddb * RPM/RedHat-based Linux: .. console:: - sudo yum install GeoIP-devel + sudo yum install libmaxminddb-devel * DEB/Debian-based Linux: .. console:: - sudo apt-get install libgeoip-dev + sudo apt-get install libmaxminddb-dev * Mac OS X: You need to install from your preferred package management system (e.g. MacPorts, Fink, or Homebrew). The name of the package that you need - may be libgeoip, geoip, or geoip-dev, depending on which package management - system you are using. + may be libmaxminddb, maxminddb, or libmaxminddb-dev, depending on which + package management system you are using. -GeoIPLite Database Installation -------------------------------- +GeoLite2-City Database Installation +----------------------------------- -A country database for GeoIPLite is included when you do the C API -install, but for Bro, we are using the city database which includes -cities and regions in addition to countries. +Bro can use the city or country database. The city database includes cities +and regions in addition to countries. -`Download `__ the GeoLite city -binary database: +`Download `__ +the GeoLite2 city binary database: .. console:: - wget http://geolite.maxmind.com/download/geoip/database/GeoLiteCity.dat.gz - gunzip GeoLiteCity.dat.gz + wget http://geolite.maxmind.com/download/geoip/database/GeoLite2-City.tar.gz + tar zxf GeoLite2-City.tar.gz -Next, the file needs to be renamed and put in the GeoIP database directory. -This directory should already exist and will vary depending on which platform -and package you are using. For FreeBSD, use ``/usr/local/share/GeoIP``. For -Linux, use ``/usr/share/GeoIP`` or ``/var/lib/GeoIP`` (choose whichever one -already exists). +Next, the file "GeoLite2-City_YYYYMMDD/GeoLite2-City.mmdb" needs to be renamed +and put in the GeoIP database directory. This directory should already exist +and will vary depending on which platform and package you are using. For +FreeBSD, use ``/usr/local/share/GeoIP``. For Linux, use ``/usr/share/GeoIP`` +or ``/var/lib/GeoIP`` (choose whichever one already exists). .. console:: - mv GeoLiteCity.dat /GeoIPCity.dat - -Note that there is a separate database for IPv6 addresses, which can also -be installed if you want GeoIP functionality for IPv6. + mv /GeoLite2-City.mmdb /GeoLite2-City.mmdb Testing ------- Before using the GeoIP functionality, it is a good idea to verify that -everything is setup correctly. After installing libGeoIP and the GeoIP city -database, and building Bro, you can quickly check if the GeoIP functionality -works by running a command like this: +everything is setup correctly. After installing libmaxminddb and the GeoIP +city database, and building Bro, you can quickly check if the GeoIP +functionality works by running a command like this: .. console:: bro -e "print lookup_location(8.8.8.8);" -If you see an error message similar to "Failed to open GeoIP City database", -then you may need to either rename or move your GeoIP city database file (the -error message should give you the full pathname of the database file that -Bro is looking for). +If you see an error message similar to "Failed to open GeoIP location +database", then you may need to either rename or move your GeoIP +location database file. Bro looks for location database files in the +following order by default: + + /usr/share/GeoIP/GeoLite2-City.mmdb + /var/lib/GeoIP/GeoLite2-City.mmdb + /usr/local/share/GeoIP/GeoLite2-City.mmdb + /usr/local/var/GeoIP/GeoLite2-City.mmdb + /usr/share/GeoIP/GeoLite2-Country.mmdb + /var/lib/GeoIP/GeoLite2-Country.mmdb + /usr/local/share/GeoIP/GeoLite2-Country.mmdb + /usr/local/var/GeoIP/GeoLite2-Country.mmdb If you see an error message similar to "Bro was not configured for GeoIP -support", then you need to rebuild Bro and make sure it is linked against -libGeoIP. Normally, if libGeoIP is installed correctly then it should -automatically be found when building Bro. If this doesn't happen, then -you may need to specify the path to the libGeoIP installation -(e.g. ``./configure --with-geoip=``). +support", then you either need to rebuild Bro and make sure it is linked +against libmaxminddb or else set the :bro:see:`mmdb_dir`` value +correctly. Normally, if libmaxminddb is installed correctly then it +should automatically be found when building Bro. If this doesn't +happen, then you may need to specify the path to the libmaxminddb +installation (e.g. ``./configure --with-geoip=``). Usage ----- diff --git a/doc/install/install.rst b/doc/install/install.rst index fd05ac9d69..cc8d81b14f 100644 --- a/doc/install/install.rst +++ b/doc/install/install.rst @@ -54,18 +54,12 @@ To install the required dependencies, you can use: sudo yum install cmake make gcc gcc-c++ flex bison libpcap-devel openssl-devel python-devel swig zlib-devel - In order to build Bro on Fedora 26, install ``compat-openssl10-devel`` instead - of ``openssl-devel``. - * DEB/Debian-based Linux: .. console:: sudo apt-get install cmake make gcc g++ flex bison libpcap-dev libssl-dev python-dev swig zlib1g-dev - In order to build Bro on Debian 9 or Ubuntu 18.04, install ``libssl1.0-dev`` - instead of ``libssl-dev``. - * FreeBSD: Most required dependencies should come with a minimal FreeBSD install diff --git a/scripts/base/init-bare.bro b/scripts/base/init-bare.bro index c502607cbd..b39490ed7e 100644 --- a/scripts/base/init-bare.bro +++ b/scripts/base/init-bare.bro @@ -872,6 +872,9 @@ type geo_location: record { longitude: double &optional; ##< Longitude. } &log; +## The directory containing MaxMind DB (*.mmdb) files to use for GeoIP support. +const mmdb_dir: string = "" &redef; + ## Computed entropy values. The record captures a number of measures that are ## computed in parallel. See `A Pseudorandom Number Sequence Test Program ## `_ for more information, Bro uses the same diff --git a/src/analyzer/protocol/ntlm/ntlm-protocol.pac b/src/analyzer/protocol/ntlm/ntlm-protocol.pac index 8862be1f22..f8784c74c7 100644 --- a/src/analyzer/protocol/ntlm/ntlm-protocol.pac +++ b/src/analyzer/protocol/ntlm/ntlm-protocol.pac @@ -171,7 +171,7 @@ type NTLM_Negotiate_Flags = record { request_non_nt_session_key : bool = (flags & 0x00400000) > 0; negotiate_identify : bool = (flags & 0x00100000) > 0; - negotiate_extended_sessionsecurity : bool = (flags & 0x00040000) > 0; + negotiate_extended_sessionsecurity : bool = (flags & 0x00080000) > 0; target_type_server : bool = (flags & 0x00020000) > 0; target_type_domain : bool = (flags & 0x00010000) > 0; @@ -179,14 +179,14 @@ type NTLM_Negotiate_Flags = record { negotiate_oem_workstation_supplied : bool = (flags & 0x00002000) > 0; negotiate_oem_domain_supplied : bool = (flags & 0x00001000) > 0; - negotiate_anonymous_connection : bool = (flags & 0x00000400) > 0; - negotiate_ntlm : bool = (flags & 0x00000100) > 0; + negotiate_anonymous_connection : bool = (flags & 0x00000800) > 0; + negotiate_ntlm : bool = (flags & 0x00000200) > 0; negotiate_lm_key : bool = (flags & 0x00000080) > 0; negotiate_datagram : bool = (flags & 0x00000040) > 0; negotiate_seal : bool = (flags & 0x00000020) > 0; + negotiate_sign : bool = (flags & 0x00000010) > 0; - negotiate_sign : bool = (flags & 0x00000008) > 0; request_target : bool = (flags & 0x00000004) > 0; negotiate_oem : bool = (flags & 0x00000002) > 0; negotiate_unicode : bool = (flags & 0x00000001) > 0; diff --git a/src/bro.bif b/src/bro.bif index c7b26dea04..d6cacec96d 100644 --- a/src/bro.bif +++ b/src/bro.bif @@ -3599,77 +3599,269 @@ function lookup_hostname%(host: string%) : addr_set %%{ #ifdef USE_GEOIP extern "C" { -#include +#include +#include +#include +#include +#include } -static GeoIP* open_geoip_db(GeoIPDBTypes type) +class MMDB { +public: + MMDB(const char* filename); + + ~MMDB(); + + MMDB_lookup_result_s Lookup(const struct sockaddr* const sa); + +private: + MMDB_s mmdb; +}; + +MMDB::MMDB(const char *filename) { - GeoIP* geoip = 0; + int status = MMDB_open(filename, MMDB_MODE_MMAP, &mmdb); - if ( GeoIP_db_avail(type) ) - geoip = GeoIP_open_type(type, GEOIP_MEMORY_CACHE); - - return geoip; + if ( MMDB_SUCCESS != status ) + { + throw std::runtime_error(MMDB_strerror(status)); + } } -static GeoIP* open_geoip_city_db() +MMDB::~MMDB() { - GeoIP* geoip = open_geoip_db(GEOIP_CITY_EDITION_REV0); - - if ( ! geoip ) - geoip = open_geoip_db(GEOIP_CITY_EDITION_REV1); - - if ( ! geoip ) - { - string rev0_path = GeoIPDBFileName[GEOIP_CITY_EDITION_REV0]; - string rev1_path = GeoIPDBFileName[GEOIP_CITY_EDITION_REV1]; - string db_path = rev0_path; - - // Maybe in the future the revisions won't share a common default path. - if ( rev0_path != rev1_path ) - db_path = rev0_path + " or " + rev1_path; - - reporter->Info("Failed to open GeoIP City database: %s", - db_path.c_str()); - } - - return geoip; + MMDB_close(&mmdb); } -static GeoIP* open_geoip_city_db_v6() +MMDB_lookup_result_s MMDB::Lookup(const struct sockaddr* const sa) { - GeoIP* geoip = 0; + int mmdb_error; + MMDB_lookup_result_s result = MMDB_lookup_sockaddr(&mmdb, sa, &mmdb_error); - // Both city edition revisions for IPv6 show up in libGeoIP 1.4.7. -#ifdef HAVE_GEOIP_CITY_EDITION_REV0_V6 - geoip = open_geoip_db(GEOIP_CITY_EDITION_REV0_V6); - - if ( ! geoip ) - geoip = open_geoip_db(GEOIP_CITY_EDITION_REV1_V6); - - if ( ! geoip ) + if ( MMDB_SUCCESS != mmdb_error ) { - string rev0_path = GeoIPDBFileName[GEOIP_CITY_EDITION_REV0_V6]; - string rev1_path = GeoIPDBFileName[GEOIP_CITY_EDITION_REV1_V6]; - string db_path = rev0_path; - - // Maybe in the future the revisions won't share a common default path. - if ( rev0_path != rev1_path ) - db_path = rev0_path + " or " + rev1_path; - - reporter->Info("Failed to open GeoIP Cityv6 database: %s", - db_path.c_str()); + throw std::runtime_error(MMDB_strerror(mmdb_error)); } -#endif - return geoip; + return result; + } + +std::unique_ptr mmdb_loc; +std::unique_ptr mmdb_asn; + +static bool mmdb_open(const char* filename, bool asn) + { + struct stat buf; + + if ( 0 != stat(filename, &buf) ) + { + return false; + } + + try + { + if ( asn ) + { + mmdb_asn.reset(new MMDB(filename)); + } + else + { + mmdb_loc.reset(new MMDB(filename)); + } + } + + catch ( const std::exception& e ) + { + reporter->Info("Failed to open MaxMind DB: %s [%s]", filename, + e.what()); + return false; + } + + return true; + } + +static bool mmdb_open_loc(const char* filename) + { + return mmdb_open(filename, false); + } + +static bool mmdb_open_asn(const char* filename) + { + return mmdb_open(filename, true); + } + +static bool mmdb_lookup(const IPAddr& addr, MMDB_lookup_result_s& result, + bool asn) + { + struct sockaddr_storage ss = {0}; + + if ( IPv4 == addr.GetFamily() ) + { + struct sockaddr_in* sa = (struct sockaddr_in*)&ss; + sa->sin_family = AF_INET; + addr.CopyIPv4(&sa->sin_addr); + } + + else + { + struct sockaddr_in6* sa = (struct sockaddr_in6*)&ss; + sa->sin6_family = AF_INET6; + addr.CopyIPv6(&sa->sin6_addr); + } + + try + { + result = asn ? mmdb_asn->Lookup((struct sockaddr*)&ss) + : mmdb_loc->Lookup((struct sockaddr*)&ss); + } + + catch ( const std::exception& e ) + { + reporter->Info("MaxMind DB lookup location error [%s]", + e.what()); + return false; + } + + return result.found_entry; + } + +static bool mmdb_lookup_loc(const IPAddr& addr, MMDB_lookup_result_s& result) + { + return mmdb_lookup(addr, result, false); + } + +static bool mmdb_lookup_asn(const IPAddr& addr, MMDB_lookup_result_s& result) + { + return mmdb_lookup(addr, result, true); + } + +static Val* mmdb_getvalue(MMDB_entry_data_s* entry_data, int status, + int data_type ) + { + switch (status) + { + case MMDB_SUCCESS: + if ( entry_data->has_data ) + { + switch (data_type) + { + case MMDB_DATA_TYPE_UTF8_STRING: + return new StringVal(entry_data->data_size, + entry_data->utf8_string); + break; + + case MMDB_DATA_TYPE_DOUBLE: + return new Val(entry_data->double_value, TYPE_DOUBLE); + break; + + case MMDB_DATA_TYPE_UINT32: + return new Val(entry_data->uint32, TYPE_COUNT); + + default: + break; + } + } + break; + + case MMDB_LOOKUP_PATH_DOES_NOT_MATCH_DATA_ERROR: + // key doesn't exist, nothing to do + break; + + default: + reporter->Info("MaxMind DB error [%s]", MMDB_strerror(status)); + break; + } + + return nullptr; + } + +static bool mmdb_try_open_loc () + { + // City database is always preferred over Country database. + auto mmdb_dir_val = global_scope()->Lookup("mmdb_dir")->ID_Val(); + std::string mmdb_dir = mmdb_dir_val->AsString()->CheckString(); + + if ( ! mmdb_dir.empty() ) + { + auto d = mmdb_dir + "/GeoLite2-City.mmdb"; + + if ( mmdb_open_loc(d.data()) ) + return true; + + d = mmdb_dir + "/GeoLite2-Country.mmdb"; + + if ( mmdb_open_loc(d.data()) ) + return true;; + } + + return mmdb_open_loc("/usr/share/GeoIP/GeoLite2-City.mmdb") + || mmdb_open_loc("/var/lib/GeoIP/GeoLite2-City.mmdb") + || mmdb_open_loc("/usr/local/share/GeoIP/GeoLite2-City.mmdb") + || mmdb_open_loc("/usr/local/var/GeoIP/GeoLite2-City.mmdb") + || mmdb_open_loc("/usr/share/GeoIP/GeoLite2-Country.mmdb") + || mmdb_open_loc("/var/lib/GeoIP/GeoLite2-Country.mmdb") + || mmdb_open_loc("/usr/local/share/GeoIP/GeoLite2-Country.mmdb") + || mmdb_open_loc("/usr/local/var/GeoIP/GeoLite2-Country.mmdb"); + } + +static bool mmdb_try_open_asn () + { + auto mmdb_dir_val = global_scope()->Lookup("mmdb_dir")->ID_Val(); + std::string mmdb_dir = mmdb_dir_val->AsString()->CheckString(); + + if ( ! mmdb_dir.empty() ) + { + auto d = mmdb_dir + "/GeoLite2-ASN.mmdb"; + + if ( mmdb_open_asn(d.data()) ) + return true; + } + + return mmdb_open_asn("/usr/share/GeoIP/GeoLite2-ASN.mmdb") + || mmdb_open_asn("/var/lib/GeoIP/GeoLite2-ASN.mmdb") + || mmdb_open_asn("/usr/local/share/GeoIP/GeoLite2-ASN.mmdb") + || mmdb_open_asn("/usr/local/var/GeoIP/GeoLite2-ASN.mmdb"); } #endif %%} +## Initializes MMDB for later use of lookup_location. +## Requires Bro to be built with ``libmaxminddb``. +## +## f: The filename of the MaxMind City or Country DB. +## +## Returns: A boolean indicating whether the db was successfully opened. +## +## .. bro:see:: lookup_asn +function mmdb_open_location_db%(f: string%) : bool + %{ +#ifdef USE_GEOIP + return new Val(mmdb_open_loc(f->CheckString()), TYPE_BOOL); +#else + return new Val(0, TYPE_BOOL); +#endif + %} + +## Initializes MMDB for later use of lookup_asn. +## Requires Bro to be built with ``libmaxminddb``. +## +## f: The filename of the MaxMind ASN DB. +## +## Returns: A boolean indicating whether the db was successfully opened. +## +## .. bro:see:: lookup_asn +function mmdb_open_asn_db%(f: string%) : bool + %{ +#ifdef USE_GEOIP + return new Val(mmdb_open_asn(f->CheckString()), TYPE_BOOL); +#else + return new Val(0, TYPE_BOOL); +#endif + %} + ## Performs a geo-lookup of an IP address. -## Requires Bro to be built with ``libgeoip``. +## Requires Bro to be built with ``libmaxminddb``. ## ## a: The IP address to lookup. ## @@ -3681,104 +3873,52 @@ function lookup_location%(a: addr%) : geo_location RecordVal* location = new RecordVal(geo_location); #ifdef USE_GEOIP - static bool geoip_initialized = false; - static GeoIP* geoip = 0; - static GeoIP* geoip_v6 = 0; - static bool have_city_db = false; - static bool have_cityv6_db = false; - GeoIPRecord* gir = 0; - const char* cc = 0; - - if ( ! geoip_initialized ) + if ( ! mmdb_loc ) { - geoip_initialized = true; - geoip = open_geoip_city_db(); - - if ( ! geoip ) + if ( ! mmdb_try_open_loc() ) { - geoip = open_geoip_db(GEOIP_COUNTRY_EDITION); - string db_path = GeoIPDBFileName[GEOIP_COUNTRY_EDITION]; - - if ( ! geoip ) - builtin_error(fmt("Failed fall back to GeoIP Country " - "database: %s", - GeoIPDBFileName[GEOIP_COUNTRY_EDITION])); - else - reporter->Info("Fell back to GeoIP Country database"); + builtin_error("Failed to open GeoIP location database"); + return location; } - else - have_city_db = true; - - geoip_v6 = open_geoip_city_db_v6(); - - if ( geoip_v6 ) - have_cityv6_db = true; - -#ifdef HAVE_GEOIP_COUNTRY_EDITION_V6 - if ( ! geoip_v6 ) - { - geoip_v6 = open_geoip_db(GEOIP_COUNTRY_EDITION_V6); - - if ( ! geoip_v6 ) - reporter->Info("Failed to open GeoIPv6 Country database: %s", - GeoIPDBFileName[GEOIP_COUNTRY_EDITION_V6]); - } -#endif - - if ( ! geoip_v6 ) - builtin_error("Can't open GeoIPv6 City/Country database"); } -#ifdef HAVE_GEOIP_COUNTRY_EDITION_V6 - if ( geoip_v6 && a->AsAddr().GetFamily() == IPv6 ) + MMDB_lookup_result_s result; + + if ( mmdb_lookup_loc(a->AsAddr(), result) ) { - geoipv6_t ga; - a->AsAddr().CopyIPv6(&ga); - if ( have_cityv6_db ) - gir = GeoIP_record_by_ipnum_v6(geoip_v6, ga); - else - cc = GeoIP_country_code_by_ipnum_v6(geoip_v6, ga); - } - else -#endif + MMDB_entry_data_s entry_data; + int status; - if ( geoip && a->AsAddr().GetFamily() == IPv4 ) - { - const uint32* bytes; - a->AsAddr().GetBytes(&bytes); - if ( have_city_db ) - gir = GeoIP_record_by_ipnum(geoip, ntohl(*bytes)); - else - cc = GeoIP_country_code_by_ipnum(geoip, ntohl(*bytes)); - } + // Get Country ISO Code + status = MMDB_get_value(&result.entry, &entry_data, + "country", "iso_code", nullptr); + location->Assign(0, mmdb_getvalue(&entry_data, status, + MMDB_DATA_TYPE_UTF8_STRING)); - if ( gir ) - { - if ( gir->country_code ) - location->Assign(0, new StringVal(gir->country_code)); + // Get Major Subdivision ISO Code + status = MMDB_get_value(&result.entry, &entry_data, + "subdivisions", "0", "iso_code", nullptr); + location->Assign(1, mmdb_getvalue(&entry_data, status, + MMDB_DATA_TYPE_UTF8_STRING)); - if ( gir->region ) - location->Assign(1, new StringVal(gir->region)); + // Get City English Name + status = MMDB_get_value(&result.entry, &entry_data, + "city", "names", "en", nullptr); + location->Assign(2, mmdb_getvalue(&entry_data, status, + MMDB_DATA_TYPE_UTF8_STRING)); - if ( gir->city ) - location->Assign(2, new StringVal(gir->city)); + // Get Location Latitude + status = MMDB_get_value(&result.entry, &entry_data, + "location", "latitude", nullptr); + location->Assign(3, mmdb_getvalue(&entry_data, status, + MMDB_DATA_TYPE_DOUBLE)); - if ( gir->latitude ) - location->Assign(3, new Val(gir->latitude, - TYPE_DOUBLE)); + // Get Location Longitude + status = MMDB_get_value(&result.entry, &entry_data, + "location", "longitude", nullptr); + location->Assign(4, mmdb_getvalue(&entry_data, status, + MMDB_DATA_TYPE_DOUBLE)); - if ( gir->longitude ) - location->Assign(4, new Val(gir->longitude, - TYPE_DOUBLE)); - - GeoIPRecord_delete(gir); - - return location; - } - - else if ( cc ) - { - location->Assign(0, new StringVal(cc)); return location; } @@ -3792,64 +3932,45 @@ function lookup_location%(a: addr%) : geo_location } #endif - // We can get here even if we have GeoIP support if we weren't + // We can get here even if we have MMDB support if we weren't // able to initialize it or it didn't return any information for // the address. return location; %} -## Performs an AS lookup of an IP address. -## Requires Bro to be built with ``libgeoip``. +## Performs an ASN lookup of an IP address. +## Requires Bro to be built with ``libmaxminddb``. ## ## a: The IP address to lookup. ## -## Returns: The number of the AS that contains *a*. +## Returns: The number of the ASN that contains *a*. ## ## .. bro:see:: lookup_location function lookup_asn%(a: addr%) : count %{ #ifdef USE_GEOIP - static GeoIP* geoip_asn = 0; - static bool geoip_asn_initialized = false; - char* gir = 0; - - if ( ! geoip_asn_initialized ) + if ( ! mmdb_asn ) { - geoip_asn_initialized = true; - geoip_asn = open_geoip_db(GEOIP_ASNUM_EDITION); - - if ( ! geoip_asn ) - builtin_error(fmt("Can't open GeoIP ASNUM database: %s", - GeoIPDBFileName[GEOIP_ASNUM_EDITION])); - } - - if ( geoip_asn ) - { -// IPv6 support showed up in 1.4.5. -#ifdef HAVE_GEOIP_COUNTRY_EDITION_V6 - if ( a->AsAddr().GetFamily() == IPv6 ) + if ( ! mmdb_try_open_asn() ) { - geoipv6_t ga; - a->AsAddr().CopyIPv6(&ga); - gir = GeoIP_name_by_ipnum_v6(geoip_asn, ga); - } - else -#endif - - if ( a->AsAddr().GetFamily() == IPv4 ) - { - const uint32* bytes; - a->AsAddr().GetBytes(&bytes); - gir = GeoIP_name_by_ipnum(geoip_asn, ntohl(*bytes)); + builtin_error("No open GeoIP ASN database"); + return new Val(0, TYPE_COUNT); } } - if ( gir ) + MMDB_lookup_result_s result; + + if ( mmdb_lookup_asn(a->AsAddr(), result) ) { - // Move the pointer +2 so we don't return - // the first two characters: "AS". - return new Val(atoi(gir+2), TYPE_COUNT); + MMDB_entry_data_s entry_data; + int status; + + // Get Autonomous System Number + status = MMDB_get_value(&result.entry, &entry_data, + "autonomous_system_number", nullptr); + Val* asn = mmdb_getvalue(&entry_data, status, MMDB_DATA_TYPE_UINT32); + return asn == nullptr ? new Val(0, TYPE_COUNT) : asn; } #else // not USE_GEOIP diff --git a/src/file_analysis/analyzer/x509/OCSP.cc b/src/file_analysis/analyzer/x509/OCSP.cc index 02c9274999..bea88fc747 100644 --- a/src/file_analysis/analyzer/x509/OCSP.cc +++ b/src/file_analysis/analyzer/x509/OCSP.cc @@ -21,7 +21,7 @@ // helper function of sk_X509_value to avoid namespace problem // sk_X509_value(X,Y) = > SKM_sk_value(X509,X,Y) // X509 => file_analysis::X509 -X509 *helper_sk_X509_value(STACK_OF(X509) *certs, int i) +X509* helper_sk_X509_value(const STACK_OF(X509)* certs, int i) { return sk_X509_value(certs, i); } @@ -42,38 +42,84 @@ static Val* get_ocsp_type(RecordVal* args, const char* name) return rval; } -static void OCSP_RESPID_bio(OCSP_RESPID *resp_id, BIO* bio) +static bool OCSP_RESPID_bio(OCSP_BASICRESP* basic_resp, BIO* bio) { - if (resp_id->type == V_OCSP_RESPID_NAME) - X509_NAME_print_ex(bio, resp_id->value.byName, 0, XN_FLAG_ONELINE); - else if (resp_id->type == V_OCSP_RESPID_KEY) - i2a_ASN1_STRING(bio, resp_id->value.byKey, V_ASN1_OCTET_STRING); +#if ( OPENSSL_VERSION_NUMBER < 0x10100000L ) + ASN1_OCTET_STRING* key = nullptr; + X509_NAME* name = nullptr; + + if ( ! basic_resp->tbsResponseData ) + return false; + + auto resp_id = basic_resp->tbsResponseData->responderId; + + if ( resp_id->type == V_OCSP_RESPID_NAME ) + name = resp_id->value.byName; + else if ( resp_id->type == V_OCSP_RESPID_KEY ) + key = resp_id->value.byKey; + else + return false; +#else + const ASN1_OCTET_STRING* key = nullptr; + const X509_NAME* name = nullptr; + + if ( ! OCSP_resp_get0_id(basic_resp, &key, &name) ) + return false; +#endif + + if ( name ) + X509_NAME_print_ex(bio, name, 0, XN_FLAG_ONELINE); + else + i2a_ASN1_STRING(bio, key, V_ASN1_OCTET_STRING); + + return true; } -void ocsp_add_cert_id(OCSP_CERTID *cert_id, val_list* vl, BIO* bio) +bool ocsp_add_cert_id(const OCSP_CERTID* cert_id, val_list* vl, BIO* bio) { + ASN1_OBJECT* hash_alg = nullptr; + ASN1_OCTET_STRING* issuer_name_hash = nullptr; + ASN1_OCTET_STRING* issuer_key_hash = nullptr; + ASN1_INTEGER* serial_number = nullptr; + + auto res = OCSP_id_get0_info(&issuer_name_hash, &hash_alg, + &issuer_key_hash, &serial_number, + const_cast(cert_id)); + + if ( ! res ) + { + reporter->Weird("OpenSSL failed to get OCSP_CERTID info"); + vl->append(new StringVal("")); + vl->append(new StringVal("")); + vl->append(new StringVal("")); + vl->append(new StringVal("")); + return false; + } + char buf[OCSP_STRING_BUF_SIZE]; memset(buf, 0, sizeof(buf)); - i2a_ASN1_OBJECT(bio, cert_id->hashAlgorithm->algorithm); + i2a_ASN1_OBJECT(bio, hash_alg); int len = BIO_read(bio, buf, sizeof(buf)); vl->append(new StringVal(len, buf)); BIO_reset(bio); - i2a_ASN1_STRING(bio, cert_id->issuerNameHash, V_ASN1_OCTET_STRING); + i2a_ASN1_STRING(bio, issuer_name_hash, V_ASN1_OCTET_STRING); len = BIO_read(bio, buf, sizeof(buf)); vl->append(new StringVal(len, buf)); BIO_reset(bio); - i2a_ASN1_STRING(bio, cert_id->issuerKeyHash, V_ASN1_OCTET_STRING); + i2a_ASN1_STRING(bio, issuer_key_hash, V_ASN1_OCTET_STRING); len = BIO_read(bio, buf, sizeof(buf)); vl->append(new StringVal(len, buf)); BIO_reset(bio); - i2a_ASN1_INTEGER(bio, cert_id->serialNumber); + i2a_ASN1_INTEGER(bio, serial_number); len = BIO_read(bio, buf, sizeof(buf)); vl->append(new StringVal(len, buf)); BIO_reset(bio); + + return true; } file_analysis::Analyzer* OCSP::InstantiateRequest(RecordVal* args, File* file) @@ -124,6 +170,7 @@ bool file_analysis::OCSP::EndOfFile() else { OCSP_RESPONSE *resp = d2i_OCSP_RESPONSE(NULL, &ocsp_char, ocsp_data.size()); + if (!resp) { reporter->Weird(fmt("OPENSSL Could not parse OCSP response (fuid %s)", GetFile()->GetID().c_str())); @@ -138,28 +185,255 @@ bool file_analysis::OCSP::EndOfFile() return true; } -void file_analysis::OCSP::ParseRequest(OCSP_REQUEST *req, const char* fid) +#if ( OPENSSL_VERSION_NUMBER >= 0x10100000L ) +// Re-encode and then parse out ASN1 structures to get at what we need... +/*- BasicOCSPResponse ::= SEQUENCE { + * tbsResponseData ResponseData, + * signatureAlgorithm AlgorithmIdentifier, + * signature BIT STRING, + * certs [0] EXPLICIT SEQUENCE OF Certificate OPTIONAL } +typedef struct ocsp_basic_response_st { + OCSP_RESPDATA *tbsResponseData; + X509_ALGOR *signatureAlgorithm; + ASN1_BIT_STRING *signature; + STACK_OF(X509) *certs; +} OCSP_BASICRESP; +*/ +static StringVal* parse_basic_resp_sig_alg(OCSP_BASICRESP* basic_resp, + BIO* bio, char* buf, size_t buf_len) { - OCSP_REQINFO *inf = req->tbsRequest; + int der_basic_resp_len = 0; + unsigned char* der_basic_resp_dat = nullptr; + der_basic_resp_len = i2d_OCSP_BASICRESP(basic_resp, &der_basic_resp_dat); + + if ( der_basic_resp_len <= 0 ) + return new StringVal(""); + + const unsigned char* const_der_basic_resp_dat = der_basic_resp_dat; + + auto bseq = d2i_ASN1_SEQUENCE_ANY(nullptr, &const_der_basic_resp_dat, + der_basic_resp_len); + + if ( ! bseq ) + { + OPENSSL_free(der_basic_resp_dat); + return new StringVal(""); + } + + if ( sk_ASN1_TYPE_num(bseq) < 3 ) + { + sk_ASN1_TYPE_free(bseq); + OPENSSL_free(der_basic_resp_dat); + return new StringVal(""); + } + + auto constexpr sig_alg_idx = 1u; + auto aseq_type = sk_ASN1_TYPE_value(bseq, sig_alg_idx); + + if ( ASN1_TYPE_get(aseq_type) != V_ASN1_SEQUENCE ) + { + sk_ASN1_TYPE_free(bseq); + OPENSSL_free(der_basic_resp_dat); + return new StringVal(""); + } + + auto aseq_str = aseq_type->value.asn1_string; + auto aseq_len = ASN1_STRING_length(aseq_str); + auto aseq_dat = ASN1_STRING_get0_data(aseq_str); + + auto aseq = d2i_ASN1_SEQUENCE_ANY(nullptr, &aseq_dat, aseq_len); + + if ( ! aseq ) + { + sk_ASN1_TYPE_free(bseq); + OPENSSL_free(der_basic_resp_dat); + return new StringVal(""); + } + + if ( sk_ASN1_TYPE_num(aseq) < 1 ) + { + sk_ASN1_TYPE_free(aseq); + sk_ASN1_TYPE_free(bseq); + OPENSSL_free(der_basic_resp_dat); + return new StringVal(""); + } + + auto constexpr alg_obj_idx = 0u; + auto alg_obj_type = sk_ASN1_TYPE_value(aseq, alg_obj_idx); + + if ( ASN1_TYPE_get(alg_obj_type) != V_ASN1_OBJECT ) + { + sk_ASN1_TYPE_free(aseq); + sk_ASN1_TYPE_free(bseq); + OPENSSL_free(der_basic_resp_dat); + return new StringVal(""); + } + + auto alg_obj = alg_obj_type->value.object; + i2a_ASN1_OBJECT(bio, alg_obj); + auto alg_len = BIO_read(bio, buf, buf_len); + auto rval = new StringVal(alg_len, buf); + BIO_reset(bio); + + sk_ASN1_TYPE_free(aseq); + sk_ASN1_TYPE_free(bseq); + OPENSSL_free(der_basic_resp_dat); + return rval; + } + +static Val* parse_basic_resp_data_version(OCSP_BASICRESP* basic_resp) + { + int der_basic_resp_len = 0; + unsigned char* der_basic_resp_dat = nullptr; + + der_basic_resp_len = i2d_OCSP_BASICRESP(basic_resp, &der_basic_resp_dat); + + if ( der_basic_resp_len <= 0 ) + return new Val(-1, TYPE_COUNT); + + const unsigned char* const_der_basic_resp_dat = der_basic_resp_dat; + + auto bseq = d2i_ASN1_SEQUENCE_ANY(nullptr, &const_der_basic_resp_dat, + der_basic_resp_len); + + if ( ! bseq ) + { + OPENSSL_free(der_basic_resp_dat); + return new Val(-1, TYPE_COUNT); + } + + if ( sk_ASN1_TYPE_num(bseq) < 3 ) + { + sk_ASN1_TYPE_free(bseq); + OPENSSL_free(der_basic_resp_dat); + return new Val(-1, TYPE_COUNT); + } + + auto constexpr resp_data_idx = 0u; + auto dseq_type = sk_ASN1_TYPE_value(bseq, resp_data_idx); + + if ( ASN1_TYPE_get(dseq_type) != V_ASN1_SEQUENCE ) + { + sk_ASN1_TYPE_free(bseq); + OPENSSL_free(der_basic_resp_dat); + return new Val(-1, TYPE_COUNT); + } + + auto dseq_str = dseq_type->value.asn1_string; + auto dseq_len = ASN1_STRING_length(dseq_str); + auto dseq_dat = ASN1_STRING_get0_data(dseq_str); + + auto dseq = d2i_ASN1_SEQUENCE_ANY(nullptr, &dseq_dat, dseq_len); + + if ( ! dseq ) + { + sk_ASN1_TYPE_free(bseq); + OPENSSL_free(der_basic_resp_dat); + return new StringVal(""); + } + + if ( sk_ASN1_TYPE_num(dseq) < 1 ) + { + sk_ASN1_TYPE_free(dseq); + sk_ASN1_TYPE_free(bseq); + OPENSSL_free(der_basic_resp_dat); + return new StringVal(""); + } + +/*- ResponseData ::= SEQUENCE { + * version [0] EXPLICIT Version DEFAULT v1, + * responderID ResponderID, + * producedAt GeneralizedTime, + * responses SEQUENCE OF SingleResponse, + * responseExtensions [1] EXPLICIT Extensions OPTIONAL } + */ + + auto constexpr version_idx = 0u; + auto version_type = sk_ASN1_TYPE_value(dseq, version_idx); + + if ( ASN1_TYPE_get(version_type) != V_ASN1_INTEGER ) + { + sk_ASN1_TYPE_free(dseq); + sk_ASN1_TYPE_free(bseq); + OPENSSL_free(der_basic_resp_dat); + // Not present, use default value. + return new Val(0, TYPE_COUNT); + } + + uint64_t asn1_int = ASN1_INTEGER_get(version_type->value.integer); + sk_ASN1_TYPE_free(dseq); + sk_ASN1_TYPE_free(bseq); + OPENSSL_free(der_basic_resp_dat); + return new Val(asn1_int, TYPE_COUNT); + } + +static uint64 parse_request_version(OCSP_REQUEST* req) + { + int der_req_len = 0; + unsigned char* der_req_dat = nullptr; + der_req_len = i2d_OCSP_REQUEST(req, &der_req_dat); + const unsigned char* const_der_req_dat = der_req_dat; + + if ( ! der_req_dat ) + return -1; + + auto rseq = d2i_ASN1_SEQUENCE_ANY(nullptr, &const_der_req_dat, + der_req_len); + + if ( ! rseq ) + { + OPENSSL_free(der_req_dat); + return -1; + } + + if ( sk_ASN1_TYPE_num(rseq) < 1 ) + { + sk_ASN1_TYPE_free(rseq); + OPENSSL_free(der_req_dat); + return -1; + } + + auto constexpr version_idx = 0u; + auto version_type = sk_ASN1_TYPE_value(rseq, version_idx); + + if ( ASN1_TYPE_get(version_type) != V_ASN1_INTEGER ) + { + sk_ASN1_TYPE_free(rseq); + OPENSSL_free(der_req_dat); + // Not present, use default value. + return 0; + } + + uint64_t asn1_int = ASN1_INTEGER_get(version_type->value.integer); + sk_ASN1_TYPE_free(rseq); + OPENSSL_free(der_req_dat); + return asn1_int; + } +#endif + +void file_analysis::OCSP::ParseRequest(OCSP_REQUEST* req, const char* fid) + { char buf[OCSP_STRING_BUF_SIZE]; // we need a buffer for some of the openssl functions memset(buf, 0, sizeof(buf)); // build up our response as we go along... val_list* vl = new val_list(); vl->append(GetFile()->GetVal()->Ref()); - vl->append(new Val((uint64)ASN1_INTEGER_get(inf->version), TYPE_COUNT)); - BIO *bio = BIO_new(BIO_s_mem()); - if (inf->requestorName != NULL) - { - GENERAL_NAME_print(bio, inf->requestorName); - int len = BIO_read(bio, buf, sizeof(buf)); - vl->append(new StringVal(len, buf)); - BIO_reset(bio); - } - else - vl->append(new StringVal(0, "")); + uint64 version = 0; + +#if ( OPENSSL_VERSION_NUMBER < 0x10100000L ) + if ( req->tbsRequest->version ) + version = (uint64)ASN1_INTEGER_get(req->tbsRequest->version); +#else + version = parse_request_version(req); + // TODO: try to parse out general name ? +#endif + + vl->append(new Val(version, TYPE_COUNT)); + + BIO *bio = BIO_new(BIO_s_mem()); mgr.QueueEvent(ocsp_request, vl); @@ -182,10 +456,12 @@ void file_analysis::OCSP::ParseRequest(OCSP_REQUEST *req, const char* fid) void file_analysis::OCSP::ParseResponse(OCSP_RESPVal *resp_val, const char* fid) { OCSP_RESPONSE *resp = resp_val->GetResp(); - OCSP_RESPBYTES *resp_bytes = resp->responseBytes; + //OCSP_RESPBYTES *resp_bytes = resp->responseBytes; OCSP_BASICRESP *basic_resp = nullptr; OCSP_RESPDATA *resp_data = nullptr; OCSP_RESPID *resp_id = nullptr; + const ASN1_GENERALIZEDTIME* produced_at = nullptr; + const STACK_OF(X509)* certs = nullptr; int resp_count, num_ext = 0; VectorVal *certs_vector = nullptr; @@ -203,11 +479,11 @@ void file_analysis::OCSP::ParseResponse(OCSP_RESPVal *resp_val, const char* fid) mgr.QueueEvent(ocsp_response_status, vl); vl = nullptr; - if (!resp_bytes) - { - Unref(status_val); - return; - } + //if (!resp_bytes) + // { + // Unref(status_val); + // return; + // } BIO *bio = BIO_new(BIO_s_mem()); //i2a_ASN1_OBJECT(bio, resp_bytes->responseType); @@ -219,31 +495,53 @@ void file_analysis::OCSP::ParseResponse(OCSP_RESPVal *resp_val, const char* fid) if ( !basic_resp ) goto clean_up; +#if ( OPENSSL_VERSION_NUMBER < 0x10100000L ) resp_data = basic_resp->tbsResponseData; if ( !resp_data ) goto clean_up; +#endif vl = new val_list(); vl->append(GetFile()->GetVal()->Ref()); vl->append(resp_val->Ref()); vl->append(status_val); + +#if ( OPENSSL_VERSION_NUMBER < 0x10100000L ) vl->append(new Val((uint64)ASN1_INTEGER_get(resp_data->version), TYPE_COUNT)); +#else + vl->append(parse_basic_resp_data_version(basic_resp)); +#endif // responderID - resp_id = resp_data->responderId; - OCSP_RESPID_bio(resp_id, bio); - len = BIO_read(bio, buf, sizeof(buf)); - vl->append(new StringVal(len, buf)); - BIO_reset(bio); + if ( OCSP_RESPID_bio(basic_resp, bio) ) + { + len = BIO_read(bio, buf, sizeof(buf)); + vl->append(new StringVal(len, buf)); + BIO_reset(bio); + } + else + { + reporter->Weird("OpenSSL failed to get OCSP responder id"); + vl->append(new StringVal("")); + } // producedAt - vl->append(new Val(GetTimeFromAsn1(resp_data->producedAt, fid, reporter), TYPE_TIME)); +#if ( OPENSSL_VERSION_NUMBER < 0x10100000L ) + produced_at = resp_data->producedAt; +#else + produced_at = OCSP_resp_get0_produced_at(basic_resp); +#endif + + vl->append(new Val(GetTimeFromAsn1(produced_at, fid, reporter), TYPE_TIME)); // responses - resp_count = sk_OCSP_SINGLERESP_num(resp_data->responses); + + resp_count = OCSP_resp_count(basic_resp); + for ( int i=0; iresponses, i); + OCSP_SINGLERESP* single_resp = OCSP_resp_get0(basic_resp, i); + if ( !single_resp ) continue; @@ -251,24 +549,41 @@ void file_analysis::OCSP::ParseResponse(OCSP_RESPVal *resp_val, const char* fid) rvl->append(GetFile()->GetVal()->Ref()); // cert id - OCSP_CERTID *cert_id = single_resp->certId; + const OCSP_CERTID* cert_id = nullptr; + +#if ( OPENSSL_VERSION_NUMBER < 0x10100000L ) + cert_id = single_resp->certId; +#else + cert_id = OCSP_SINGLERESP_get0_id(single_resp); +#endif + ocsp_add_cert_id(cert_id, rvl, bio); BIO_reset(bio); // certStatus - OCSP_CERTSTATUS *cert_status = single_resp->certStatus; - const char* cert_status_str = OCSP_cert_status_str(cert_status->type); + int status = V_OCSP_CERTSTATUS_UNKNOWN; + int reason = OCSP_REVOKED_STATUS_NOSTATUS; + ASN1_GENERALIZEDTIME* revoke_time = nullptr; + ASN1_GENERALIZEDTIME* this_update = nullptr; + ASN1_GENERALIZEDTIME* next_update = nullptr; + + if ( ! OCSP_resp_find_status(basic_resp, + const_cast(cert_id), + &status, &reason, &revoke_time, + &this_update, &next_update) ) + reporter->Weird("OpenSSL failed to find status of OCSP response"); + + const char* cert_status_str = OCSP_cert_status_str(status); rvl->append(new StringVal(strlen(cert_status_str), cert_status_str)); // revocation time and reason if revoked - if ( cert_status->type == V_OCSP_CERTSTATUS_REVOKED ) + if ( status == V_OCSP_CERTSTATUS_REVOKED ) { - OCSP_REVOKEDINFO *revoked_info = cert_status->value.revoked; - rvl->append(new Val(GetTimeFromAsn1(revoked_info->revocationTime, fid, reporter), TYPE_TIME)); + rvl->append(new Val(GetTimeFromAsn1(revoke_time, fid, reporter), TYPE_TIME)); - if ( revoked_info->revocationReason ) + if ( reason != OCSP_REVOKED_STATUS_NOSTATUS ) { - const char* revoke_reason = OCSP_crl_reason_str(ASN1_ENUMERATED_get(revoked_info->revocationReason)); + const char* revoke_reason = OCSP_crl_reason_str(reason); rvl->append(new StringVal(strlen(revoke_reason), revoke_reason)); } else @@ -280,9 +595,13 @@ void file_analysis::OCSP::ParseResponse(OCSP_RESPVal *resp_val, const char* fid) rvl->append(new StringVal(0, "")); } - rvl->append(new Val(GetTimeFromAsn1(single_resp->thisUpdate, fid, reporter), TYPE_TIME)); - if ( single_resp->nextUpdate ) - rvl->append(new Val(GetTimeFromAsn1(single_resp->nextUpdate, fid, reporter), TYPE_TIME)); + if ( this_update ) + rvl->append(new Val(GetTimeFromAsn1(this_update, fid, reporter), TYPE_TIME)); + else + rvl->append(new Val(0, TYPE_TIME)); + + if ( next_update ) + rvl->append(new Val(GetTimeFromAsn1(next_update, fid, reporter), TYPE_TIME)); else rvl->append(new Val(0, TYPE_TIME)); @@ -299,10 +618,14 @@ void file_analysis::OCSP::ParseResponse(OCSP_RESPVal *resp_val, const char* fid) } } +#if ( OPENSSL_VERSION_NUMBER < 0x10100000L ) i2a_ASN1_OBJECT(bio, basic_resp->signatureAlgorithm->algorithm); len = BIO_read(bio, buf, sizeof(buf)); vl->append(new StringVal(len, buf)); BIO_reset(bio); +#else + vl->append(parse_basic_resp_sig_alg(basic_resp, bio, buf, sizeof(buf))); +#endif //i2a_ASN1_OBJECT(bio, basic_resp->signature); //len = BIO_read(bio, buf, sizeof(buf)); @@ -311,13 +634,20 @@ void file_analysis::OCSP::ParseResponse(OCSP_RESPVal *resp_val, const char* fid) certs_vector = new VectorVal(internal_type("x509_opaque_vector")->AsVectorType()); vl->append(certs_vector); - if ( basic_resp->certs ) + +#if ( OPENSSL_VERSION_NUMBER < 0x10100000L ) + certs = basic_resp->certs; +#else + certs = OCSP_resp_get0_certs(basic_resp); +#endif + + if ( certs ) { - int num_certs = sk_X509_num(basic_resp->certs); + int num_certs = sk_X509_num(certs); for ( int i=0; icerts, i)); - //::X509 *this_cert = X509_dup(sk_X509_value(basic_resp->certs, i)); + ::X509 *this_cert = X509_dup(helper_sk_X509_value(certs, i)); + //::X509 *this_cert = X509_dup(sk_X509_value(certs, i)); if (this_cert) certs_vector->Assign(i, new file_analysis::X509Val(this_cert)); else diff --git a/src/file_analysis/analyzer/x509/OCSP.h b/src/file_analysis/analyzer/x509/OCSP.h index ca8c2c563e..75caf3120a 100644 --- a/src/file_analysis/analyzer/x509/OCSP.h +++ b/src/file_analysis/analyzer/x509/OCSP.h @@ -29,8 +29,8 @@ protected: OCSP(RecordVal* args, File* file, bool request); private: - void ParseResponse(OCSP_RESPVal *, const char* fid = 0); - void ParseRequest(OCSP_REQUEST *, const char* fid = 0); + void ParseResponse(OCSP_RESPVal*, const char* fid = 0); + void ParseRequest(OCSP_REQUEST*, const char* fid = 0); void ParseExtensionsSpecific(X509_EXTENSION* ex, bool, ASN1_OBJECT*, const char*) override; std::string ocsp_data; diff --git a/src/file_analysis/analyzer/x509/X509.cc b/src/file_analysis/analyzer/x509/X509.cc index 0426b59923..7571915207 100644 --- a/src/file_analysis/analyzer/x509/X509.cc +++ b/src/file_analysis/analyzer/x509/X509.cc @@ -290,7 +290,11 @@ void file_analysis::X509::ParseSAN(X509_EXTENSION* ext) continue; } +#if ( OPENSSL_VERSION_NUMBER < 0x10100000L ) const char* name = (const char*) ASN1_STRING_data(gen->d.ia5); +#else + const char* name = (const char*) ASN1_STRING_get0_data(gen->d.ia5); +#endif StringVal* bs = new StringVal(name); switch ( gen->type ) diff --git a/src/file_analysis/analyzer/x509/functions.bif b/src/file_analysis/analyzer/x509/functions.bif index 18f56758f5..3622e0d13a 100644 --- a/src/file_analysis/analyzer/x509/functions.bif +++ b/src/file_analysis/analyzer/x509/functions.bif @@ -109,21 +109,36 @@ STACK_OF(X509)* x509_get_untrusted_stack(VectorVal* certs_vec) // We need this function to be able to identify the signer certificate of an // OCSP request out of a list of possible certificates. -X509* x509_get_ocsp_signer(STACK_OF(X509) *certs, OCSP_RESPID *rid) +X509* x509_get_ocsp_signer(const STACK_OF(X509)* certs, + OCSP_BASICRESP* basic_resp) { - // We support two lookup types - either by response id or by key. - if ( rid->type == V_OCSP_RESPID_NAME ) - return X509_find_by_subject(certs, rid->value.byName); + const ASN1_OCTET_STRING* key = nullptr; + const X509_NAME* name = nullptr; - // There only should be name and type - but let's be sure... - if ( rid->type != V_OCSP_RESPID_KEY ) +#if ( OPENSSL_VERSION_NUMBER < 0x10100000L ) + OCSP_RESPID* resp_id = basic_resp->tbsResponseData->responderId; + + if ( resp_id->type == V_OCSP_RESPID_NAME ) + name = resp_id->value.byName; + else if ( resp_id->type == V_OCSP_RESPID_KEY ) + key = resp_id->value.byKey; + else return 0; +#else + if ( ! OCSP_resp_get0_id(basic_resp, &key, &name) ) + return 0; +#endif + + if ( name ) + return X509_find_by_subject(const_cast(certs), + const_cast(name)); // Just like OpenSSL, we just support SHA-1 lookups and bail out otherwhise. - if ( rid->value.byKey->length != SHA_DIGEST_LENGTH ) + if ( key->length != SHA_DIGEST_LENGTH ) return 0; - unsigned char* key_hash = rid->value.byKey->data; + unsigned char* key_hash = key->data; + for ( int i = 0; i < sk_X509_num(certs); ++i ) { unsigned char digest[SHA_DIGEST_LENGTH]; @@ -328,15 +343,19 @@ function x509_ocsp_verify%(certs: x509_opaque_vector, ocsp_reply: string, root_c // Because we actually want to be able to give nice error messages that show why we were // not able to verify the OCSP response - do our own verification logic first. - signer = x509_get_ocsp_signer(basic->certs, basic->tbsResponseData->responderId); +#if ( OPENSSL_VERSION_NUMBER < 0x10100000L ) + signer = x509_get_ocsp_signer(basic->certs, basic); +#else + signer = x509_get_ocsp_signer(OCSP_resp_get0_certs(basic), basic); +#endif /* Do this perhaps - OpenSSL also cannot do it, so I do not really feel bad about it. Needs a different lookup because the root store is no stack of X509 certs - if ( !s igner ) + if ( ! signer ) // if we did not find it in the certificates that were sent, search in the root store - signer = x509_get_ocsp_signer(ocsp_certs, basic->tbsResponseData->responderId); + signer = x509_get_ocsp_signer(ocsp_certs, basic); */ if ( ! signer ) diff --git a/src/file_analysis/analyzer/x509/ocsp_events.bif b/src/file_analysis/analyzer/x509/ocsp_events.bif index 4e0ed6e9dc..f49208d238 100644 --- a/src/file_analysis/analyzer/x509/ocsp_events.bif +++ b/src/file_analysis/analyzer/x509/ocsp_events.bif @@ -7,13 +7,10 @@ ## ## req: version: the version of the OCSP request. Typically 0 (Version 1). ## -## requestorName: name of the OCSP requestor. This attribute is optional; if -## it is not set, an empty string is returned here. -## ## .. bro:see:: ocsp_request_certificate ocsp_response_status ## ocsp_response_bytes ocsp_response_certificate ocsp_extension ## x509_ocsp_ext_signed_certificate_timestamp -event ocsp_request%(f: fa_file, version: count, requestorName: string%); +event ocsp_request%(f: fa_file, version: count%); ## Event that is raised when encountering an OCSP request for a certificate, ## e.g. in an HTTP connection. See :rfc:`6960` for more details. diff --git a/src/iosource/Packet.cc b/src/iosource/Packet.cc index 5199765f51..3aa0e28b92 100644 --- a/src/iosource/Packet.cc +++ b/src/iosource/Packet.cc @@ -140,6 +140,20 @@ void Packet::ProcessLayer2() case DLT_EN10MB: { + // Skip past Cisco FabricPath to encapsulated ethernet frame. + if ( pdata[12] == 0x89 && pdata[13] == 0x03 ) + { + auto constexpr cfplen = 16; + + if ( pdata + cfplen + GetLinkHeaderSize(link_type) >= end_of_data ) + { + Weird("truncated_link_header_cfp"); + return; + } + + pdata += cfplen; + } + // Get protocol being carried from the ethernet frame. int protocol = (pdata[12] << 8) + pdata[13]; diff --git a/testing/btest/Baseline/bifs.x509_verify/.stdout b/testing/btest/Baseline/bifs.x509_verify/stdout-openssl-1.0 similarity index 100% rename from testing/btest/Baseline/bifs.x509_verify/.stdout rename to testing/btest/Baseline/bifs.x509_verify/stdout-openssl-1.0 diff --git a/testing/btest/Baseline/bifs.x509_verify/stdout-openssl-1.1 b/testing/btest/Baseline/bifs.x509_verify/stdout-openssl-1.1 new file mode 100644 index 0000000000..29660eade5 --- /dev/null +++ b/testing/btest/Baseline/bifs.x509_verify/stdout-openssl-1.1 @@ -0,0 +1,6 @@ +Validation result: certificate has expired +Validation result: ok +Resulting chain: +Fingerprint: 70829f77ff4b6e908324a3f4e1940fce6c489098, Subject: CN=www.tobu-estate.com,OU=Terms of use at www.verisign.com/rpa (c)05,O=TOBU RAILWAY Co.\,Ltd.,L=Sumida-ku,ST=Tokyo,C=JP +Fingerprint: 5deb8f339e264c19f6686f5f8f32b54a4c46b476, Subject: CN=VeriSign Class 3 Secure Server CA - G3,OU=Terms of use at https://www.verisign.com/rpa (c)10,OU=VeriSign Trust Network,O=VeriSign\, Inc.,C=US +Fingerprint: 4eb6d578499b1ccf5f581ead56be3d9b6744a5e5, Subject: CN=VeriSign Class 3 Public Primary Certification Authority - G5,OU=(c) 2006 VeriSign\, Inc. - For authorized use only,OU=VeriSign Trust Network,O=VeriSign\, Inc.,C=US diff --git a/testing/btest/Baseline/core.cisco-fabric-path/conn.log b/testing/btest/Baseline/core.cisco-fabric-path/conn.log new file mode 100644 index 0000000000..eae407aceb --- /dev/null +++ b/testing/btest/Baseline/core.cisco-fabric-path/conn.log @@ -0,0 +1,41 @@ +#separator \x09 +#set_separator , +#empty_field (empty) +#unset_field - +#path conn +#open 2018-07-09-14-17-29 +#fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p proto service duration orig_bytes resp_bytes conn_state local_orig local_resp missed_bytes history orig_pkts orig_ip_bytes resp_pkts resp_ip_bytes tunnel_parents +#types time string addr port addr port enum string interval count count string bool bool count string count count count count set[string] +1529347003.860008 C7fIlMZDuRiqjpYbb 1.1.1.6 57005 2.2.2.2 48879 tcp - 0.001018 0 0 S0 - - 0 S 2 80 0 0 - +1529347003.861732 CykQaM33ztNt0csB9a 1.1.1.4 57005 2.2.2.2 48879 tcp - 0.000928 0 0 S0 - - 0 S 2 80 0 0 - +1529347003.863372 CtxTCR2Yer0FR1tIBg 1.1.1.14 57005 2.2.2.2 48879 tcp - 0.000928 0 0 S0 - - 0 S 2 80 0 0 - +1529347003.865002 CpmdRlaUoJLN3uIRa 1.1.1.12 57005 2.2.2.2 48879 tcp - 0.000926 0 0 S0 - - 0 S 2 80 0 0 - +1529347003.866648 C1Xkzz2MaGtLrc1Tla 1.1.1.0 57005 2.2.2.2 48879 tcp - 0.001042 0 0 S0 - - 0 S 2 80 0 0 - +1529347003.868394 CqlVyW1YwZ15RhTBc4 1.1.1.2 57005 2.2.2.2 48879 tcp - 0.000920 0 0 S0 - - 0 S 2 80 0 0 - +1529347003.870014 CLNN1k2QMum1aexUK7 1.1.1.8 57005 2.2.2.2 48879 tcp - 0.000930 0 0 S0 - - 0 S 2 80 0 0 - +1529347003.871649 CBA8792iHmnhPLksKa 1.1.1.10 57005 2.2.2.2 48879 tcp - 0.000928 0 0 S0 - - 0 S 2 80 0 0 - +1529347003.873385 CGLPPc35OzDQij1XX8 1234::e 57005 5678:: 48879 tcp - 0.001139 0 0 S0 - - 0 S 2 120 0 0 - +1529347003.875322 CiyBAq1bBLNaTiTAc 1234::c 57005 5678:: 48879 tcp - 0.001027 0 0 S0 - - 0 S 2 120 0 0 - +1529347003.877182 CFSwNi4CNGxcuffo49 1234::6 57005 5678:: 48879 tcp - 0.001055 0 0 S0 - - 0 S 2 120 0 0 - +1529347003.879034 Cipfzj1BEnhejw8cGf 1234::4 57005 5678:: 48879 tcp - 0.001018 0 0 S0 - - 0 S 2 120 0 0 - +1529347003.881330 CV5WJ42jPYbNW9JNWf 1234::8 57005 5678:: 48879 tcp - 0.001029 0 0 S0 - - 0 S 2 120 0 0 - +1529347003.883152 CPhDKt12KQPUVbQz06 1234::a 57005 5678:: 48879 tcp - 0.001005 0 0 S0 - - 0 S 2 120 0 0 - +1529347003.884945 CAnFrb2Cvxr5T7quOc 1234:: 57005 5678:: 48879 tcp - 0.001005 0 0 S0 - - 0 S 2 120 0 0 - +1529347003.886751 C8rquZ3DjgNW06JGLl 1234::2 57005 5678:: 48879 tcp - 0.001120 0 0 S0 - - 0 S 2 120 0 0 - +1529347003.851951 CFLRIC3zaTU1loLGxh 1234::4 57005 5678:: 48879 udp - 0.000905 0 0 S0 - - 0 D 2 96 0 0 - +1529347003.855232 Ck51lg1bScffFj34Ri 1234::a 57005 5678:: 48879 udp - 0.000894 0 0 S0 - - 0 D 2 96 0 0 - +1529347003.839636 CtPZjS20MLrsMUOJi2 1.1.1.12 57005 2.2.2.2 48879 udp - 0.000847 0 0 S0 - - 0 D 2 56 0 0 - +1529347003.858393 CNnMIj2QSd84NKf7U3 1234::2 57005 5678:: 48879 udp - 0.000902 0 0 S0 - - 0 D 2 96 0 0 - +1529347003.842649 CmES5u32sYpV7JYN 1.1.1.2 57005 2.2.2.2 48879 udp - 0.000830 0 0 S0 - - 0 D 2 56 0 0 - +1529347003.850367 C0LAHyvtKSQHyJxIl 1234::6 57005 5678:: 48879 udp - 0.000898 0 0 S0 - - 0 D 2 96 0 0 - +1529347003.848776 CwjjYJ2WqgTbAqiHl6 1234::c 57005 5678:: 48879 udp - 0.000902 0 0 S0 - - 0 D 2 96 0 0 - +1529347003.856801 C9mvWx3ezztgzcexV7 1234:: 57005 5678:: 48879 udp - 0.000898 0 0 S0 - - 0 D 2 96 0 0 - +1529347003.841103 CUM0KZ3MLUfNB0cl11 1.1.1.0 57005 2.2.2.2 48879 udp - 0.000926 0 0 S0 - - 0 D 2 56 0 0 - +1529347003.845524 C37jN32gN3y3AZzyf6 1.1.1.10 57005 2.2.2.2 48879 udp - 0.000843 0 0 S0 - - 0 D 2 56 0 0 - +1529347003.847079 C3eiCBGOLw3VtHfOj 1234::e 57005 5678:: 48879 udp - 0.001014 0 0 S0 - - 0 D 2 96 0 0 - +1529347003.853544 C9rXSW3KSpTYvPrlI1 1234::8 57005 5678:: 48879 udp - 0.001010 0 0 S0 - - 0 D 2 96 0 0 - +1529347003.836659 ClEkJM2Vm5giqnMf4h 1.1.1.4 57005 2.2.2.2 48879 udp - 0.000847 0 0 S0 - - 0 D 2 56 0 0 - +1529347003.838130 C4J4Th3PJpwUYZZ6gc 1.1.1.14 57005 2.2.2.2 48879 udp - 0.000880 0 0 S0 - - 0 D 2 56 0 0 - +1529347003.844086 CP5puj4I8PtEU4qzYg 1.1.1.8 57005 2.2.2.2 48879 udp - 0.000830 0 0 S0 - - 0 D 2 56 0 0 - +1529347003.834704 CHhAvVGS1DHFjwGM9 1.1.1.6 57005 2.2.2.2 48879 udp - 0.001243 0 0 S0 - - 0 D 2 56 0 0 - +#close 2018-07-09-14-17-29 diff --git a/testing/btest/Traces/cisco-fabric-path.pcap b/testing/btest/Traces/cisco-fabric-path.pcap new file mode 100644 index 0000000000..f238a0600d Binary files /dev/null and b/testing/btest/Traces/cisco-fabric-path.pcap differ diff --git a/testing/btest/bifs/x509_verify.bro b/testing/btest/bifs/x509_verify.bro index 59939180c8..2afc735172 100644 --- a/testing/btest/bifs/x509_verify.bro +++ b/testing/btest/bifs/x509_verify.bro @@ -1,5 +1,14 @@ # @TEST-EXEC: bro -r $TRACES/tls/tls-expired-cert.trace %INPUT -# @TEST-EXEC: btest-diff .stdout + +# This is a hack: the results of OpenSSL 1.1's vs 1.0's +# X509_verify_cert() -> X509_STORE_CTX_get1_chain() calls +# differ. Word seems to be that OpenSSL 1.1's cert-chain-building +# code is significantly different/rewritten so may be the reason... + +# @TEST-EXEC: cp .stdout stdout-openssl-1.0 +# @TEST-EXEC: cp .stdout stdout-openssl-1.1 + +# @TEST-EXEC: grep -q "BRO_HAVE_OPENSSL_1_1" $BUILD/CMakeCache.txt && btest-diff stdout-openssl-1.1 || btest-diff stdout-openssl-1.0 redef SSL::root_certs += { ["OU=Class 3 Public Primary Certification Authority,O=VeriSign\, Inc.,C=US"] = "\x30\x82\x02\x3C\x30\x82\x01\xA5\x02\x10\x70\xBA\xE4\x1D\x10\xD9\x29\x34\xB6\x38\xCA\x7B\x03\xCC\xBA\xBF\x30\x0D\x06\x09\x2A\x86\x48\x86\xF7\x0D\x01\x01\x02\x05\x00\x30\x5F\x31\x0B\x30\x09\x06\x03\x55\x04\x06\x13\x02\x55\x53\x31\x17\x30\x15\x06\x03\x55\x04\x0A\x13\x0E\x56\x65\x72\x69\x53\x69\x67\x6E\x2C\x20\x49\x6E\x63\x2E\x31\x37\x30\x35\x06\x03\x55\x04\x0B\x13\x2E\x43\x6C\x61\x73\x73\x20\x33\x20\x50\x75\x62\x6C\x69\x63\x20\x50\x72\x69\x6D\x61\x72\x79\x20\x43\x65\x72\x74\x69\x66\x69\x63\x61\x74\x69\x6F\x6E\x20\x41\x75\x74\x68\x6F\x72\x69\x74\x79\x30\x1E\x17\x0D\x39\x36\x30\x31\x32\x39\x30\x30\x30\x30\x30\x30\x5A\x17\x0D\x32\x38\x30\x38\x30\x31\x32\x33\x35\x39\x35\x39\x5A\x30\x5F\x31\x0B\x30\x09\x06\x03\x55\x04\x06\x13\x02\x55\x53\x31\x17\x30\x15\x06\x03\x55\x04\x0A\x13\x0E\x56\x65\x72\x69\x53\x69\x67\x6E\x2C\x20\x49\x6E\x63\x2E\x31\x37\x30\x35\x06\x03\x55\x04\x0B\x13\x2E\x43\x6C\x61\x73\x73\x20\x33\x20\x50\x75\x62\x6C\x69\x63\x20\x50\x72\x69\x6D\x61\x72\x79\x20\x43\x65\x72\x74\x69\x66\x69\x63\x61\x74\x69\x6F\x6E\x20\x41\x75\x74\x68\x6F\x72\x69\x74\x79\x30\x81\x9F\x30\x0D\x06\x09\x2A\x86\x48\x86\xF7\x0D\x01\x01\x01\x05\x00\x03\x81\x8D\x00\x30\x81\x89\x02\x81\x81\x00\xC9\x5C\x59\x9E\xF2\x1B\x8A\x01\x14\xB4\x10\xDF\x04\x40\xDB\xE3\x57\xAF\x6A\x45\x40\x8F\x84\x0C\x0B\xD1\x33\xD9\xD9\x11\xCF\xEE\x02\x58\x1F\x25\xF7\x2A\xA8\x44\x05\xAA\xEC\x03\x1F\x78\x7F\x9E\x93\xB9\x9A\x00\xAA\x23\x7D\xD6\xAC\x85\xA2\x63\x45\xC7\x72\x27\xCC\xF4\x4C\xC6\x75\x71\xD2\x39\xEF\x4F\x42\xF0\x75\xDF\x0A\x90\xC6\x8E\x20\x6F\x98\x0F\xF8\xAC\x23\x5F\x70\x29\x36\xA4\xC9\x86\xE7\xB1\x9A\x20\xCB\x53\xA5\x85\xE7\x3D\xBE\x7D\x9A\xFE\x24\x45\x33\xDC\x76\x15\xED\x0F\xA2\x71\x64\x4C\x65\x2E\x81\x68\x45\xA7\x02\x03\x01\x00\x01\x30\x0D\x06\x09\x2A\x86\x48\x86\xF7\x0D\x01\x01\x02\x05\x00\x03\x81\x81\x00\xBB\x4C\x12\x2B\xCF\x2C\x26\x00\x4F\x14\x13\xDD\xA6\xFB\xFC\x0A\x11\x84\x8C\xF3\x28\x1C\x67\x92\x2F\x7C\xB6\xC5\xFA\xDF\xF0\xE8\x95\xBC\x1D\x8F\x6C\x2C\xA8\x51\xCC\x73\xD8\xA4\xC0\x53\xF0\x4E\xD6\x26\xC0\x76\x01\x57\x81\x92\x5E\x21\xF1\xD1\xB1\xFF\xE7\xD0\x21\x58\xCD\x69\x17\xE3\x44\x1C\x9C\x19\x44\x39\x89\x5C\xDC\x9C\x00\x0F\x56\x8D\x02\x99\xED\xA2\x90\x45\x4C\xE4\xBB\x10\xA4\x3D\xF0\x32\x03\x0E\xF1\xCE\xF8\xE8\xC9\x51\x8C\xE6\x62\x9F\xE6\x9F\xC0\x7D\xB7\x72\x9C\xC9\x36\x3A\x6B\x9F\x4E\xA8\xFF\x64\x0D\x64" diff --git a/testing/btest/core/cisco-fabric-path.bro b/testing/btest/core/cisco-fabric-path.bro new file mode 100644 index 0000000000..ff7fa298e3 --- /dev/null +++ b/testing/btest/core/cisco-fabric-path.bro @@ -0,0 +1,2 @@ +# @TEST-EXEC: bro -C -r $TRACES/cisco-fabric-path.pcap +# @TEST-EXEC: btest-diff conn.log diff --git a/testing/btest/scripts/base/protocols/rdp/rdp-x509.bro b/testing/btest/scripts/base/protocols/rdp/rdp-x509.bro index ae1eb8b542..2fed0d7d19 100644 --- a/testing/btest/scripts/base/protocols/rdp/rdp-x509.bro +++ b/testing/btest/scripts/base/protocols/rdp/rdp-x509.bro @@ -1,5 +1,5 @@ # @TEST-EXEC: bro -r $TRACES/rdp/rdp-x509.pcap %INPUT # @TEST-EXEC: btest-diff rdp.log -# @TEST-EXEC: btest-diff x509.log +# @TEST-EXEC: TEST_DIFF_CANONIFIER="$SCRIPTS/diff-remove-timestamps | $SCRIPTS/diff-remove-x509-key-info" btest-diff x509.log @load base/protocols/rdp diff --git a/testing/btest/scripts/base/protocols/ssl/ocsp-http-get.test b/testing/btest/scripts/base/protocols/ssl/ocsp-http-get.test index ff48772b6a..c8c8acc589 100644 --- a/testing/btest/scripts/base/protocols/ssl/ocsp-http-get.test +++ b/testing/btest/scripts/base/protocols/ssl/ocsp-http-get.test @@ -17,9 +17,9 @@ event ocsp_extension(f: fa_file, ext: X509::Extension, global_resp: bool) print "extension: ", ext, global_resp; } -event ocsp_request(f: fa_file, version: count, requestorName: string) +event ocsp_request(f: fa_file, version: count) { - print "request", version, requestorName; + print "request", version, ""; } event ocsp_request_certificate(f: fa_file, hashAlgorithm: string, issuerNameHash: string, issuerKeyHash: string, serialNumber: string) diff --git a/testing/btest/scripts/base/protocols/ssl/ocsp-request-only.test b/testing/btest/scripts/base/protocols/ssl/ocsp-request-only.test index 0176716553..05483717b0 100644 --- a/testing/btest/scripts/base/protocols/ssl/ocsp-request-only.test +++ b/testing/btest/scripts/base/protocols/ssl/ocsp-request-only.test @@ -16,9 +16,9 @@ event ocsp_extension(f: fa_file, ext: X509::Extension, global_resp: bool) print "extension: ", ext, global_resp; } -event ocsp_request(f: fa_file, version: count, requestorName: string) +event ocsp_request(f: fa_file, version: count) { - print "request", version, requestorName; + print "request", version, ""; } event ocsp_request_certificate(f: fa_file, hashAlgorithm: string, issuerNameHash: string, issuerKeyHash: string, serialNumber: string) diff --git a/testing/btest/scripts/base/protocols/ssl/ocsp-request-response.test b/testing/btest/scripts/base/protocols/ssl/ocsp-request-response.test index 3adfab9aa2..b95203dfd8 100644 --- a/testing/btest/scripts/base/protocols/ssl/ocsp-request-response.test +++ b/testing/btest/scripts/base/protocols/ssl/ocsp-request-response.test @@ -17,9 +17,9 @@ event ocsp_extension(f: fa_file, ext: X509::Extension, global_resp: bool) print "extension: ", ext, global_resp; } -event ocsp_request(f: fa_file, version: count, requestorName: string) +event ocsp_request(f: fa_file, version: count) { - print "request", version, requestorName; + print "request", version, ""; } event ocsp_request_certificate(f: fa_file, hashAlgorithm: string, issuerNameHash: string, issuerKeyHash: string, serialNumber: string) diff --git a/testing/btest/scripts/base/protocols/ssl/ocsp-response-only.test b/testing/btest/scripts/base/protocols/ssl/ocsp-response-only.test index f99a71802c..43dbf82583 100644 --- a/testing/btest/scripts/base/protocols/ssl/ocsp-response-only.test +++ b/testing/btest/scripts/base/protocols/ssl/ocsp-response-only.test @@ -17,9 +17,9 @@ event ocsp_extension(f: fa_file, ext: X509::Extension, global_resp: bool) print "extension: ", ext, global_resp; } -event ocsp_request(f: fa_file, version: count, requestorName: string) +event ocsp_request(f: fa_file, version: count) { - print "request", version, requestorName; + print "request", version, ""; } event ocsp_request_certificate(f: fa_file, hashAlgorithm: string, issuerNameHash: string, issuerKeyHash: string, serialNumber: string) diff --git a/testing/btest/scripts/base/protocols/ssl/ocsp-revoked.test b/testing/btest/scripts/base/protocols/ssl/ocsp-revoked.test index ae39640f3f..e4378135ad 100644 --- a/testing/btest/scripts/base/protocols/ssl/ocsp-revoked.test +++ b/testing/btest/scripts/base/protocols/ssl/ocsp-revoked.test @@ -17,9 +17,9 @@ event ocsp_extension(f: fa_file, ext: X509::Extension, global_resp: bool) print "extension: ", ext, global_resp; } -event ocsp_request(f: fa_file, version: count, requestorName: string) +event ocsp_request(f: fa_file, version: count) { - print "request", version, requestorName; + print "request", version, ""; } event ocsp_request_certificate(f: fa_file, hashAlgorithm: string, issuerNameHash: string, issuerKeyHash: string, serialNumber: string) diff --git a/testing/scripts/diff-remove-x509-key-info b/testing/scripts/diff-remove-x509-key-info new file mode 100755 index 0000000000..85404fb30d --- /dev/null +++ b/testing/scripts/diff-remove-x509-key-info @@ -0,0 +1,55 @@ +#! /usr/bin/env bash +# +# A diff canonifier that removes all X.509 public key information +# which, in the specific case of the RDP protocol's misuse of +# md5WithRSAEncryption, seems that OpenSSL 1.0 is able to manually +# workaround by setting to rsaEncryption, but OpenSSL 1.1 still fails +# to extract the key, so the corresponding fields are always removed here. + +awk ' +BEGIN { FS="\t"; OFS="\t"; key_type_col = -1; key_length_col = -1; exponent_col = -1; curve_col = -1 } + +/^#/ { + if ( $1 == "#fields" ) + { + for ( i = 2; i <= NF; ++i ) + { + if ( $i == "certificate.key_type" ) + key_type_col = i-1; + if ( $i == "certificate.key_length" ) + key_length_col = i-1; + if ( $i == "certificate.exponent" ) + exponent_col = i-1; + if ( $i == "certificate.curve" ) + curve_col = i-1; + } + } + + print; + next; +} + +key_type_col > 0 { + # Mark it regardless of whether it is set. + $key_type_col = "x"; +} + +key_length_col > 0 { + # Mark it regardless of whether it is set. + $key_length_col = "x"; +} + +exponent_col > 0 { + # Mark it regardless of whether it is set. + $exponent_col = "x"; +} + +curve_col > 0 { + # Mark it regardless of whether it is set. + $curve_col = "x"; +} + +{ + print; +} +'