diff --git a/.gitmodules b/.gitmodules index 87826d2ef6..24375ce23d 100644 --- a/.gitmodules +++ b/.gitmodules @@ -16,9 +16,9 @@ [submodule "cmake"] path = cmake url = git://git.bro.org/cmake -[submodule "magic"] - path = magic - url = git://git.bro.org/bromagic [submodule "src/3rdparty"] path = src/3rdparty url = git://git.bro.org/bro-3rdparty +[submodule "aux/plugins"] + path = aux/plugins + url = git://git.bro.org/bro-plugins diff --git a/CHANGES b/CHANGES index 7751f1cb6f..bb795d114a 100644 --- a/CHANGES +++ b/CHANGES @@ -1,4 +1,1031 @@ +2.3-343 | 2014-12-12 12:43:46 -0800 + + * Fix PIA packet replay to deliver copy of IP header. This prevented + one from writing a packet-wise analyzer that needs access to IP + headers and can be attached to a connection via signature match. + Addresses BIT-1298 (Jon Siwek) + +2.3-338 | 2014-12-08 13:56:19 -0800 + + * Add man page for Bro. (Raúl Benencia) + + * Updating doc baselines. (Robin Sommer) + +2.3-334 | 2014-12-03 14:22:07 -0800 + + * Fix compound assignment to require proper L-value. Addresses + BIT-1295. (Jon Siwek) + +2.3-332 | 2014-12-03 14:14:11 -0800 + + * Make using local IDs in @if directives an error. Addresses + BIT-1296. (Jon Siwek) + +2.3-330 | 2014-12-03 14:10:39 -0800 + + * Fix some "make doc" warnings and update some doc tests. (Daniel + Thayer) + +2.3-328 | 2014-12-02 08:13:10 -0500 + + * Update windows-version-detection.bro to add support for + Windows 10. (Michal Purzynski) + +2.3-326 | 2014-12-01 12:10:27 -0600 + + * BIFScanner: fix invalid characters in generated preprocessor macros. + (Hilko Bengen) + + * BIT-1294: fix exec.bro from mutating Input::end_of_data event + parameters. (Johanna Amann) + + * Add/invoke "distclean" for testing directories. (Raúl Benencia) + + * Delete prebuilt python bytecode files from git. (Jon Siwek) + + * Add Windows detection based on CryptoAPI HTTP traffic as a software + framework policy script. (Vlad Grigorescu) + +2.3-316 | 2014-11-25 17:35:06 -0800 + + * Make the SSL analyzer skip further processing once encountering + situations which are very probably non-recoverable. (Johanna + Amann) + +2.3-313 | 2014-11-25 14:27:07 -0800 + + * Make SSL v2 protocol tests more strict. In its former state they + triggered on http traffic over port 443 sometimes. Found by Michał + Purzyński. (Johanna Amann) + + * Fix X509 analyzer to correctly return ECDSA as the key_type for + ECDSA certs. Bug found by Michał Purzyński. (Johanna Amann) + +2.3-310 | 2014-11-19 10:56:59 -0600 + + * Disable verbose bison output. (Jon Siwek) + +2.3-309 | 2014-11-18 12:17:53 -0800 + + * New decompose_uri() function in base/utils/urls that splits a URI + into its pieces. (Anthony Kasza). + +2.3-305 | 2014-11-18 11:09:04 -0800 + + * Improve coercion of &default expressions. Addresses BIT-1288. (Jon + Siwek) + +2.3-303 | 2014-11-18 10:53:04 -0800 + + * For DH key exchanges, use p as the parameter for weak key + exchanges. (Johanna Amann) + +2.3-301 | 2014-11-11 13:47:27 -0800 + + * Add builtin function enum_to_int() that converts an enum into a + integer. (Christian Struck) + +2.3-297 | 2014-11-11 11:50:47 -0800 + + * Removing method from SSL analyzer that's no longer used. (Robin + Sommer) + +2.3-296 | 2014-11-11 11:42:38 -0800 + + * A new analyzer parsing the MySQL wire protocol. Activity gets + logged into mysql.log. Supports protocol versions 9 and 10. (Vlad + Grigorescu) + +2.3-280 | 2014-11-05 09:46:33 -0500 + + * Add Windows detection based on CryptoAPI HTTP traffic as a + software framework policy script. (Vlad Grigorescu) + +2.3-278 | 2014-11-03 18:55:18 -0800 + + * Add new curves from draft-ietf-tls-negotiated-ff-dhe to SSL + analysis. (Johanna Amann) + +2.3-274 | 2014-10-31 17:45:25 -0700 + + * Adding call to new binpac::init() function. (Robin Sommer) + +2.3-272 | 2014-10-31 16:29:42 -0700 + + * Fix segfault if when statement's RHS is unitialized. Addresses + BIT-1176. (Jon Siwek) + + * Fix checking vector indices via "in". Addresses BIT-1280. (Jon + Siwek) + +2.3-268 | 2014-10-31 12:12:22 -0500 + + * BIT-1283: Fix crash when using &encrypt. (Jon Siwek) + +2.3-267 | 2014-10-31 10:35:02 -0500 + + * BIT-1284: Allow arbitrary when statement timeout expressions + (Jon Siwek) + +2.3-266 | 2014-10-31 09:21:28 -0500 + + * BIT-1166: Add configure options to fine tune local state dirs used + by BroControl. (Jon Siwek) + +2.3-264 | 2014-10-30 13:25:57 -0500 + + * Fix some minor Coverity Scan complaints. (Jon Siwek) + +2.3-263 | 2014-10-28 15:09:10 -0500 + + * Fix checking of fwrite return values (Johanna Amann) + +2.3-260 | 2014-10-27 12:54:17 -0500 + + * Fix errors/warnings when compiling with -std=c++11 (Jon Siwek) + +2.3-259 | 2014-10-27 10:04:04 -0500 + + * Documentation fixes. (Vicente Jimenez Aguilar and Stefano Azzalini) + +2.3-256 | 2014-10-24 15:33:45 -0700 + + * Adding missing test baseline. (Robin Sommer) + +2.3-255 | 2014-10-24 13:39:44 -0700 + + * Fixing unstable active-http test. (Robin Sommer) + +2.3-254 | 2014-10-24 11:40:51 -0700 + + * Fix active-http.bro to deal reliably with empty server responses, + which will now be passed back as empty files. (Christian Struck) + +2.3-248 | 2014-10-23 14:20:59 -0700 + + * Change order in which a plugin's scripts are loaded at startup. + (Robin Sommer) + +2.3-247 | 2014-10-21 13:42:38 -0700 + + * Updates to the SSL analyzer. (Johanna Amann) + + * Mark everything below 2048 bit as a weak key. + + * Fix notice suppression. + + * Add information about server-chosen protocol to ssl.log, if + provided by application_layer_next_protocol. + + * Add boolean flag to ssl.log signaling if a session was + resumed. Remove the (usually not really that useful) session + ID that the client sent. + +2.3-240 | 2014-10-21 13:36:33 -0700 + + * Fix Coverity-reported issues in DNP3 analyzer. (Seth Hall) + +2.3-238 | 2014-10-16 06:51:49 -0700 + + * Fix multipart HTTP/MIME entity file analysis so that (1) singular + CR or LF characters in multipart body content are no longer + converted to a full CRLF (thus corrupting the file) and (2) it + also no longer considers the CRLF before the multipart boundary as + part of the content. Addresses BIT-1235. (Jon Siwek) + +2.3-235 | 2014-10-15 10:20:47 -0500 + + * BIT-1273: Add error message for bad enum declaration syntax. + (Jon Siwek) + +2.3-234 | 2014-10-14 14:42:09 -0500 + + * Documentation fixes. (Steve Smoot) + +2.3-233 | 2014-10-09 16:00:27 -0500 + + * Change find-bro-logs unit test to follow symlinks. (Jon Siwek) + + * Add error checks and messages to a test script (Daniel Thayer) + +2.3-230 | 2014-10-08 08:15:17 -0700 + + * Further baseline normalization for plugin test portability. (Robin + Sommer) + +2.3-229 | 2014-10-07 20:18:11 -0700 + + * Fix for test portability. (Robin Sommer) + +2.3-228 | 2014-10-07 15:32:37 -0700 + + * Include plugin unit tests into the top-level btest configuration. (Robin Sommer) + + * Switching the prefix separator for packet source/dumper plugins + once more, now to "::". Addresses BIT-1267. (Robin Sommer) + + * Fix for allowing a packet source/dumper plugin to support multiple + prefixes with a colon. (Robin Sommer) + +2.3-225 | 2014-10-07 15:13:35 -0700 + + * Updating plugin documentation. (Robin Sommer) + +2.3-224 | 2014-10-07 14:32:17 -0700 + + * Improved the log file reference documentation. (Jeannette Dopheide + and Daniel Thayer) + + * Improves shockwave flash file signatures. (Seth Hall) + + - This moves the signatures out of the libmagic imported signatures + and into our own general.sig. + + - Expand the detection to LZMA compressed flash files. + + * Add new script language reference documentation on operators, + statements, and directives. Also improved the documentation on + types and attributes by splitting them into two docs, and + providing more examples and adding a chart on the top of each page + with links to each type and attribute for easier access to the + information. (Daniel Thayer) + + * Split the types and attributes reference doc into two docs. + (Daniel Thayer) + +2.3-208 | 2014-10-03 09:38:52 -0500 + + * BIT-1268: Fix uninitialized router_list argument in + dhcp_offer/dhcp_ack. (Jon Siwek) + +2.3-207 | 2014-10-02 16:39:17 -0700 + + * Updating plugin docs. (Robin Sommer) + + * Fix packet sources being treated as idle when a packet is + available. Addresses BIT-1266. (Jon Siwek) + + * Fix regression causing the main loop to spin more frequently. + Addresses BIT-1266. (Jon Siwek) + +2.3-203 | 2014-09-29 20:06:54 -0700 + + * Fix to use length parameter in DNP3 time conversion correctly now. + (Robin Sommer) + +2.3-202 | 2014-09-29 17:05:18 -0700 + + * New SSL extension type from IANA and a few other SSL const + changes. (Johanna Amann) + + * Make unexpected pipe errors fatal as precaution. Addresses + BIT-1260. (Jon Siwek) + + * Adding a function for DNP3 to translate the timestamp format. (Hui + Lin) + +2.3-197 | 2014-09-29 10:42:01 -0500 + + * Fix possible seg fault in TCP reassembler. (Jon Siwek) + +2.3-196 | 2014-09-25 17:53:27 -0700 + + * Changing prefix for packet sources/dumper from ':' to '%'. + Addresses BIT-1249. (Robin Sommer) + + * Remove timeouts from remote communication loop. The select() now + blocks until there's work to do instead of relying on a small + timeout value which can cause unproductive use of cpu cycles. (Jon + Siwek) + + * Improve error message when failing to activate a plugin. Also fix + a unit test helper script that checks plugin availability. (Jon + Siwek) + +2.3-183 | 2014-09-24 10:08:04 -0500 + + * Add a "node" field to Intel::Seen struture and intel.log to + indicate which node discovered a hit on an intel item. (Seth Hall) + + * BIT-1261: Fixes to plugin quick start doc. (Jon Siwek) + +2.3-180 | 2014-09-22 12:52:41 -0500 + + * BIT-1259: Fix issue w/ duplicate TCP reassembly deliveries. + (Jon Siwek) + +2.3-178 | 2014-09-18 14:29:46 -0500 + + * BIT-1256: Fix file analysis events from coming after bro_done(). + (Jon Siwek) + +2.3-177 | 2014-09-17 09:41:27 -0500 + + * Documentation fixes. (Chris Mavrakis) + +2.3-174 | 2014-09-17 09:37:09 -0500 + + * Fixed some "make doc" warnings caused by reST formatting + (Daniel Thayer). + +2.3-172 | 2014-09-15 13:38:52 -0500 + + * Remove unneeded allocations for HTTP messages. (Jon Siwek) + +2.3-171 | 2014-09-15 11:14:57 -0500 + + * Fix a compile error on systems without pcap-int.h. (Jon Siwek) + +2.3-170 | 2014-09-12 19:28:01 -0700 + + * Fix incorrect data delivery skips after gap in HTTP Content-Range. + Addresses BIT-1247. (Jon Siwek) + + * Fix file analysis placement of data after gap in HTTP + Content-Range. Addresses BIT-1248. (Jon Siwek) + + * Fix issue w/ TCP reassembler not delivering some segments. + Addresses BIT-1246. (Jon Siwek) + + * Fix MIME entity file data/gap ordering and raise http_entity_data + in line with data arrival. Addresses BIT-1240. (Jon Siwek) + + * Implement file ID caching for MIME_Mail. (Jon Siwek) + + * Fix a compile error. (Jon Siwek) + +2.3-161 | 2014-09-09 12:35:38 -0500 + + * Bugfixes and test updates/additions. (Robin Sommer) + + * Interface tweaks and docs for PktSrc/PktDumper. (Robin Sommer) + + * Moving PCAP-related bifs to iosource/pcap.bif. (Robin Sommer) + + * Moving some of the BPF filtering code into base class. + This will allow packet sources that don't support BPF natively to + emulate the filtering via libpcap. (Robin Sommer) + + * Removing FlowSrc. (Robin Sommer) + + * Removing remaining pieces of the 2ndary path, and left-over + files of packet sorter. (Robin Sommer) + + * A bunch of infrastructure work to move IOSource, IOSourceRegistry + (now iosource::Manager) and PktSrc/PktDumper code into iosource/, + and over to a plugin structure. (Robin Sommer) + +2.3-137 | 2014-09-08 19:01:13 -0500 + + * Fix Broxygen's rendering of opaque types. (Jon Siwek) + +2.3-136 | 2014-09-07 20:50:46 -0700 + + * Change more http links to https. (Johanna Amann) + +2.3-134 | 2014-09-04 16:16:36 -0700 + + * Fixed a number of issues with OCSP reply validation. Addresses + BIT-1212. (Johanna Amann) + + * Fix null pointer dereference in OCSP verification code in case no + certificate is sent as part as the ocsp reply. Addresses BIT-1212. + (Johanna Amann) + +2.3-131 | 2014-09-04 16:10:32 -0700 + + * Make links in documentation templates protocol relative. (Johanna + Amann) + +2.3-129 | 2014-09-02 17:21:21 -0700 + + * Simplify a conditional with equivalent branches. (Jon Siwek) + + * Change EDNS parsing code to use rdlength more cautiously. (Jon + Siwek) + + * Fix a memory leak when bind() fails due to EADDRINUSE. (Jon Siwek) + + * Fix possible buffer over-read in DNS TSIG parsing. (Jon Siwek) + +2.3-124 | 2014-08-26 09:24:19 -0500 + + * Better documentation for sub_bytes (Jimmy Jones) + + * BIT-1234: Fix build on systems that already have ntohll/htonll + (Jon Siwek) + +2.3-121 | 2014-08-22 15:22:15 -0700 + + * Detect functions that try to bind variables from an outer scope + and raise an error saying that's not supported. Addresses + BIT-1233. (Jon Siwek) + +2.3-116 | 2014-08-21 16:04:13 -0500 + + * Adding plugin testing to Makefile's test-all. (Robin Sommer) + + * Converting log writers and input readers to plugins. + DataSeries and ElasticSearch plugins have moved to the new + bro-plugins repository, which is now a git submodule in the + aux/plugins directory. (Robin Sommer) + +2.3-98 | 2014-08-19 11:03:46 -0500 + + * Silence some doc-related warnings when using `bro -e`. + Closes BIT-1232. (Jon Siwek) + + * Fix possible null ptr derefs reported by Coverity. (Jon Siwek) + +2.3-96 | 2014-08-01 14:35:01 -0700 + + * Small change to DHCP documentation. In server->client messages the + host name may differ from the one requested by the client. + (Johanna Amann) + + * Split DHCP log writing from record creation. This allows users to + customize dhcp.log by changing the record in their own dhcp_ack + event. (Johanna Amann) + + * Update PATH so that documentation btests can find bro-cut. (Daniel + Thayer) + + * Remove gawk from list of optional packages in documentation. + (Daniel Thayer) + + * Fix for redefining built-in constants. (Robin Sommer) + +2.3-86 | 2014-07-31 14:19:58 -0700 + + * Fix for redefining built-in constants. (Robin Sommer) + + * Adding missing check that a plugin's API version matches what Bro + defines. (Robin Sommer) + + * Adding NEWS entry for plugins. (Robin Sommer) + +2.3-83 | 2014-07-30 16:26:11 -0500 + + * Minor adjustments to plugin code/docs. (Jon Siwek) + + * Dynamic plugin support. (Rpbin Sommer) + + Bro now supports extending core functionality, like protocol and + file analysis, dynamically with external plugins in the form of + shared libraries. See doc/devel/plugins.rst for an overview of the + main functionality. Changes coming with this: + + - Replacing the old Plugin macro magic with a new API. + + - The plugin API changed to generally use std::strings instead + of const char*. + + - There are a number of invocations of PLUGIN_HOOK_ + {VOID,WITH_RESULT} across the code base, which allow plugins + to hook into the processing at those locations. + + - A few new accessor methods to various classes to allow + plugins to get to that information. + + - network_time cannot be just assigned to anymore, there's now + function net_update_time() for that. + + - Redoing how builtin variables are initialized, so that it + works for plugins as well. No more init_net_var(), but + instead bifcl-generated code that registers them. + + - Various changes for adjusting to the now dynamic generation + of analyzer instances. + + - same_type() gets an optional extra argument allowing record type + comparision to ignore if field names don't match. (Robin Sommer) + + - Further unify file analysis API with the protocol analyzer API + (assigning IDs to analyzers; adding Init()/Done() methods; + adding subtypes). (Robin Sommer) + + - A new command line option -Q that prints some basic execution + time stats. (Robin Sommer) + + - Add support to the file analysis for activating analyzers by + MIME type. (Robin Sommer) + + - File::register_for_mime_type(tag: Analyzer::Tag, mt: + string): Associates a file analyzer with a MIME type. + + - File::add_analyzers_for_mime_type(f: fa_file, mtype: + string): Activates all analyzers registered for a MIME + type for the file. + + - The default file_new() handler calls + File::add_analyzers_for_mime_type() with the file's MIME + type. + +2.3-20 | 2014-07-22 17:41:02 -0700 + + * Updating submodule(s). + +2.3-19 | 2014-07-22 17:29:19 -0700 + + * Implement bytestring_to_coils() in Modbus analyzer so that coils + gets passed to the corresponding events. (Hui Lin) + + * Add length field to ModbusHeaders. (Hui Lin) + +2.3-12 | 2014-07-10 19:17:37 -0500 + + * Include yield of vectors in Broxygen's type descriptions. + Addresses BIT-1217. (Jon Siwek) + +2.3-11 | 2014-07-10 14:49:27 -0700 + + * Fixing DataSeries output. It was using a now illegal value as its + default compression level. (Robin Sommer) + +2.3-7 | 2014-06-26 17:35:18 -0700 + + * Extending "make test-all" to include aux/bro-aux. (Robin Sommer) + +2.3-6 | 2014-06-26 17:24:10 -0700 + + * DataSeries compilation issue fixed. (mlaterman) + + * Fix a reference counting bug in ListVal ctor. (Jon Siwek) + +2.3-3 | 2014-06-26 15:41:04 -0500 + + * Support tilde expansion when Bro tries to find its own path. (Jon + Siwek) + +2.3-2 | 2014-06-23 16:54:15 -0500 + + * Remove references to line numbers in tutorial text. (Daniel Thayer) + +2.3 | 2014-06-16 09:48:25 -0500 + + * Release 2.3. + +2.3-beta-33 | 2014-06-12 11:59:28 -0500 + + * Documentation improvements/fixes. (Daniel Thayer) + +2.3-beta-24 | 2014-06-11 15:35:31 -0500 + + * Fix SMTP state tracking when server response is missing. + (Robin Sommer) + +2.3-beta-22 | 2014-06-11 12:31:38 -0500 + + * Fix doc/test that broke due to a Bro script change. (Jon Siwek) + + * Remove unused --with-libmagic configure option. (Jon Siwek) + +2.3-beta-20 | 2014-06-10 18:16:51 -0700 + + * Fix use-after-free in some cases of reassigning a table index. + Addresses BIT-1202. (Jon Siwek) + +2.3-beta-18 | 2014-06-06 13:11:50 -0700 + + * Add two more SSL events, one triggered for each handshake message + and one triggered for the tls change cipherspec message. (Bernhard + Amann) + + * Small SSL bug fix. In case SSL::disable_analyzer_after_detection + was set to false, the ssl_established event would fire after each + data packet once the session is established. (Bernhard Amann) + +2.3-beta-16 | 2014-06-06 13:05:44 -0700 + + * Re-activate notice suppression for expiring certificates. + (Bernhard Amann) + +2.3-beta-14 | 2014-06-05 14:43:33 -0700 + + * Add new TLS extension type numbers from IANA (Bernhard Amann) + + * Switch to double hashing for Bloomfilters for better performance. + (Matthias Vallentin) + + * Bugfix to use full digest length instead of just one byte for + Bloomfilter's universal hash function. Addresses BIT-1140. + (Matthias Vallentin) + + * Make buffer for X509 certificate subjects larger. Addresses + BIT-1195 (Bernhard Amann) + +2.3-beta-5 | 2014-05-29 15:34:42 -0500 + + * Fix misc/load-balancing.bro's reference to + PacketFilter::sampling_filter (Jon Siwek) + +2.3-beta-4 | 2014-05-28 14:55:24 -0500 + + * Fix potential mem leak in remote function/event unserialization. + (Jon Siwek) + + * Fix reference counting bug in table coercion expressions (Jon Siwek) + + * Fix an "unused value" warning. (Jon Siwek) + + * Remove a duplicate unit test baseline dir. (Jon Siwek) + +2.3-beta | 2014-05-19 16:36:50 -0500 + + * Release 2.3-beta + + * Clean up OpenSSL data structures on exit. (Bernhard Amann) + + * Fixes for OCSP & x509 analysis memory leak issues. (Bernhard Amann) + + * Remove remaining references to BROMAGIC (Daniel Thayer) + + * Fix typos and formatting in event and BiF documentation (Daniel Thayer) + + * Update intel framework plugin for ssl server_name extension API + changes. (Bernhard Amann, Justin Azoff) + + * Fix expression errors in SSL/x509 scripts when unparseable data + is in certificate chain. (Bernhard Amann) + +2.2-478 | 2014-05-19 15:31:33 -0500 + + * Change record ctors to only allow record-field-assignment + expressions. (Jon Siwek) + +2.2-477 | 2014-05-19 14:13:00 -0500 + + * Fix X509::Result record's "result" field to be set internally as type int instead of type count. (Bernhard Amann) + + * Fix a couple of doc build warnings (Daniel Thayer) + +2.2-470 | 2014-05-16 15:16:32 -0700 + + * Add a new section "Cluster Configuration" to the docs that is + intended as a how-to for configuring a Bro cluster. Most of this + content was moved here from the BroControl doc (which is now + intended as more of a reference guide for more experienced users) + and the load balancing FAQ on the website. (Daniel Thayer) + + * Update some doc tests and line numbers (Daniel Thayer) + +2.2-457 | 2014-05-16 14:38:31 -0700 + + * New script policy/protocols/ssl/validate-ocsp.bro that adds OSCP + validation to ssl.log. The work is done by a new bif + x509_ocsp_verify(). (Bernhard Amann) + + * STARTTLS support for POP3 and SMTP. The SSL analyzer takes over + when seen. smtp.log now logs when a connection switches to SSL. + (Bernhard Amann) + + * Replace errors when parsing x509 certs with weirds. (Bernhard + Amann) + + * Improved Heartbleed attack/scan detection. (Bernhard Amann) + + * Let TLS analyzer fail better when no longer in sync with the data + stream. (Bernhard Amann) + +2.2-444 | 2014-05-16 14:10:32 -0500 + + * Disable all default AppStat plugins except facebook. (Jon Siwek) + + * Update for the active http test to force it to use ipv4. (Seth Hall) + +2.2-441 | 2014-05-15 11:29:56 -0700 + + * A new RADIUS analyzer. (Vlad Grigorescu) + + It produces a radius.log and generates two events: + + event radius_message(c: connection, result: RADIUS::Message); + event radius_attribute(c: connection, attr_type: count, value: string); + +2.2-427 | 2014-05-15 13:37:23 -0400 + + * Fix dynamic SumStats update on clusters (Bernhard Amann) + +2.2-425 | 2014-05-08 16:34:44 -0700 + + * Fix reassembly of data w/ sizes beyond 32-bit capacities. (Jon Siwek) + + Reassembly code (e.g. for TCP) now uses int64/uint64 (signedness + is situational) data types in place of int types in order to + support delivering data to analyzers that pass 2GB thresholds. + There's also changes in logic that accompany the change in data + types, e.g. to fix TCP sequence space arithmetic inconsistencies. + + Another significant change is in the Analyzer API: the *Packet and + *Undelivered methods now use a uint64 in place of an int for the + relative sequence space offset parameter. + + Addresses BIT-348. + + * Fixing compiler warnings. (Robin Sommer) + + * Update SNMP analyzer's DeliverPacket method signature. (Jon Siwek) + +2.2-417 | 2014-05-07 10:59:22 -0500 + + * Change handling of atypical OpenSSL error case in x509 verification. (Jon Siwek) + + * Fix memory leaks in X509 certificate parsing/verification. (Jon Siwek) + + * Fix new []/delete mismatch in input::reader::Raw::DoClose(). (Jon Siwek) + + * Fix buffer over-reads in file_analysis::Manager::Terminate() (Jon Siwek) + + * Fix buffer overlows in IP address masking logic. (Jon Siwek) + + That could occur either in taking a zero-length mask on an IPv6 address + (e.g. [fe80::]/0) or a reverse mask of length 128 on any address (e.g. + via the remask_addr BuiltIn Function). + + * Fix new []/delete mismatch in ~Base64Converter. (Jon Siwek) + +2.2-410 | 2014-05-02 12:49:53 -0500 + + * Replace an unneeded OPENSSL_malloc call. (Jon Siwek) + +2.2-409 | 2014-05-02 12:09:06 -0500 + + * Clean up and documentation for base SNMP script. (Jon Siwek) + + * Update base SNMP script to now produce a snmp.log. (Seth Hall) + + * Add DH support to SSL analyzer. When using DHE or DH-Anon, sever + key parameters are now available in scriptland. Also add script to + alert on weak certificate keys or weak dh-params. (Bernhard Amann) + + * Add a few more ciphers Bro did not know at all so far. (Bernhard Amann) + + * Log chosen curve when using ec cipher suite in TLS. (Bernhard Amann) + +2.2-397 | 2014-05-01 20:29:20 -0700 + + * Fix reference counting for lookup_ID() usages. (Jon Siwek) + +2.2-395 | 2014-05-01 20:25:48 -0700 + + * Fix missing "irc-dcc-data" service field from IRC DCC connections. + (Jon Siwek) + + * Correct a notice for heartbleed. The notice is thrown correctly, + just the message conteined wrong values. (Bernhard Amann) + + * Improve/standardize some malloc/realloc return value checks. (Jon + Siwek) + + * Improve file analysis manager shutdown/cleanup. (Jon Siwek) + +2.2-388 | 2014-04-24 18:38:07 -0700 + + * Fix decoding of MIME quoted-printable. (Mareq) + +2.2-386 | 2014-04-24 18:22:29 -0700 + + * Do a Intel::ADDR lookup for host field if we find an IP address + there. (jshlbrd) + +2.2-381 | 2014-04-24 17:08:45 -0700 + + * Add Java version to software framework. (Brian Little) + +2.2-379 | 2014-04-24 17:06:21 -0700 + + * Remove unused Val::attribs member. (Jon Siwek) + +2.2-377 | 2014-04-24 16:57:54 -0700 + + * A larger set of SSL improvements and extensions. Addresses + BIT-1178. (Bernhard Amann) + + - Fixes TLS protocol version detection. It also should + bail-out correctly on non-tls-connections now + + - Adds support for a few TLS extensions, including + server_name, alpn, and ec-curves. + + - Adds support for the heartbeat events. + + - Add Heartbleed detector script. + + - Adds basic support for OCSP stapling. + + * Fix parsing of DNS TXT RRs w/ multiple character-strings. + Addresses BIT-1156. (Jon Siwek) + +2.2-353 | 2014-04-24 16:12:30 -0700 + + * Adapt HTTP partial content to cache file analysis IDs. (Jon Siwek) + + * Adapt SSL analyzer to generate file analysis handles itself. (Jon + Siwek) + + * Adapt more of HTTP analyzer to use cached file analysis IDs. (Jon + Siwek) + + * Adapt IRC/FTP analyzers to cache file analysis IDs. (Jon Siwek) + + * Refactor regex/signature AcceptingSet data structure and usages. + (Jon Siwek) + + * Enforce data size limit when checking files for MIME matches. (Jon + Siwek) + + * Refactor file analysis file ID lookup. (Jon Siwek) + +2.2-344 | 2014-04-22 20:13:30 -0700 + + * Refactor various hex escaping code. (Jon Siwek) + +2.2-341 | 2014-04-17 18:01:41 -0500 + + * Fix duplicate DNS log entries. (Robin Sommer) + +2.2-341 | 2014-04-17 18:01:01 -0500 + + * Refactor initialization of ASCII log writer options. (Jon Siwek) + + * Fix a memory leak in ASCII log writer. (Jon Siwek) + +2.2-338 | 2014-04-17 17:48:17 -0500 + + * Disable input/logging threads setting their names on every + heartbeat. (Jon Siwek) + + * Fix bug when clearing Bloom filter contents. Reported by + @colonelxc. (Matthias Vallentin) + +2.2-335 | 2014-04-10 15:04:57 -0700 + + * Small logic fix for main SSL script. (Bernhard Amann) + + * Update DPD signatures for detecting TLS 1.2. (Bernhard Amann) + + * Remove unused data member of SMTP_Analyzer to silence a Coverity + warning. (Jon Siwek) + + * Fix missing @load dependencies in some scripts. Also update the + unit test which is supposed to catch such errors. (Jon Siwek) + +2.2-326 | 2014-04-08 15:21:51 -0700 + + * Add SNMP datagram parsing support.This supports parsing of SNMPv1 + (RFC 1157), SNMPv2 (RFC 1901/3416), and SNMPv2 (RFC 3412). An + event is raised for each SNMP PDU type, though there's not + currently any event handlers for them and not a default snmp.log + either. However, simple presence of SNMP is currently visible now + in conn.log service field and known_services.log. (Jon Siwek) + +2.2-319 | 2014-04-03 15:53:25 -0700 + + * Improve __load__.bro creation for .bif.bro stubs. (Jon Siwek) + +2.2-317 | 2014-04-03 10:51:31 -0400 + + * Add a uid field to the signatures.log. Addresses BIT-1171 + (Anthony Verez) + +2.2-315 | 2014-04-01 16:50:01 -0700 + + * Change logging's "#types" description of sets to "set". Addresses + BIT-1163 (Bernhard Amann) + +2.2-313 | 2014-04-01 16:40:19 -0700 + + * Fix a couple nits reported by Coverity.(Jon Siwek) + + * Fix potential memory leak in IP frag reassembly reported by + Coverity. (Jon Siwek) + +2.2-310 | 2014-03-31 18:52:22 -0700 + + * Fix memory leak and unchecked dynamic cast reported by Coverity. + (Jon Siwek) + + * Fix potential memory leak in x509 parser reported by Coverity. + (Bernhard Amann) + +2.2-304 | 2014-03-30 23:05:54 +0200 + + * Replace libmagic w/ Bro signatures for file MIME type + identification. Addresses BIT-1143. (Jon Siwek) + + Includes: + + - libmagic is no longer used at all. All MIME type detection is + done through new Bro signatures, and there's no longer a means + to get verbose file type descriptions. The majority of the + default file magic signatures are derived from the default magic + database of libmagic ~5.17. + + - File magic signatures consist of two new constructs in the + signature rule parsing grammar: "file-magic" gives a regular + expression to match against, and "file-mime" gives the MIME type + string of content that matches the magic and an optional strength + value for the match. + + - Modified signature/rule syntax for identifiers: they can no + longer start with a '-', which made for ambiguous syntax when + doing negative strength values in "file-mime". Also brought + syntax for Bro script identifiers in line with reality (they + can't start with numbers or include '-' at all). + + - A new built-in function, "file_magic", can be used to get all + file magic matches and their corresponding strength against a + given chunk of data. + + - The second parameter of the "identify_data" built-in function + can no longer be used to get verbose file type descriptions, + though it can still be used to get the strongest matching file + magic signature. + + - The "file_transferred" event's "descr" parameter no longer + contains verbose file type descriptions. + + - The BROMAGIC environment variable no longer changes any behavior + in Bro as magic databases are no longer used/installed. + + - Removed "binary" and "octet-stream" mime type detections. They + don' provide any more information than an uninitialized + mime_type field which implicitly means no magic signature + matches and so the media type is unknown to Bro. + + - The "fa_file" record now contains a "mime_types" field that + contains all magic signatures that matched the file content + (where the "mime_type" field is just a shortcut for the + strongest match). + + - Reverted back to minimum requirement of CMake 2.6.3 from 2.8.0. + + * The logic for adding file ids to {orig,resp}_fuids fields of the + http.log incorrectly depended on the state of + {orig,resp}_mime_types fields, so sometimes not all file ids + associated w/ the session were logged. (Jon Siwek) + + * Fix MHR script's use of fa_file$mime_type before checking if it's + initialized. (Jon Siwek) + +2.2-294 | 2014-03-30 22:08:25 +0200 + + * Rework and move X509 certificate processing from the SSL protocol + analyzer to a dedicated file analyzer. This will allow us to + examine X509 certificates from sources other than SSL in the + future. Furthermore, Bro now parses more fields and extensions + from the certificates (e.g. elliptic curve information, subject + alternative names, basic constraints). Certificate validation also + was improved, should be easier to use and exposes information like + the full verified certificate chain. (Bernhard Amann) + + This update changes the format of ssl.log and adds a new x509.log + with certificate information. Furthermore all x509 events and + handling functions have changed. + +2.2-271 | 2014-03-30 20:25:17 +0200 + + * Add unit tests covering vector/set/table ctors/inits. (Jon Siwek) + + * Fix parsing of "local" named table constructors. (Jon Siwek) + + * Improve type checking of records. Addresses BIT-1159. (Jon Siwek) + +2.2-267 | 2014-03-30 20:21:43 +0200 + + * Improve documentation of Bro clusters. Addresses BIT-1160. + (Daniel Thayer) + +2.2-263 | 2014-03-30 20:19:05 +0200 + + * Don't include locations into serialization when cloning values. + (Robin Sommer) + +2.2-262 | 2014-03-30 20:12:47 +0200 + + * Refactor SerializationFormat::EndWrite and ChunkedIO::Chunk memory + management. (Jon Siwek) + + * Improve SerializationFormat's write buffer growth strategy. (Jon + Siwek) + + * Add --parse-only option to exit after parsing scripts. May be + useful for syntax-checking tools. (Jon Siwek) + +2.2-256 | 2014-03-30 19:57:28 +0200 + + * For the summary statistics framewirk, change all &create_expire + attributes to &read_expire in the cluster part. (Bernhard Amann) + +2.2-254 | 2014-03-30 19:55:22 +0200 + + * Update instructions on how to build Bro docs. (Daniel Thayer) + +2.2-251 | 2014-03-28 08:37:37 -0400 + + * Quick fix to the ElasticSearch writer. (Seth Hall) + 2.2-250 | 2014-03-19 17:20:55 -0400 * Improve performance of MHR script by reducing cloned Vals in @@ -24,7 +1051,7 @@ formatter work. * Fixing compiler error. (Robin Sommer) - + * Fixing (very unlikely) double delete in HTTP analyzer when decapsulating CONNECTs. (Robin Sommer) @@ -50,7 +1077,7 @@ Addresses BIT-1134. (Jon Siwek) * Enable fake DNS mode for test suites. - + * Improve analysis of TCP SYN/SYN-ACK reversal situations. (Jon Siwek) @@ -82,7 +1109,7 @@ (Jon Siwek) * Silences some documentation warnings from Sphinx. (Jon Siwek) - + 2.2-215 | 2014-03-10 11:10:15 -0700 * Fix non-deterministic logging of unmatched DNS msgs. Addresses @@ -120,7 +1147,7 @@ HTTP traffic. (Seth Hall) * Fixing removal of support analyzers. (Robin Sommer) - + 2.2-199 | 2014-03-03 16:34:20 -0800 * Allow iterating over bif functions with result type vector of any. @@ -135,11 +1162,11 @@ 2.2-194 | 2014-02-28 14:50:53 -0800 * Remove packet sorter. Addresses BIT-700. (Bernhard Amann) - + 2.2-192 | 2014-02-28 09:46:43 -0800 * Update Mozilla root bundle. (Bernhard Amann) - + 2.2-190 | 2014-02-27 07:34:44 -0800 * Adjust timings of a few leak tests. (Bernhard Amann) @@ -169,7 +1196,7 @@ 2.2-177 | 2014-02-20 17:27:46 -0800 * Update to libmagic version 5.17. Addresses BIT-1136. (Jon Siwek) - + 2.2-174 | 2014-02-14 12:07:04 -0800 * Support for MPLS over VLAN. (Chris Kanich) @@ -198,7 +1225,7 @@ defined, but we see it being actively used. (Bernhard Amann) * Test baseline updates for DNS change. (Robin Sommer) - + 2.2-158 | 2014-02-09 23:45:39 -0500 * Change dns.log to include only standard DNS queries. (Jon Siwek) @@ -274,7 +1301,7 @@ 2.2-128 | 2014-01-30 15:58:47 -0800 * Add leak test for Exec module. (Bernhard Amann) - + * Fix file_over_new_connection event to trigger when entire file is missed. (Jon Siwek) @@ -287,7 +1314,7 @@ "detect_filtered_trace". (Jon Siwek) * Improve TCP FIN retransmission handling. (Jon Siwek) - + 2.2-120 | 2014-01-28 10:25:23 -0800 * Fix and extend x509_extension() event, which now actually returns @@ -305,10 +1332,10 @@ 2.2-115 | 2014-01-22 12:11:18 -0800 * Add unit tests for new Bro Manual docs. (Jon Siwek) - + * New content for the "Using Bro" section of the manual. (Rafael Bonilla/Jon Siwek) - + 2.2-105 | 2014-01-20 12:16:48 -0800 * Support GRE tunnel decapsulation, including enhanced GRE headers. @@ -317,7 +1344,7 @@ Addresses BIT-867. (Jon Siwek) * Simplify FragReassembler memory management. (Jon Siwek) - + 2.2-102 | 2014-01-20 12:00:29 -0800 * Include file information (MIME type and description) into notice @@ -326,7 +1353,7 @@ 2.2-100 | 2014-01-20 11:54:58 -0800 * Fix caching of recently validated SSL certifcates. (Justin Azoff) - + 2.2-98 | 2014-01-20 11:50:32 -0800 * For notice suppresion, instead of storing the entire notice in @@ -354,7 +1381,7 @@ the city database instead of just the former. (Jon Siwek) * Broxygen init fixes. Addresses BIT-1110. (Jon Siwek) - + - Don't check mtime of bro binary if BRO_DISABLE_BROXYGEN env var set. - Fix failure to locate bro binary if invoking from a relative @@ -367,7 +1394,7 @@ len field being set to zero. (Seth Hall) * Canonify output of a unit test. (Jon Siwek) - + * A set of documentation updates. (Daniel Thayer) - Fix typo in Bro 2.2 NEWS on string indexing. @@ -414,9 +1441,9 @@ (Jon Siwek) * Close signature files after done parsing. (Jon Siwek) - + * Fix unlikely null ptr deref in broxygen::Manager. (Jon Siwek) - + * FreeBSD build fix addendum: unintended variable shadowing. (Jon Siwek) @@ -435,7 +1462,7 @@ were cleaned up. Addresses BIT-1103. (Bernhard Amann) * Minor Broxygen improvements. Addresses BIT-1098. (Jon Siwek) - + 2.2-51 | 2013-12-05 07:53:37 -0800 * Improve a unit test involving 'when' conditionals. (Jon Siwek) @@ -460,7 +1487,7 @@ 2.2-44 | 2013-12-04 12:41:51 -0800 * Fix string slice notation. Addresses BIT-1097. (Jon Siwek) - + Slice ranges were not correctly determined for negative indices and also off by one in general (included one more element at the end of the substring than what actually matched the index range). @@ -515,7 +1542,7 @@ 2.2-11 | 2013-12-03 10:56:28 -0800 * Unit test for broccoli vector support. (Jon Siwek) - + * Changed ordering of Bro type tag enum, which was out of sync. (Jon Siwek) @@ -527,14 +1554,14 @@ 2.2-6 | 2013-11-15 07:05:15 -0800 * Make "install-example-configs" target use DESTDIR. (Jon Siwek) - + 2.2-5 | 2013-11-11 13:47:54 -0800 * Fix the irc_reply event for certain server message types. (Seth Hall) * Fixed Segmentation fault in SQLite Writer. (Jon Crussell) - + 2.2 | 2013-11-07 10:25:50 -0800 * Release 2.2. diff --git a/CMakeLists.txt b/CMakeLists.txt index 71f6a6a8fe..7a287ef5b4 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1,5 +1,9 @@ project(Bro C CXX) -cmake_minimum_required(VERSION 2.8.0 FATAL_ERROR) + +# When changing the minimum version here, also adapt +# aux/bro-aux/plugin-support/skeleton/CMakeLists.txt +cmake_minimum_required(VERSION 2.6.3 FATAL_ERROR) + include(cmake/CommonCMakeConfig.cmake) ######################################################################## @@ -11,22 +15,28 @@ if (NOT BRO_SCRIPT_INSTALL_PATH) set(BRO_SCRIPT_INSTALL_PATH ${BRO_ROOT_DIR}/share/bro) endif () +if (NOT BRO_MAN_INSTALL_PATH) + # set the default Bro man page installation path (user did not specify one) + set(BRO_MAN_INSTALL_PATH ${BRO_ROOT_DIR}/share/man) +endif () + # sanitize the Bro script install directory into an absolute path # (CMake is confused by ~ as a representation of home directory) get_filename_component(BRO_SCRIPT_INSTALL_PATH ${BRO_SCRIPT_INSTALL_PATH} ABSOLUTE) -set(BRO_MAGIC_INSTALL_PATH ${BRO_ROOT_DIR}/share/bro/magic) -set(BRO_MAGIC_SOURCE_PATH ${CMAKE_CURRENT_SOURCE_DIR}/magic/database) +set(BRO_PLUGIN_INSTALL_PATH ${BRO_ROOT_DIR}/lib/bro/plugins CACHE STRING "Installation path for plugins" FORCE) configure_file(bro-path-dev.in ${CMAKE_CURRENT_BINARY_DIR}/bro-path-dev) + file(WRITE ${CMAKE_CURRENT_BINARY_DIR}/bro-path-dev.sh "export BROPATH=`${CMAKE_CURRENT_BINARY_DIR}/bro-path-dev`\n" - "export BROMAGIC=\"${BRO_MAGIC_SOURCE_PATH}\"\n" + "export BRO_PLUGIN_PATH=\"${CMAKE_CURRENT_BINARY_DIR}/src:${BRO_PLUGIN_INSTALL_PATH}\"\n" "export PATH=\"${CMAKE_CURRENT_BINARY_DIR}/src\":$PATH\n") + file(WRITE ${CMAKE_CURRENT_BINARY_DIR}/bro-path-dev.csh "setenv BROPATH `${CMAKE_CURRENT_BINARY_DIR}/bro-path-dev`\n" - "setenv BROMAGIC \"${BRO_MAGIC_SOURCE_PATH}\"\n" + "setenv BRO_PLUGIN_PATH \"${CMAKE_CURRENT_BINARY_DIR}/src:${BRO_PLUGIN_INSTALL_PATH}\"\n" "setenv PATH \"${CMAKE_CURRENT_BINARY_DIR}/src\":$PATH\n") file(STRINGS "${CMAKE_CURRENT_SOURCE_DIR}/VERSION" VERSION LIMIT_COUNT 1) @@ -39,32 +49,6 @@ set(VERSION_MAJ_MIN "${VERSION_MAJOR}.${VERSION_MINOR}") ######################################################################## ## Dependency Configuration -include(ExternalProject) - -# LOG_* options to ExternalProject_Add appear in CMake 2.8.3. If -# available, using them hides external project configure/build output. -if("${CMAKE_VERSION}" VERSION_GREATER 2.8.2) - set(EXTERNAL_PROJECT_LOG_OPTIONS - LOG_DOWNLOAD 1 LOG_UPDATE 1 LOG_CONFIGURE 1 LOG_BUILD 1 LOG_INSTALL 1) -else() - set(EXTERNAL_PROJECT_LOG_OPTIONS) -endif() - -set(LIBMAGIC_PREFIX ${CMAKE_CURRENT_BINARY_DIR}/libmagic-prefix) -set(LIBMAGIC_INCLUDE_DIR ${LIBMAGIC_PREFIX}/include) -set(LIBMAGIC_LIB_DIR ${LIBMAGIC_PREFIX}/lib) -set(LIBMAGIC_LIBRARY ${LIBMAGIC_LIB_DIR}/libmagic.a) -ExternalProject_Add(libmagic - PREFIX ${LIBMAGIC_PREFIX} - URL ${CMAKE_CURRENT_SOURCE_DIR}/src/3rdparty/file-5.17.tar.gz - CONFIGURE_COMMAND ./configure --enable-static --disable-shared - --prefix=${LIBMAGIC_PREFIX} - --includedir=${LIBMAGIC_INCLUDE_DIR} - --libdir=${LIBMAGIC_LIB_DIR} - BUILD_IN_SOURCE 1 - ${EXTERNAL_PROJECT_LOG_OPTIONS} -) - include(FindRequiredPackage) # Check cache value first to avoid displaying "Found sed" messages everytime @@ -107,7 +91,6 @@ include_directories(BEFORE ${OpenSSL_INCLUDE_DIR} ${BIND_INCLUDE_DIR} ${BinPAC_INCLUDE_DIR} - ${LIBMAGIC_INCLUDE_DIR} ${ZLIB_INCLUDE_DIR} ${JEMALLOC_INCLUDE_DIR} ) @@ -149,33 +132,6 @@ if (GOOGLEPERFTOOLS_FOUND) endif () endif () -set(USE_DATASERIES false) -find_package(Lintel) -find_package(DataSeries) -find_package(LibXML2) - -if (NOT DISABLE_DATASERIES AND - LINTEL_FOUND AND DATASERIES_FOUND AND LIBXML2_FOUND) - set(USE_DATASERIES true) - include_directories(BEFORE ${Lintel_INCLUDE_DIR}) - include_directories(BEFORE ${DataSeries_INCLUDE_DIR}) - include_directories(BEFORE ${LibXML2_INCLUDE_DIR}) - list(APPEND OPTLIBS ${Lintel_LIBRARIES}) - list(APPEND OPTLIBS ${DataSeries_LIBRARIES}) - list(APPEND OPTLIBS ${LibXML2_LIBRARIES}) -endif() - -set(USE_ELASTICSEARCH false) -set(USE_CURL false) -find_package(LibCURL) - -if (NOT DISABLE_ELASTICSEARCH AND LIBCURL_FOUND) - set(USE_ELASTICSEARCH true) - set(USE_CURL true) - include_directories(BEFORE ${LibCURL_INCLUDE_DIR}) - list(APPEND OPTLIBS ${LibCURL_LIBRARIES}) -endif() - if (ENABLE_PERFTOOLS_DEBUG OR ENABLE_PERFTOOLS) # Just a no op to prevent CMake from complaining about manually-specified # ENABLE_PERFTOOLS_DEBUG or ENABLE_PERFTOOLS not being used if google @@ -187,7 +143,6 @@ set(brodeps ${PCAP_LIBRARY} ${OpenSSL_LIBRARIES} ${BIND_LIBRARY} - ${LIBMAGIC_LIBRARY} ${ZLIB_LIBRARY} ${JEMALLOC_LIBRARIES} ${OPTLIBS} @@ -198,6 +153,8 @@ set(brodeps include(TestBigEndian) test_big_endian(WORDS_BIGENDIAN) +include(CheckSymbolExists) +check_symbol_exists(htonll arpa/inet.h HAVE_BYTEORDER_64) include(OSSpecific) include(CheckTypes) @@ -207,6 +164,10 @@ include(MiscTests) include(PCAPTests) include(OpenSSLTests) include(CheckNameserCompat) +include(GetArchitecture) + +# Tell the plugin code that we're building as part of the main tree. +set(BRO_PLUGIN_INTERNAL_BUILD true CACHE INTERNAL "" FORCE) configure_file(${CMAKE_CURRENT_SOURCE_DIR}/config.h.in ${CMAKE_CURRENT_BINARY_DIR}/config.h) @@ -219,6 +180,7 @@ include_directories(${CMAKE_CURRENT_BINARY_DIR}) add_subdirectory(src) add_subdirectory(scripts) add_subdirectory(doc) +add_subdirectory(man) include(CheckOptionalBuildSources) @@ -226,10 +188,6 @@ CheckOptionalBuildSources(aux/broctl Broctl INSTALL_BROCTL) CheckOptionalBuildSources(aux/bro-aux Bro-Aux INSTALL_AUX_TOOLS) CheckOptionalBuildSources(aux/broccoli Broccoli INSTALL_BROCCOLI) -install(DIRECTORY ./magic/database/ - DESTINATION ${BRO_MAGIC_INSTALL_PATH} -) - ######################################################################## ## Packaging Setup @@ -275,10 +233,6 @@ message( "\n tcmalloc: ${USE_PERFTOOLS_TCMALLOC}" "\n debugging: ${USE_PERFTOOLS_DEBUG}" "\njemalloc: ${ENABLE_JEMALLOC}" - "\ncURL: ${USE_CURL}" - "\n" - "\nDataSeries: ${USE_DATASERIES}" - "\nElasticSearch: ${USE_ELASTICSEARCH}" "\n" "\n================================================================\n" ) diff --git a/Makefile b/Makefile index f4b2104c73..9feaecd656 100644 --- a/Makefile +++ b/Makefile @@ -48,6 +48,7 @@ bindist: distclean: rm -rf $(BUILD) + $(MAKE) -C testing $@ test: @( cd testing && make ) @@ -55,6 +56,8 @@ test: test-all: test test -d aux/broctl && ( cd aux/broctl && make test ) test -d aux/btest && ( cd aux/btest && make test ) + test -d aux/bro-aux && ( cd aux/bro-aux && make test ) + test -d aux/plugins && ( cd aux/plugins && make test-all ) configured: @test -d $(BUILD) || ( echo "Error: No build/ directory found. Did you run configure?" && exit 1 ) diff --git a/NEWS b/NEWS index 1c7cf66f1c..6de9bd8f3e 100644 --- a/NEWS +++ b/NEWS @@ -4,18 +4,42 @@ release. For an exhaustive list of changes, see the ``CHANGES`` file (note that submodules, such as BroControl and Broccoli, come with their own ``CHANGES``.) -Bro 2.3 -======= - -[In progress] +Bro 2.4 (in progress) +===================== Dependencies ------------ -- Bro no longer requires a pre-installed libmagic (because it now - ships its own). +New Functionality +----------------- -- Compiling from source now needs a CMake version >= 2.8.0. +- Bro now has support for external plugins that can extend its core + functionality, like protocol/file analysis, via shared libraries. + Plugins can be developed and distributed externally, and will be + pulled in dynamically at startup. Currently, a plugin can provide + custom protocol analyzers, file analyzers, log writers[TODO], input + readers[TODO], packet sources[TODO], and new built-in functions. A + plugin can furthermore hook into Bro's processing a number of places + to add custom logic. + + See https://www.bro.org/sphinx-git/devel/plugins.html for more + information on writing plugins. + +- Bro now has supoprt for the MySQL wire protocol. Activity gets + logged into mysql.log. + +Changed Functionality +--------------------- + +- bro-cut has been rewritten in C, and is hence much faster. + +Bro 2.3 +======= + +Dependencies +------------ + +- Libmagic is no longer a dependency. New Functionality ----------------- @@ -28,6 +52,51 @@ New Functionality - The DNS analyzer now actually generates the dns_SRV_reply() event. It had been documented before, yet was never raised. +- Bro now uses "file magic signatures" to identify file types. These + are defined via two new constructs in the signature rule parsing + grammar: "file-magic" gives a regular expression to match against, + and "file-mime" gives the MIME type string of content that matches + the magic and an optional strength value for the match. (See also + "Changed Functionality" below for changes due to switching from + using libmagic to such signatures.) + +- A new built-in function, "file_magic", can be used to get all file + magic matches and their corresponding strength against a given chunk + of data. + +- The SSL analyzer now supports heartbeats as well as a few + extensions, including server_name, alpn, and ec-curves. + +- The SSL analyzer comes with Heartbleed detector script in + protocols/ssl/heartbleed.bro. Note that loading this script changes + the default value of "SSL::disable_analyzer_after_detection" from true + to false to prevent encrypted heartbeats from being ignored. + +- StartTLS is now supported for SMTP and POP3. + +- The X509 analyzer can now perform OSCP validation. + +- Bro now has analyzers for SNMP and Radius, which produce corresponding + snmp.log and radius.log output (as well as various events of course). + +- BroControl has a new option "BroPort" which allows a user to specify + the starting port number for Bro. + +- BroControl has a new option "StatsLogExpireInterval" which allows a + user to specify when entries in the stats.log file expire. + +- BroControl has a new option "PFRINGClusterType" which allows a user + to specify a PF_RING cluster type. + +- BroControl now supports PF_RING+DNA. There is also a new option + "PFRINGFirstAppInstance" that allows a user to specify the starting + application instance number for processes running on a DNA cluster. + See the BroControl documentation for more details. + +- BroControl now warns a user to run "broctl install" if Bro has + been upgraded or if the broctl or node configuration has changed + since the most recent install. + Changed Functionality --------------------- @@ -44,14 +113,62 @@ Changed Functionality event x509_extension(c: connection, is_orig: bool, cert: X509, ext: X509_extension_info); +- In addition, there are several new, more specialized events for a + number of x509 extensions. + +- Generally, all x509 events and handling functions have changed their + signatures. + +- X509 certificate verification now returns the complete certificate + chain that was used for verification. + - Bro no longer special-cases SYN/FIN/RST-filtered traces by not - reporting missing data. The old behavior can be reverted by + reporting missing data. Instead, if Bro never sees any data segments + for analyzed TCP connections, the new + base/misc/find-filtered-trace.bro script will log a warning in + reporter.log and to stderr. The old behavior can be reverted by redef'ing "detect_filtered_trace". - TODO: Update if we add a detector for filtered traces. - - We have removed the packet sorter component. +- Bro no longer uses libmagic to identify file types but instead now + comes with its own signature library (which initially is still + derived from libmagic's database). This leads to a number of further + changes with regards to MIME types: + + * The second parameter of the "identify_data" built-in function + can no longer be used to get verbose file type descriptions, + though it can still be used to get the strongest matching file + magic signature. + + * The "file_transferred" event's "descr" parameter no longer + contains verbose file type descriptions. + + * The BROMAGIC environment variable no longer changes any behavior + in Bro as magic databases are no longer used/installed. + + * Removed "binary" and "octet-stream" mime type detections. They + don't provide any more information than an uninitialized + mime_type field. + + * The "fa_file" record now contains a "mime_types" field that + contains all magic signatures that matched the file content + (where the "mime_type" field is just a shortcut for the + strongest match). + +- dns_TXT_reply() now supports more than one string entry by receiving + a vector of strings. + +- BroControl now runs the "exec" and "df" broctl commands only once + per host, instead of once per Bro node. The output of these + commands has been changed slightly to include both the host and + node names. + +- Several performance improvements were made. Particular emphasis + was put on the File Analysis system, which generally will now emit + far fewer file handle request events due to protocol analyzers now + caching that information internally. + Bro 2.2 ======= diff --git a/VERSION b/VERSION index be38aad62b..df7c5848c1 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -2.2-250 +2.3-343 diff --git a/aux/binpac b/aux/binpac index fe27162849..77a86591dc 160000 --- a/aux/binpac +++ b/aux/binpac @@ -1 +1 @@ -Subproject commit fe271628492b7b837b3fbcf4626061c8b3568589 +Subproject commit 77a86591dcf89d7252d3676d3f1199d6c927d073 diff --git a/aux/bro-aux b/aux/bro-aux index d7ac87294f..43a9f360c9 160000 --- a/aux/bro-aux +++ b/aux/bro-aux @@ -1 +1 @@ -Subproject commit d7ac87294f415b5ddf3fc81bcae29815d2f835b1 +Subproject commit 43a9f360c9bf6b35fcb25d61ebff80c7feb1812b diff --git a/aux/broccoli b/aux/broccoli index 3138e5068e..acb8fbe8e7 160000 --- a/aux/broccoli +++ b/aux/broccoli @@ -1 +1 @@ -Subproject commit 3138e5068eeeb374c39c3d3b05b482b84d1f6e9c +Subproject commit acb8fbe8e7bc6ace5135fb73dca8e29432cdc1ca diff --git a/aux/broctl b/aux/broctl index f8273c9eb5..90f9ca0ffa 160000 --- a/aux/broctl +++ b/aux/broctl @@ -1 +1 @@ -Subproject commit f8273c9eb5db1168c21b298cd3af4f9b4b008717 +Subproject commit 90f9ca0ffa2306f0d1d2ac208cdbb7787199f890 diff --git a/aux/btest b/aux/btest index 4e2ec35917..d67d89aaee 160000 --- a/aux/btest +++ b/aux/btest @@ -1 +1 @@ -Subproject commit 4e2ec35917acb883c7d2ab19af487f3863c687ae +Subproject commit d67d89aaee32ad5edb9068db55d1310c2f36970a diff --git a/aux/plugins b/aux/plugins new file mode 160000 index 0000000000..ad600b5bdc --- /dev/null +++ b/aux/plugins @@ -0,0 +1 @@ +Subproject commit ad600b5bdcd56a2723e323c0f2c8e1708956ca4f diff --git a/cmake b/cmake index 58c64e663c..1316c07f70 160000 --- a/cmake +++ b/cmake @@ -1 +1 @@ -Subproject commit 58c64e663ca9f035f7741775acefce1f6c8d1ed3 +Subproject commit 1316c07f7059647b6c4a496ea36e4b83bb5d8f0f diff --git a/config.h.in b/config.h.in index 2d065f755e..755a9eee98 100644 --- a/config.h.in +++ b/config.h.in @@ -129,6 +129,9 @@ /* whether words are stored with the most significant byte first */ #cmakedefine WORDS_BIGENDIAN +/* whether htonll/ntohll is defined in */ +#cmakedefine HAVE_BYTEORDER_64 + /* ultrix can't hack const */ #cmakedefine NEED_ULTRIX_CONST_HACK #ifdef NEED_ULTRIX_CONST_HACK @@ -209,3 +212,14 @@ /* Common IPv6 extension structure */ #cmakedefine HAVE_IP6_EXT + +/* String with host architecture (e.g., "linux-x86_64") */ +#define HOST_ARCHITECTURE "@HOST_ARCHITECTURE@" + +/* String with extension of dynamic libraries (e.g., ".so") */ +#define DYNAMIC_PLUGIN_SUFFIX "@CMAKE_SHARED_MODULE_SUFFIX@" + +/* True if we're building outside of the main Bro source code tree. */ +#ifndef BRO_PLUGIN_INTERNAL_BUILD +#define BRO_PLUGIN_INTERNAL_BUILD @BRO_PLUGIN_INTERNAL_BUILD@ +#endif diff --git a/configure b/configure index 5af2f25c8f..2b1c568b26 100755 --- a/configure +++ b/configure @@ -24,6 +24,13 @@ Usage: $0 [OPTION]... [VAR=VALUE]... --prefix=PREFIX installation directory [/usr/local/bro] --scriptdir=PATH root installation directory for Bro scripts [PREFIX/share/bro] + --localstatedir=PATH when using BroControl, path to store log files + and run-time data (within log/ and spool/ subdirs) + [PREFIX] + --spooldir=PATH when using BroControl, path to store run-time data + [PREFIX/spool] + --logdir=PATH when using BroControl, path to store log file + [PREFIX/logs] --conf-files-dir=PATH config files installation directory [PREFIX/etc] Optional Features: @@ -39,8 +46,6 @@ Usage: $0 [OPTION]... [VAR=VALUE]... --disable-auxtools don't build or install auxiliary tools --disable-perftools don't try to build with Google Perftools --disable-python don't try to build python bindings for broccoli - --disable-dataseries don't use the optional DataSeries log writer - --disable-elasticsearch don't use the optional ElasticSearch log writer Required Packages in Non-Standard Locations: --with-openssl=PATH path to OpenSSL install root @@ -50,7 +55,6 @@ Usage: $0 [OPTION]... [VAR=VALUE]... --with-flex=PATH path to flex executable --with-bison=PATH path to bison executable --with-perl=PATH path to perl executable - --with-libmagic=PATH path to libmagic install root Optional Packages in Non-Standard Locations: --with-geoip=PATH path to the libGeoIP install root @@ -63,9 +67,6 @@ Usage: $0 [OPTION]... [VAR=VALUE]... --with-ruby-lib=PATH path to ruby library --with-ruby-inc=PATH path to ruby headers --with-swig=PATH path to SWIG executable - --with-dataseries=PATH path to DataSeries and Lintel libraries - --with-xml2=PATH path to libxml2 installation (for DataSeries) - --with-curl=PATH path to libcurl install root (for ElasticSearch) Packaging Options (for developers): --binary-package toggle special logic for binary packaging @@ -150,6 +151,15 @@ while [ $# -ne 0 ]; do append_cache_entry BRO_ETC_INSTALL_DIR PATH $optarg user_set_conffilesdir="true" ;; + --localstatedir=*) + append_cache_entry BRO_LOCAL_STATE_DIR PATH $optarg + ;; + --spooldir=*) + append_cache_entry BRO_SPOOL_DIR PATH $optarg + ;; + --logdir=*) + append_cache_entry BRO_LOG_DIR PATH $optarg + ;; --enable-debug) append_cache_entry ENABLE_DEBUG BOOL true ;; @@ -184,12 +194,6 @@ while [ $# -ne 0 ]; do --enable-ruby) append_cache_entry DISABLE_RUBY_BINDINGS BOOL false ;; - --disable-dataseries) - append_cache_entry DISABLE_DATASERIES BOOL true - ;; - --disable-elasticsearch) - append_cache_entry DISABLE_ELASTICSEARCH BOOL true - ;; --with-openssl=*) append_cache_entry OpenSSL_ROOT_DIR PATH $optarg ;; @@ -211,9 +215,6 @@ while [ $# -ne 0 ]; do --with-perl=*) append_cache_entry PERL_EXECUTABLE PATH $optarg ;; - --with-libmagic=*) - append_cache_entry LibMagic_ROOT_DIR PATH $optarg - ;; --with-geoip=*) append_cache_entry LibGeoIP_ROOT_DIR PATH $optarg ;; @@ -247,16 +248,6 @@ while [ $# -ne 0 ]; do --with-swig=*) append_cache_entry SWIG_EXECUTABLE PATH $optarg ;; - --with-dataseries=*) - append_cache_entry DataSeries_ROOT_DIR PATH $optarg - append_cache_entry Lintel_ROOT_DIR PATH $optarg - ;; - --with-xml2=*) - append_cache_entry LibXML2_ROOT_DIR PATH $optarg - ;; - --with-curl=*) - append_cache_entry LibCURL_ROOT_DIR PATH $optarg - ;; --binary-package) append_cache_entry BINARY_PACKAGING_MODE BOOL true ;; diff --git a/doc/CMakeLists.txt b/doc/CMakeLists.txt index 9498556edc..414cf56b0c 100644 --- a/doc/CMakeLists.txt +++ b/doc/CMakeLists.txt @@ -14,8 +14,6 @@ if (NOT ${retval} EQUAL 0) message(FATAL_ERROR "Problem setting BROPATH") endif () -set(BROMAGIC ${BRO_MAGIC_SOURCE_PATH}) - # Configure the Sphinx config file (expand variables CMake might know about). configure_file(${CMAKE_CURRENT_SOURCE_DIR}/conf.py.in ${CMAKE_CURRENT_BINARY_DIR}/conf.py @@ -34,7 +32,6 @@ add_custom_target(sphinxdoc ${CMAKE_CURRENT_SOURCE_DIR}/ ${SPHINX_INPUT_DIR} # Use Bro/Broxygen to dynamically generate reST for all Bro scripts. COMMAND BROPATH=${BROPATH} - BROMAGIC=${BROMAGIC} ${CMAKE_BINARY_DIR}/src/bro -X ${CMAKE_CURRENT_BINARY_DIR}/broxygen.conf broxygen >/dev/null diff --git a/doc/README b/doc/README index 5104f79801..5d1b685d40 100644 --- a/doc/README +++ b/doc/README @@ -10,7 +10,7 @@ common/general documentation, style sheets, JavaScript, etc. The Sphinx config file is produced from ``conf.py.in``, and can be edited to change various Sphinx options. -There is also a custom Sphinx domain implemented in ``source/ext/bro.py`` +There is also a custom Sphinx domain implemented in ``ext/bro.py`` which adds some reST directives and roles that aid in generating useful index entries and cross-references. Other extensions can be added in a similar fashion. @@ -19,7 +19,8 @@ The ``make doc`` target in the top-level Makefile can be used to locally render the reST files into HTML. That target depends on: * Python interpreter >= 2.5 -* `Sphinx `_ >= 1.0.1 +* `Sphinx `_ >= 1.0.1 +* Doxygen (required only for building the Broccoli API doc) After completion, HTML documentation is symlinked in ``build/html``. diff --git a/doc/_templates/layout.html b/doc/_templates/layout.html index 2f8ea02aff..3df56a12ff 100644 --- a/doc/_templates/layout.html +++ b/doc/_templates/layout.html @@ -10,7 +10,7 @@ {% endblock %} {% block header %} - {% endblock %} @@ -108,6 +108,6 @@ {% endblock %} {% block footer %} - {% endblock %} diff --git a/doc/cluster/index.rst b/doc/cluster/index.rst index 2e6cb2e782..544ca5e0f8 100644 --- a/doc/cluster/index.rst +++ b/doc/cluster/index.rst @@ -1,18 +1,19 @@ ======================== -Setting up a Bro Cluster +Bro Cluster Architecture ======================== -Intro ------- Bro is not multithreaded, so once the limitations of a single processor core are reached the only option currently is to spread the workload across many cores, or even many physical computers. The cluster deployment scenario for -Bro is the current solution to build these larger systems. The accompanying -tools and scripts provide the structure to easily manage many Bro processes -examining packets and doing correlation activities but acting as a singular, -cohesive entity. +Bro is the current solution to build these larger systems. The tools and +scripts that accompany Bro provide the structure to easily manage many Bro +processes examining packets and doing correlation activities but acting as +a singular, cohesive entity. This document describes the Bro cluster +architecture. For information on how to configure a Bro cluster, +see the documentation for +:doc:`BroControl <../components/broctl/README>`. Architecture --------------- @@ -41,11 +42,11 @@ messages and notices from the rest of the nodes in the cluster using the Bro communications protocol. The result is a single log instead of many discrete logs that you have to combine in some manner with post-processing. The manager also takes the opportunity to de-duplicate notices, and it has the -ability to do so since it’s acting as the choke point for notices and how notices -might be processed into actions (e.g., emailing, paging, or blocking). +ability to do so since it's acting as the choke point for notices and how +notices might be processed into actions (e.g., emailing, paging, or blocking). The manager process is started first by BroControl and it only opens its -designated port and waits for connections, it doesn’t initiate any +designated port and waits for connections, it doesn't initiate any connections to the rest of the cluster. Once the workers are started and connect to the manager, logs and notices will start arriving to the manager process from the workers. @@ -58,12 +59,11 @@ the workers by alleviating the need for all of the workers to connect directly to each other. Examples of synchronized state from the scripts that ship with Bro include -the full list of “known” hosts and services (which are hosts or services +the full list of "known" hosts and services (which are hosts or services identified as performing full TCP handshakes) or an analyzed protocol has been found on the connection. If worker A detects host 1.2.3.4 as an active host, it would be beneficial for worker B to know that as well. So worker A shares -that information as an insertion to a set - which travels to the cluster’s +that information as an insertion to a set which travels to the cluster's proxy and the proxy sends that same set insertion to worker B. The result is that worker A and worker B have shared knowledge about host and services that are active on the network being monitored. @@ -79,7 +79,7 @@ necessary for the number of workers they are serving. It is best to start with a single proxy and add more if communication performance problems are found. -Bro processes acting as proxies don’t tend to be extremely hard on CPU +Bro processes acting as proxies don't tend to be extremely hard on CPU or memory and users frequently run proxy processes on the same physical host as the manager. @@ -106,7 +106,7 @@ dedicated to being workers with each one containing dual 6-core processors. Once a flow-based load balancer is put into place this model is extremely easy to scale. It is recommended that you estimate the amount of -hardware you will need to fully analyze your traffic. If more is needed it’s +hardware you will need to fully analyze your traffic. If more is needed it's relatively easy to increase the size of the cluster in most cases. Frontend Options @@ -147,14 +147,13 @@ On host flow balancing PF_RING ^^^^^^^ -The PF_RING software for Linux has a “clustering” feature which will do +The PF_RING software for Linux has a "clustering" feature which will do flow-based load balancing across a number of processes that are sniffing the same interface. This allows you to easily take advantage of multiple -cores in a single physical host because Bro’s main event loop is single -threaded and can’t natively utilize all of the cores. More information about -Bro with PF_RING can be found here: (someone want to write a quick Bro/PF_RING -tutorial to link to here? document installing kernel module, libpcap -wrapper, building Bro with the --with-pcap configure option) +cores in a single physical host because Bro's main event loop is single +threaded and can't natively utilize all of the cores. If you want to use +PF_RING, see the documentation on `how to configure Bro with PF_RING +`_. Netmap ^^^^^^ @@ -167,7 +166,7 @@ Click! Software Router ^^^^^^^^^^^^^^^^^^^^^^ Click! can be used for flow based load balancing with a simple configuration. -(link to an example for the config). This solution is not recommended on -Linux due to Bro’s PF_RING support and only as a last resort on other +This solution is not recommended on +Linux due to Bro's PF_RING support and only as a last resort on other operating systems since it causes a lot of overhead due to context switching back and forth between kernel and userland several times per packet. diff --git a/doc/conf.py.in b/doc/conf.py.in index 91e16452f3..4faebed3b8 100644 --- a/doc/conf.py.in +++ b/doc/conf.py.in @@ -21,7 +21,7 @@ sys.path.insert(0, os.path.abspath('sphinx_input/ext')) # ----- Begin of BTest configuration. ----- btest = os.path.abspath("@CMAKE_SOURCE_DIR@/aux/btest") -brocut = os.path.abspath("@CMAKE_SOURCE_DIR@/aux/bro-aux/bro-cut") +brocut = os.path.abspath("@CMAKE_SOURCE_DIR@/build/aux/bro-aux/bro-cut") bro = os.path.abspath("@CMAKE_SOURCE_DIR@/build/src") os.environ["PATH"] += (":%s:%s/sphinx:%s:%s" % (btest, btest, bro, brocut)) @@ -38,7 +38,6 @@ extensions += ["broxygen"] bro_binary = os.path.abspath("@CMAKE_SOURCE_DIR@/build/src/bro") broxygen_cache="@BROXYGEN_CACHE_DIR@" os.environ["BROPATH"] = "@BROPATH@" -os.environ["BROMAGIC"] = "@BROMAGIC@" # ----- End of Broxygen configuration. ----- # -- General configuration ----------------------------------------------------- diff --git a/doc/configuration/index.rst b/doc/configuration/index.rst new file mode 100644 index 0000000000..800d746e72 --- /dev/null +++ b/doc/configuration/index.rst @@ -0,0 +1,263 @@ + +.. _configuration: + +===================== +Cluster Configuration +===================== + +.. contents:: + +A *Bro Cluster* is a set of systems jointly analyzing the traffic of +a network link in a coordinated fashion. You can operate such a setup from +a central manager system easily using BroControl because BroControl +hides much of the complexity of the multi-machine installation. + +This section gives examples of how to setup common cluster configurations +using BroControl. For a full reference on BroControl, see the +:doc:`BroControl <../components/broctl/README>` documentation. + + +Preparing to Setup a Cluster +============================ + +In this document we refer to the user account used to set up the cluster +as the "Bro user". When setting up a cluster the Bro user must be set up +on all hosts, and this user must have ssh access from the manager to all +machines in the cluster, and it must work without being prompted for a +password/passphrase (for example, using ssh public key authentication). +Also, on the worker nodes this user must have access to the target +network interface in promiscuous mode. + +Additional storage must be available on all hosts under the same path, +which we will call the cluster's prefix path. We refer to this directory +as ````. If you build Bro from source, then ```` is +the directory specified with the ``--prefix`` configure option, +or ``/usr/local/bro`` by default. The Bro user must be able to either +create this directory or, where it already exists, must have write +permission inside this directory on all hosts. + +When trying to decide how to configure the Bro nodes, keep in mind that +there can be multiple Bro instances running on the same host. For example, +it's possible to run a proxy and the manager on the same host. However, it is +recommended to run workers on a different machine than the manager because +workers can consume a lot of CPU resources. The maximum recommended +number of workers to run on a machine should be one or two less than +the number of CPU cores available on that machine. Using a load-balancing +method (such as PF_RING) along with CPU pinning can decrease the load on +the worker machines. + + +Basic Cluster Configuration +=========================== + +With all prerequisites in place, perform the following steps to setup +a Bro cluster (do this as the Bro user on the manager host only): + +- Edit the BroControl configuration file, ``/etc/broctl.cfg``, + and change the value of any BroControl options to be more suitable for + your environment. You will most likely want to change the value of + the ``MailTo`` and ``LogRotationInterval`` options. A complete + reference of all BroControl options can be found in the + :doc:`BroControl <../components/broctl/README>` documentation. + +- Edit the BroControl node configuration file, ``/etc/node.cfg`` + to define where manager, proxies, and workers are to run. For a cluster + configuration, you must comment-out (or remove) the standalone node + in that file, and either uncomment or add node entries for each node + in your cluster (manager, proxy, and workers). For example, if you wanted + to run four Bro nodes (two workers, one proxy, and a manager) on a cluster + consisting of three machines, your cluster configuration would look like + this:: + + [manager] + type=manager + host=10.0.0.10 + + [proxy-1] + type=proxy + host=10.0.0.10 + + [worker-1] + type=worker + host=10.0.0.11 + interface=eth0 + + [worker-2] + type=worker + host=10.0.0.12 + interface=eth0 + + For a complete reference of all options that are allowed in the ``node.cfg`` + file, see the :doc:`BroControl <../components/broctl/README>` documentation. + +- Edit the network configuration file ``/etc/networks.cfg``. This + file lists all of the networks which the cluster should consider as local + to the monitored environment. + +- Install workers and proxies using BroControl:: + + > broctl install + +- Some tasks need to be run on a regular basis. On the manager node, + insert a line like this into the crontab of the user running the + cluster:: + + 0-59/5 * * * * /bin/broctl cron + + (Note: if you are editing the system crontab instead of a user's own + crontab, then you need to also specify the user which the command + will be run as. The username must be placed after the time fields + and before the broctl command.) + + Note that on some systems (FreeBSD in particular), the default PATH + for cron jobs does not include the directories where bash and python + are installed (the symptoms of this problem would be that "broctl cron" + works when run directly by the user, but does not work from a cron job). + To solve this problem, you would either need to create symlinks + to bash and python in a directory that is in the default PATH for + cron jobs, or specify a new PATH in the crontab. + + +PF_RING Cluster Configuration +============================= + +`PF_RING `_ allows speeding up the +packet capture process by installing a new type of socket in Linux systems. +It supports 10Gbit hardware packet filtering using standard network adapters, +and user-space DNA (Direct NIC Access) for fast packet capture/transmission. + +Installing PF_RING +^^^^^^^^^^^^^^^^^^ + +1. Download and install PF_RING for your system following the instructions + `here `_. The following + commands will install the PF_RING libraries and kernel module (replace + the version number 5.6.2 in this example with the version that you + downloaded):: + + cd /usr/src + tar xvzf PF_RING-5.6.2.tar.gz + cd PF_RING-5.6.2/userland/lib + ./configure --prefix=/opt/pfring + make install + + cd ../libpcap + ./configure --prefix=/opt/pfring + make install + + cd ../tcpdump-4.1.1 + ./configure --prefix=/opt/pfring + make install + + cd ../../kernel + make install + + modprobe pf_ring enable_tx_capture=0 min_num_slots=32768 + + Refer to the documentation for your Linux distribution on how to load the + pf_ring module at boot time. You will need to install the PF_RING + library files and kernel module on all of the workers in your cluster. + +2. Download the Bro source code. + +3. Configure and install Bro using the following commands:: + + ./configure --with-pcap=/opt/pfring + make + make install + +4. Make sure Bro is correctly linked to the PF_RING libpcap libraries:: + + ldd /usr/local/bro/bin/bro | grep pcap + libpcap.so.1 => /opt/pfring/lib/libpcap.so.1 (0x00007fa6d7d24000) + +5. Configure BroControl to use PF_RING (explained below). + +6. Run "broctl install" on the manager. This command will install Bro and + all required scripts to the other machines in your cluster. + +Using PF_RING +^^^^^^^^^^^^^ + +In order to use PF_RING, you need to specify the correct configuration +options for your worker nodes in BroControl's node configuration file. +Edit the ``node.cfg`` file and specify ``lb_method=pf_ring`` for each of +your worker nodes. Next, use the ``lb_procs`` node option to specify how +many Bro processes you'd like that worker node to run, and optionally pin +those processes to certain CPU cores with the ``pin_cpus`` option (CPU +numbering starts at zero). The correct ``pin_cpus`` setting to use is +dependent on your CPU architecture (Intel and AMD systems enumerate +processors in different ways). Using the wrong ``pin_cpus`` setting +can cause poor performance. Here is what a worker node entry should +look like when using PF_RING and CPU pinning:: + + [worker-1] + type=worker + host=10.0.0.50 + interface=eth0 + lb_method=pf_ring + lb_procs=10 + pin_cpus=2,3,4,5,6,7,8,9,10,11 + + +Using PF_RING+DNA with symmetric RSS +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +You must have a PF_RING+DNA license in order to do this. You can sniff +each packet only once. + +1. Load the DNA NIC driver (i.e. ixgbe) on each worker host. + +2. Run "ethtool -L dna0 combined 10" (this will establish 10 RSS queues + on your NIC) on each worker host. You must make sure that you set the + number of RSS queues to the same as the number you specify for the + lb_procs option in the node.cfg file. + +3. On the manager, configure your worker(s) in node.cfg:: + + [worker-1] + type=worker + host=10.0.0.50 + interface=dna0 + lb_method=pf_ring + lb_procs=10 + + +Using PF_RING+DNA with pfdnacluster_master +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +You must have a PF_RING+DNA license and a libzero license in order to do +this. You can load balance between multiple applications and sniff the +same packets multiple times with different tools. + +1. Load the DNA NIC driver (i.e. ixgbe) on each worker host. + +2. Run "ethtool -L dna0 1" (this will establish 1 RSS queues on your NIC) + on each worker host. + +3. Run the pfdnacluster_master command on each worker host. For example:: + + pfdnacluster_master -c 21 -i dna0 -n 10 + + Make sure that your cluster ID (21 in this example) matches the interface + name you specify in the node.cfg file. Also make sure that the number + of processes you're balancing across (10 in this example) matches + the lb_procs option in the node.cfg file. + +4. If you are load balancing to other processes, you can use the + pfringfirstappinstance variable in broctl.cfg to set the first + application instance that Bro should use. For example, if you are running + pfdnacluster_master with "-n 10,4" you would set + pfringfirstappinstance=4. Unfortunately that's still a global setting + in broctl.cfg at the moment but we may change that to something you can + set in node.cfg eventually. + +5. On the manager, configure your worker(s) in node.cfg:: + + [worker-1] + type=worker + host=10.0.0.50 + interface=dnacluster:21 + lb_method=pf_ring + lb_procs=10 + diff --git a/doc/devel/plugins.rst b/doc/devel/plugins.rst new file mode 100644 index 0000000000..c703345891 --- /dev/null +++ b/doc/devel/plugins.rst @@ -0,0 +1,447 @@ + +=================== +Writing Bro Plugins +=================== + +Bro is internally moving to a plugin structure that enables extending +the system dynamically, without modifying the core code base. That way +custom code remains self-contained and can be maintained, compiled, +and installed independently. Currently, plugins can add the following +functionality to Bro: + + - Bro scripts. + + - Builtin functions/events/types for the scripting language. + + - Protocol analyzers. + + - File analyzers. + + - Packet sources and packet dumpers. + + - Logging framework backends. + + - Input framework readers. + +A plugin's functionality is available to the user just as if Bro had +the corresponding code built-in. Indeed, internally many of Bro's +pieces are structured as plugins as well, they are just statically +compiled into the binary rather than loaded dynamically at runtime. + +Quick Start +=========== + +Writing a basic plugin is quite straight-forward as long as one +follows a few conventions. In the following we walk a simple example +plugin that adds a new built-in function (bif) to Bro: we'll add +``rot13(s: string) : string``, a function that rotates every character +in a string by 13 places. + +Generally, a plugin comes in the form of a directory following a +certain structure. To get started, Bro's distribution provides a +helper script ``aux/bro-aux/plugin-support/init-plugin`` that creates +a skeleton plugin that can then be customized. Let's use that:: + + # mkdir rot13-plugin + # cd rot13-plugin + # init-plugin Demo Rot13 + +As you can see the script takes two arguments. The first is a +namespace the plugin will live in, and the second a descriptive name +for the plugin itself. Bro uses the combination of the two to identify +a plugin. The namespace serves to avoid naming conflicts between +plugins written by independent developers; pick, e.g., the name of +your organisation. The namespace ``Bro`` is reserved for functionality +distributed by the Bro Project. In our example, the plugin will be +called ``Demo::Rot13``. + +The ``init-plugin`` script puts a number of files in place. The full +layout is described later. For now, all we need is +``src/rot13.bif``. It's initially empty, but we'll add our new bif +there as follows:: + + # cat src/rot13.bif + module CaesarCipher; + + function rot13%(s: string%) : string + %{ + char* rot13 = copy_string(s->CheckString()); + + for ( char* p = rot13; *p; p++ ) + { + char b = islower(*p) ? 'a' : 'A'; + *p = (*p - b + 13) % 26 + b; + } + + BroString* bs = new BroString(1, reinterpret_cast(rot13), + strlen(rot13)); + return new StringVal(bs); + %} + +The syntax of this file is just like any other ``*.bif`` file; we +won't go into it here. + +Now we can already compile our plugin, we just need to tell the +configure script put in place by ``init-plugin`` where the Bro source +tree is located (Bro needs to have been built there first):: + + # ./configure --bro-dist=/path/to/bro/dist && make + [... cmake output ...] + +Now our ``rot13-plugin`` directory has everything that it needs +for Bro to recognize it as a dynamic plugin. Once we point Bro to it, +it will pull it in automatically, as we can check with the ``-N`` +option:: + + # export BRO_PLUGIN_PATH=/path/to/rot13-plugin + # bro -N + [...] + Plugin: Demo::Rot13 - (dynamic, version 1) + [...] + +That looks quite good, except for the dummy description that we should +replace with something nicer so that users will know what our plugin +is about. We do this by editing the ``config.description`` line in +``src/Plugin.cc``, like this:: + + [...] + plugin::Configuration Configure() + { + plugin::Configuration config; + config.name = "Demo::Rot13"; + config.description = "Caesar cipher rotating a string's characters by 13 places."; + config.version.major = 1; + config.version.minor = 0; + return config; + } + [...] + + # make + [...] + # bro -N | grep Rot13 + Plugin: Demo::Rot13 - Caesar cipher rotating a string's characters by 13 places. (dynamic, version 1) + +Better. Bro can also show us what exactly the plugin provides with the +more verbose option ``-NN``:: + + # bro -NN + [...] + Plugin: Demo::Rot13 - Caesar cipher rotating a string's characters by 13 places. (dynamic, version 1) + [Function] CaesarCipher::rot13 + [...] + +There's our function. Now let's use it:: + + # bro -e 'print CaesarCipher::rot13("Hello")' + Uryyb + +It works. We next install the plugin along with Bro itself, so that it +will find it directly without needing the ``BRO_PLUGIN_PATH`` +environment variable. If we first unset the variable, the function +will no longer be available:: + + # unset BRO_PLUGIN_PATH + # bro -e 'print CaesarCipher::rot13("Hello")' + error in , line 1: unknown identifier CaesarCipher::rot13, at or near "CaesarCipher::rot13" + +Once we install it, it works again:: + + # make install + # bro -e 'print CaesarCipher::rot13("Hello")' + Uryyb + +The installed version went into +``/lib/bro/plugins/Demo_Rot13``. + +We can distribute the plugin in either source or binary form by using +the Makefile's ``sdist`` and ``bdist`` target, respectively. Both +create corrsponding tarballs:: + + # make sdist + [...] + Source distribution in build/sdist/Demo_Rot13.tar.gz + + # make bdist + [...] + Binary distribution in build/Demo_Rot13-darwin-x86_64.tar.gz + +The source archive will contain everything in the plugin directory +except any generated files. The binary archive will contain anything +needed to install and run the plugin, i.e., just what ``make install`` +puts into place as well. As the binary distribution is +platform-dependent, its name includes the OS and architecture the +plugin was built on. + +Plugin Directory Layout +======================= + +A plugin's directory needs to follow a set of conventions so that Bro +(1) recognizes it as a plugin, and (2) knows what to load. While +``init-plugin`` takes care of most of this, the following is the full +story. We'll use ```` to represent a plugin's top-level +directory. + +``/__bro_plugin__`` + A file that marks a directory as containing a Bro plugin. The file + must exist, and its content must consist of a single line with the + qualified name of the plugin (e.g., "Demo::Rot13"). + +``/lib/--.so`` + The shared library containing the plugin's compiled code. Bro will + load this in dynamically at run-time if OS and architecture match + the current platform. + +``scripts/`` + A directory with the plugin's custom Bro scripts. When the plugin + gets activated, this directory will be automatically added to + ``BROPATH``, so that any scripts/modules inside can be + "@load"ed. + +``scripts``/__load__.bro + A Bro script that will be loaded immediately when the plugin gets + activated. See below for more information on activating plugins. + +``lib/bif/`` + Directory with auto-generated Bro scripts that declare the plugin's + bif elements. The files here are produced by ``bifcl``. + +By convention, a plugin should put its custom scripts into sub folders +of ``scripts/``, i.e., ``scripts//