From 2b6ad76bd53ff69eec5ebcc47f7f5b42558163ae Mon Sep 17 00:00:00 2001 From: Robin Sommer Date: Wed, 9 Mar 2011 15:26:01 -0800 Subject: [PATCH] Creating a branch release/1.5 with the current 1.5.3 release code. This is so that people working from the current stable version can still start using git. --- CHANGES | 257 +++++--- Checklist-for-Release | 4 + README | 2 +- VERSION | 2 +- acinclude.m4 | 56 +- aux/broccoli/ChangeLog | 8 +- aux/broccoli/configure.in | 2 +- aux/broccoli/docs/broccoli-manual.sgml | 7 +- aux/broccoli/docs/html/a3621.html | 210 ------- aux/broccoli/docs/html/a3638.html | 210 +++++++ aux/broccoli/docs/html/broccoli-broccoli.html | 566 +++++++++--------- aux/broccoli/docs/html/c54.html | 2 +- aux/broccoli/docs/html/c84.html | 32 +- aux/broccoli/docs/html/index.html | 18 +- aux/broccoli/src/bro_lexer.l | 2 +- aux/broccoli/src/bro_openssl.c | 38 +- aux/broctl/BroControl/control.py | 59 +- aux/broctl/BroControl/cron.py | 29 +- aux/broctl/BroControl/execute.py | 11 +- aux/broctl/BroControl/install.py | 19 +- aux/broctl/BroControl/options.py | 6 +- aux/broctl/BroControl/util.py | 2 +- aux/broctl/README | 27 +- aux/broctl/README.html | 35 +- aux/broctl/README.options | 2 + aux/broctl/aux/trace-summary/trace-summary | 13 +- aux/broctl/bin/archive-log.in | 50 +- aux/broctl/bin/broctl.in | 5 +- aux/broctl/bin/check-config.in | 1 + aux/broctl/bin/crash-diag.in | 16 +- aux/broctl/bin/create-link-for-log.in | 65 ++ aux/broctl/bin/delete-log | 2 +- aux/broctl/bin/fmt-time | 19 + aux/broctl/bin/get-prof-log.in | 4 +- aux/broctl/bin/helpers/df.in | 2 +- aux/broctl/bin/helpers/to-bytes.awk | 7 + aux/broctl/bin/helpers/top.in | 4 +- aux/broctl/bin/make-archive-name | 41 ++ aux/broctl/bin/post-terminate.in | 23 +- aux/broctl/bin/postprocessors/mail-log.in | 5 +- .../postprocessors/summarize-connections.in | 4 +- aux/broctl/bin/remove-link-for-log.in | 8 + aux/broctl/bin/stat-ctime | 25 + aux/broctl/bin/stats-to-csv | 137 +++++ aux/broctl/bin/update-stats.in | 15 + aux/broctl/policy/broctl-check.bro | 6 +- aux/broctl/policy/broctl-live.bro | 3 +- aux/broctl/policy/broctl.bro | 1 + aux/broctl/policy/cluster-manager.bro | 4 +- aux/broctl/policy/cluster-manager.drop.bro | 23 +- aux/broctl/policy/cluster-manager.icmp.bro | 6 - .../policy/cluster-manager.rotate-logs.bro | 8 +- aux/broctl/policy/cluster-proxy.bro | 1 - aux/broctl/policy/cluster-proxy.remote.bro | 3 + aux/broctl/policy/cluster-worker.bro | 1 - aux/broctl/policy/cluster-worker.drop.bro | 10 +- aux/broctl/policy/cluster.scan.bro | 10 +- aux/broctl/policy/mail-alarms.bro | 77 ++- aux/broctl/policy/standalone.bro | 1 - aux/broctl/policy/standalone.rotate-logs.bro | 10 +- aux/cf/cf.1 | 8 +- aux/cf/cf.c | 6 + configure.in | 48 +- policy/http-reply.bro | 10 +- policy/http-request.bro | 3 + policy/http.bro | 3 +- policy/notice.bro | 2 + policy/rotate-logs.bro | 3 +- src/File.cc | 20 +- src/File.h | 3 + src/RemoteSerializer.cc | 71 ++- src/RemoteSerializer.h | 2 + src/X509.cc | 6 +- src/ssl-analyzer.pac | 6 +- 74 files changed, 1551 insertions(+), 856 deletions(-) create mode 100644 aux/broccoli/docs/html/a3638.html create mode 100644 aux/broctl/bin/create-link-for-log.in create mode 100644 aux/broctl/bin/fmt-time create mode 100644 aux/broctl/bin/make-archive-name create mode 100644 aux/broctl/bin/remove-link-for-log.in create mode 100644 aux/broctl/bin/stat-ctime create mode 100644 aux/broctl/bin/stats-to-csv create mode 100644 aux/broctl/bin/update-stats.in diff --git a/CHANGES b/CHANGES index ef3f8c35d1..b535b1f211 100644 --- a/CHANGES +++ b/CHANGES @@ -2,30 +2,65 @@ -+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ -1.5.2.7 Sun Sep 12 19:39:49 PDT 2010 +1.5.3 Thu Mar 3 08:55:11 PST 2011 -- Addressed a number of lint nits (Vern Paxson). +- Removing aux/broctl/policy/cluster-addrs.hot.bro from the + distribution. The script is no longer needed and could in fact break + an installation because it redefines an old variable that has went + away. (Robin Sommer) +- Smarter way to increase the communication module's pipe's socket + buffer size, resulting in a value closer to the allowed maximum. + (Craig Leres) -1.5.2.6 Sun Sep 12 17:00:13 PDT 2010 +- BroControl now also maintains links from the log archive to the + current set of logs when running in standalone mode. (Robin Sommer) + +- Bug fix for a file descriptor leak in the remote communication + module. (Scott Campbell) + +- Bug fix for BroControl to now activate trace-summary's sampling in + cluster mode, but not anymore in standalone mode. (Robin Sommer) + +- Broccoli updates: + + * Accept empty strings ("") as values in the configuration file. + (Craig Leres) + + * Support for specifying a separate host key for SSL-enabled + operation, with documentation update. (Craig Leres) + +1.5.2 Wed Jan 12 17:34:55 PST 2011 + +- Portability fixes for --enable-int64 (Vern Paxson). + +- Bug fix for Active Mapping support (Kevin Lo). + +- Broccoli compiler warning fixes (Kevin Lo). + +- Bug fixes for --enable-int64 and for avoiding bogus statistics / + bad memory references when generating profiling information upon + exit (Vern Paxson). + +- Bug fixes for terminating connections (Tyler Schoenke and Vern Paxson). + +- Removed now-quite-stale SSHv1 overflow detection, as it's more prone + to false positives than useful detection (Vern Paxson). - The SWIG file now explicitly lists those pieces from broccoli.h which it wants to wrap, rather than just including all of broccoli.h (Robin Sommer). - This fixes the problem that the SWIG bindings depend on what configure - finds out about the availability of libpcap even though the corresponding - functions don't need to be wrapped anyway. -- http-header.bro now includes a global include_header: set[string] - (Robin Sommer). If it contains any strings, then only those headers - will be processed. If left empty, then you continue to get the current - behavior of processing all headers. +- http-header.bro now includes a global "include_header: set[string]" If it + contains any strings, then only those headers will be processed. If left + empty, then you continue to get the current behavior of processing all + headers. (Robin Sommer). - Several changes to drop.bro (Robin Sommer): - * If true, the new flag Drop::dont_drop_locals indicates that + * If True, the new flag Drop::dont_drop_locals indicates that local hosts should never be dropped. On by default. - * If true, the new flag Drop::debugging activates extensive debugging + * If True, the new flag Drop::debugging activates extensive debugging output for the catch-and-release logic. Off by default. * The timeout for tracking dropping information is now 1 day @@ -39,14 +74,14 @@ Sommer). - The HTTP analyzer no longer attempts to track Server/User-Agent - versions, as these are hugely voluminous (Seth Hall). Ideally this - would still be available as an option for someone who truly wants - the full set. + versions, as these are hugely voluminous (Seth Hall). - HTTP and SMTP no longer have extra-short inactivity timeouts, as these were too often leading to premature expiration of a connection (Robin Sommer). +- Tracking of HTTP refer[r]er's by setting log_referrer. (Vern Paxson). + - The "rst" tool (aux/rst/) now takes an optional "-I " argument that instructs it to inject as payload rather than sending a RST packet (Vern Paxson). must be NUL-terminated, and the NUL is not @@ -55,39 +90,113 @@ - Bug fix for crashes in the DNS analyzer when processing replies for which no request was seen (Robin Sommer). +- Addressed a number of lint nits (Vern Paxson). -1.5.2.5 Mon Jul 19 16:20:58 PDT 2010 +- Rotation post-processors are now passed an additional argument + indicating whether Bro is terminating (Robin Sommer). -- Removed now-quite-stale SSHv1 overflow detection, as it's more prone - to false positives than useful detection (Vern Paxson). +- Bro now consistently generates a file_opened event for all fopen() calls. + (Robin Sommer). +- The "cf" utility now ignores a leading "t=" prefix, for compatibility + with Bro's "tagged" logging format (Robin Sommer). -1.5.2.4 Fri Jun 4 16:02:11 PDT 2010 +- You can now redefine the email_notice_to function (Robin Sommer). -- Bug fixes for terminating connections (Tyler Schoenke and Vern Paxson). +- Fix for packet processing resumption when a remote Bro dies during + state synchronization (Robin Sommer). +- OpenSSL/X509 portability fix, at long last (Gregor Maier & Christian + Kreibich). -1.5.2.3 Wed Mar 24 18:23:57 PDT 2010 +- Fix for compatibility with newer versions of autoconf (Gregor Maier). -- Bug fixes for --enable-int64 and for avoiding bogus statistics / - bad memory references when generating profiling information upon - exit (Vern Paxson). +- A larger BroControl update (Robin Sommer, if not marked otherwise): + o Increasing default timeouts for scan detector significantly. -1.5.2.2 Tue Jan 12 12:33:42 PST 2010 + o Increasing the manager's max_remote_events_processed to + something large, as it would slow down the process too much + otherwise and there's no other work to be interleaved with it + anyway. -- Broccoli compiler warning fixes (Kevin Lo). + o Adding debug output to cluster's part of catch-and-release + (extends the debugging already present in policy/debug.bro) + o Fixing typo in util.py. Closes #223. -1.5.2.1 Sun Jan 10 16:59:01 PST 2010 + o Added note to README pointing to HTML version. -- Bug fix for Active Mapping support (Kevin Lo). + o Disabling print_hook for proxies' remote.log. + o broctl's capstats now reports a total as well, and stats.log + tracks these totals. Closes #160. -1.5.2 Sat Dec 26 18:38:37 PST 2009 + o Avoiding spurious "waiting for lock" messages in cron mode. + Closes #206. -- Portability fixes for --enable-int64 (Vern Paxson). + o Bug fixes for installation on NFS. + o Bug fix for top command on FreeBSD 8. + + o crash-diag now checks whether gdb is available. + + o trace-summary reports the sample factor in use in its output, + and now also applies it to the top-local-networks output (not + doing the latter was a bug). + + o Removed the default twice-a-day rotation for conn.log. The + default rotation for conn.log now is now once every 24h, just + like for all other logs with the exception of mail.log (which is + still rotated twice a day, and thus the alarms are still mailed + out twice a day). + + o Fixed the problem of logs sometimes being filed into the wrong + directory (see the (now gone) FAQ entry in the README). + + o One can now customize the archive naming scheme. See the + corresponding FAQ entry in the README. + + o Cleaned up, and extended, collection of cluster statistics. + + ${logdir}/stats now looks like this: + + drwxr-xr-x 4 bro wheel 59392 Apr 5 17:55 . + drwxr-xr-x 96 bro wheel 2560 Apr 6 12:00 .. + -rw-r--r-- 1 bro wheel 576 Apr 6 16:40 meta.dat + drwxr-xr-x 2 bro wheel 2048 Apr 6 16:40 profiling + -rw-r--r-- 1 bro wheel 771834825 Apr 6 16:40 stats.log + drwxr-xr-x 2 bro wheel 2048 Apr 6 16:25 www + + stats.log accumulates cluster statistics collected every time + "cron" is called. + + - profiling/ keeps the nodes' prof.logs. + + - www/ keeps a subset of stats.log in CSV format for easy plotting. + + - meta.dat contains meta information about the current cluster + state (in particular which nodes we have, and when the last + stats update was done). + + Note that there is no Web setup yet to actually visualize the data in + www/. + + o BroControl now automatically maintains links inside today's log + archive directory pointing to the current live version of the + corresponding log file (if Bro is running). For example: + + smtp.log.11:52:18-current -> /usr/local/cluster/spool/manager/smtp.log + + o Alarms mailed out by BroControl now (1) have the notice msg in the + subject; and (2) come with the full mail.log entry in the body. + + o Fixing broctl's top output. (Seth Hall). + + o Fixing broctl's df output in certain situations. + + o BroControl fix for dealing with large vsize values reported by + "top" (Craig Leres). 1.5.1 Fri Dec 18 15:17:12 PST 2009 @@ -378,7 +487,7 @@ (1) Remote communication now no longer includes location information for serialized objects; that removes quite a bit of redundacy from the network traffic. - + (2) The new option 'remote_check_sync_consistency" disables the cross-check on the receiving side of &synchronized state of whether the current value of a variable has the value expected by the sender. Transmitting the @@ -395,7 +504,7 @@ we maintain *two* caches independently for these types of objects; one with a low turn-over one and another with a high one. This should reduce CPU load on both sender and receiver sides. - + The new scheme is only used if both communicating Bros support it; with older Bros, as well as with Broccoli, we continue using the old scheme. @@ -589,12 +698,12 @@ bro -Y 0.0.0.0:5555 netflow bro -i eth0 -Y 10.0.0.1:1234=src1 brolite netflow - -y|--flowfile [=] + -y|--flowfile [=] Used to read from a file. You can optionally include an identifier for the source. - Examples: + Examples: bro -y myflowfile netflow bro -y myflowfile=src1 otherflowfile=src2 netflow @@ -763,7 +872,7 @@ So, to drop all sources triggering a specific notice, one can now, e.g., write: - + redef notice_action_filters += { [Hot::SSH_Overflow] = drop_source }; Related to this change, notice_info has a new field $dropped, set to @@ -794,8 +903,8 @@ before starting the main packet loop and another one when finished. These snapshots can then be analyzed with pprof. - For more information about the perftools see - + For more information about the perftools see + http://code.google.com/p/google-perftools - Notice tags are now generated in a pseudo-unique fashion that, with high @@ -869,7 +978,7 @@ detector tables. - When Bro serializes functions, it now does so by default using only - their name, rather than their full value (Robin Sommer). This prevents + their name, rather than their full value (Robin Sommer). This prevents propagation of expiration functions associated with tables and sets. Note, currently there is no mechanism provided to switch from the default behavior, but the internal hooks are in place to do so. @@ -1134,7 +1243,7 @@ - An arbitrary tag can now be past to post-processors for log rotation (Robin Sommer). -- Default inactivity timeouts for interactive services shortened to +- Default inactivity timeouts for interactive services shortened to 1 hour (Robin Sommer). - The scanning variables distinct_{peers,ports,low_ports} are now @@ -1447,7 +1556,7 @@ This fixes a long-standing problem of sometimes $addl fields not showing up in connection summaries. -- The new expressions record(...), table(...), set(...) and vector(...) +- The new expressions record(...), table(...), set(...) and vector(...) are constructors for the corresponding aggregate types (Vern Paxson). For example, @@ -1601,7 +1710,7 @@ - A new notice_action_filter, tally_notice_type_and_ignore, works the same as tally_notice_type but returns IGNORE (Robin Sommer) -- Setting summary_interval == 0 disables the creation of irc-bots.summary.log +- Setting summary_interval == 0 disables the creation of irc-bots.summary.log (Robin Sommer). - If you @load foo and a directory "foo" is in your path, Bro no longer @@ -1718,9 +1827,9 @@ - Fixed using "time" values as table indices. -- Added ssh to default brolite DPD configuration. +- Added ssh to default brolite DPD configuration. -- Fixed catching up to real-time in case of lull. +- Fixed catching up to real-time in case of lull. - Fixed Broccoli "BRO_DATA_FORMAT_VERSION" to match version in Bro. @@ -1730,11 +1839,11 @@ - Added Linux tuning to brolite install script. -- Modified Makefile to include broccoli/contrib. +- Modified Makefile to include broccoli/contrib. -- Adding missing initialization to remote serializer. +- Adding missing initialization to remote serializer. -- Minor documentation updates for reference manual and Broccoli. +- Minor documentation updates for reference manual and Broccoli. 1.2 Tue Oct 17 12:09:49 PDT 2006 @@ -1953,7 +2062,7 @@ - notice_action_filters now reside in the new script notice-action-filter.bro (automatically loaded by notice.bro). - + - The notice actions NOTICE_ALARM_PER_CONN, NOTICE_ALARM_PER_ORIG, and NOTICE_ALARM_ONCE have been removed, as they were never actually implemented. @@ -1973,7 +2082,7 @@ - TRW analysis now skips UDP traffic because it currently treats all UDP connections as failures (Robin Sommer). - + - trw.bro has been split into trw-impl.bro (the algorithm) and trw.bro (which simply activates the analysis), to facilitate writing scripts that have hooks into TRW analysis but don't presume it's @@ -2106,7 +2215,7 @@ (Robin Sommer). This appears to still need some work, as now it generates redundant events. -- Fix for initial exchange of &sync state which could lead to +- Fix for initial exchange of &sync state which could lead to referencing unknown IDs (Robin Sommer). - Fix to scan detection for differing semantics of connection compressor @@ -2415,7 +2524,7 @@ - the new variable dump_backdoor_packets (default F) if set causes the packet that triggered the backdoor detection to be written to - backdoor-packets/: