diff --git a/CHANGES b/CHANGES index 5267fa9f37..f8e4444f1d 100644 --- a/CHANGES +++ b/CHANGES @@ -1,4 +1,152 @@ +2.1 | 2012-08-28 16:46:42 -0700 + + * Make bif.identify_magic robust against FreeBSD's libmagic config. + (Robin Sommer) + + * Remove automatic use of gperftools on non-Linux systems. + --enable-perftools must now explicity be supplied to ./configure + on non-Linux systems to link against the tcmalloc library. + + * Fix uninitialized value for 'is_partial' in TCP analyzer. (Jon + Siwek) + + * Parse 64-bit consts in Bro scripts correctly. (Bernhard Amann) + + * Output 64-bit counts correctly on 32-bit machines (Bernhard Amann) + + * Input framework fixes, including: (Bernhard Amann) + + - One of the change events got the wrong parameters. + + - Escape commas in sets and vectors that were unescaped before + tokenization. + + - Handling of zero-length-strings as last element in a set was + broken (sets ending with a ,). + + - Hashing of lines just containing zero-length-strings was broken. + + - Make set_separators different from , work for input framework. + + - Input framework was not handling counts and ints out of + 32-bit-range correctly. + + - Errors in single lines do not kill processing, but simply ignore + the line, log it, and continue. + + * Update documentation for builtin types. (Daniel Thayer) + + - Add missing description of interval "msec" unit. + + - Improved description of pattern by clarifying the issue of + operand order and difference between exact and embedded + matching. + + * Documentation fixes for signature 'eval' conditions. (Jon Siwek) + + * Remove orphaned 1.5 unit tests. (Jon Siwek) + + * Add type checking for signature 'eval' condition functions. (Jon + Siwek) + + * Adding an identifier to the SMTP blocklist notices for duplicate + suppression. (Seth Hall) + +2.1-beta-45 | 2012-08-22 16:11:10 -0700 + + * Add an option to the input framework that allows the user to chose + to not die upon encountering files/functions. (Bernhard Amann) + +2.1-beta-41 | 2012-08-22 16:05:21 -0700 + + * Add test serialization to "leak" unit tests that use + communication. (Jon Siwek) + + * Change to metrics/basic-cluster unit test for reliability. (Jon + Siwek) + + * Fixed ack tracking which could overflow quickly in some + situations. (Seth Hall) + + * Minor tweak to coverage.bare-mode-errors unit test to work with a + symlinked 'scripts' dir. (Jon Siwek) + +2.1-beta-35 | 2012-08-22 08:44:52 -0700 + + * Add testcase for input framework reading sets (rather than + tables). (Bernhard Amann) + +2.1-beta-31 | 2012-08-21 15:46:05 -0700 + + * Tweak to rotate-custom.bro unit test. (Jon Siwek) + + * Ignore small mem leak every rotation interval for dataseries logs. + (Jon Siwek) + +2.1-beta-28 | 2012-08-21 08:32:42 -0700 + + * Linking ES docs into logging document. (Robin Sommer) + +2.1-beta-27 | 2012-08-20 20:06:20 -0700 + + * Add the Stream record to Log:active_streams to make more dynamic + logging possible. (Seth Hall) + + * Fix portability of printing to files returned by + open("/dev/stderr"). (Jon Siwek) + + * Fix mime type diff canonifier to also skip mime_desc columns. (Jon + Siwek) + + * Unit test tweaks/fixes. (Jon Siwek) + + - Some baselines for tests in "leaks" group were outdated. + + - Changed a few of the cluster/communication tests to terminate + more explicitly instead of relying on btest-bg-wait to kill + processes. This makes the tests finish faster in the success case + and makes the reason for failing clearer in the that case. + + * Fix memory leak of serialized IDs when compiled with + --enable-debug. (Jon Siwek) + +2.1-beta-21 | 2012-08-16 11:48:56 -0700 + + * Installing a handler for running out of memory in "new". Bro will + now print an error message in that case rather than abort with an + uncaught exception. (Robin Sommer) + +2.1-beta-20 | 2012-08-16 11:43:31 -0700 + + * Fixed potential problems with ElasticSearch output plugin. (Seth + Hall) + +2.1-beta-13 | 2012-08-10 12:28:04 -0700 + + * Reporter warnings and error now print to stderr by default. New + options Reporter::warnings_to_stderr and + Reporter::errors_to_stderr to disable. (Seth Hall) + +2.1-beta-9 | 2012-08-10 12:24:29 -0700 + + * Add more BIF tests. (Daniel Thayer) + +2.1-beta-6 | 2012-08-10 12:22:52 -0700 + + * Fix bug in input framework with an edge case. (Bernhard Amann) + + * Fix small bug in input framework test script. (Bernhard Amann) + +2.1-beta-3 | 2012-08-03 10:46:49 -0700 + + * Merge branch 'master' of ssh://git.bro-ids.org/bro (Robin Sommer) + + * Fix configure script to exit with non-zero status on error (Jon + Siwek) + + * Improve ASCII output performance. (Robin Sommer) + 2.1-beta | 2012-07-30 11:59:53 -0700 * Improve log filter compatibility with remote logging. Addresses diff --git a/CMakeLists.txt b/CMakeLists.txt index f667c0cfe0..2c8a726a1a 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -88,24 +88,30 @@ if (LIBGEOIP_FOUND) list(APPEND OPTLIBS ${LibGeoIP_LIBRARY}) endif () -set(USE_PERFTOOLS false) +set(HAVE_PERFTOOLS false) set(USE_PERFTOOLS_DEBUG false) +set(USE_PERFTOOLS_TCMALLOC false) if (NOT DISABLE_PERFTOOLS) find_package(GooglePerftools) endif () if (GOOGLEPERFTOOLS_FOUND) - include_directories(BEFORE ${GooglePerftools_INCLUDE_DIR}) - set(USE_PERFTOOLS true) + set(HAVE_PERFTOOLS true) + # Non-Linux systems may not be well-supported by gperftools, so + # require explicit request from user to enable it in that case. + if (${CMAKE_SYSTEM_NAME} MATCHES "Linux" OR ENABLE_PERFTOOLS) + set(USE_PERFTOOLS_TCMALLOC true) - if (ENABLE_PERFTOOLS_DEBUG) - # Enable heap debugging with perftools. - set(USE_PERFTOOLS_DEBUG true) - list(APPEND OPTLIBS ${GooglePerftools_LIBRARIES_DEBUG}) - else () - # Link in tcmalloc for better performance. - list(APPEND OPTLIBS ${GooglePerftools_LIBRARIES}) + if (ENABLE_PERFTOOLS_DEBUG) + # Enable heap debugging with perftools. + set(USE_PERFTOOLS_DEBUG true) + include_directories(BEFORE ${GooglePerftools_INCLUDE_DIR}) + list(APPEND OPTLIBS ${GooglePerftools_LIBRARIES_DEBUG}) + else () + # Link in tcmalloc for better performance. + list(APPEND OPTLIBS ${GooglePerftools_LIBRARIES}) + endif () endif () endif () @@ -224,7 +230,8 @@ message( "\nAux. Tools: ${INSTALL_AUX_TOOLS}" "\n" "\nGeoIP: ${USE_GEOIP}" - "\nGoogle perftools: ${USE_PERFTOOLS}" + "\ngperftools found: ${HAVE_PERFTOOLS}" + "\n tcmalloc: ${USE_PERFTOOLS_TCMALLOC}" "\n debugging: ${USE_PERFTOOLS_DEBUG}" "\ncURL: ${USE_CURL}" "\n" diff --git a/NEWS b/NEWS index 949b51d832..d93e153252 100644 --- a/NEWS +++ b/NEWS @@ -7,8 +7,8 @@ release. For a complete list of changes, see the ``CHANGES`` file (note that submodules, such as BroControl and Broccoli, come with their own CHANGES.) -Bro 2.1 Beta ------------- +Bro 2.1 +------- New Functionality ~~~~~~~~~~~~~~~~~ @@ -102,9 +102,14 @@ the full set. * Bro now requires CMake >= 2.6.3. - * Bro now links in tcmalloc (part of Google perftools) if found at - configure time. Doing so can significantly improve memory and - CPU use. + * On Linux, Bro now links in tcmalloc (part of Google perftools) + if found at configure time. Doing so can significantly improve + memory and CPU use. + + On the other platforms, the new configure option + --enable-perftools can be used to enable linking to tcmalloc. + (Note that perftools's support for non-Linux platforms may be + less reliable). - The configure switch --enable-brov6 is gone. @@ -161,6 +166,7 @@ the full set. - The ASCII writers "header_*" options have been renamed to "meta_*" (because there's now also a footer). + Bro 2.0 ------- diff --git a/VERSION b/VERSION index 0fb956a360..879b416e60 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -2.1-beta +2.1 diff --git a/aux/binpac b/aux/binpac index 22120825f8..a93ef13735 160000 --- a/aux/binpac +++ b/aux/binpac @@ -1 +1 @@ -Subproject commit 22120825f8ad70e051ef4ca42f2199aa195dff40 +Subproject commit a93ef1373512c661ffcd0d0a61bd19b96667e0d5 diff --git a/aux/bro-aux b/aux/bro-aux index 941ee753f7..6748ec3a96 160000 --- a/aux/bro-aux +++ b/aux/bro-aux @@ -1 +1 @@ -Subproject commit 941ee753f7c71ec08fc29de04f09a8a83aebb69d +Subproject commit 6748ec3a96d582a977cd9114ef19c76fe75c57ff diff --git a/aux/broccoli b/aux/broccoli index 5ff3e6a8e8..ebfa4de45a 160000 --- a/aux/broccoli +++ b/aux/broccoli @@ -1 +1 @@ -Subproject commit 5ff3e6a8e8535ed91e1f70d355b815ae8eeacb71 +Subproject commit ebfa4de45a839e58aec200e7e4bad33eaab4f1ed diff --git a/aux/broctl b/aux/broctl index 903108f6b4..2fb9ff62bf 160000 --- a/aux/broctl +++ b/aux/broctl @@ -1 +1 @@ -Subproject commit 903108f6b43ad228309713da880026d50add41f4 +Subproject commit 2fb9ff62bf08f78071753016863640022fbfe338 diff --git a/configure b/configure index b4ca606103..8e4aaa8425 100755 --- a/configure +++ b/configure @@ -29,6 +29,8 @@ Usage: $0 [OPTION]... [VAR=VALUE]... Optional Features: --enable-debug compile in debugging mode --enable-mobile-ipv6 analyze mobile IPv6 features defined by RFC 6275 + --enable-perftools force use of Google perftools on non-Linux systems + (automatically on when perftools is present on Linux) --enable-perftools-debug use Google's perftools for debugging --disable-broccoli don't build or install the Broccoli library --disable-broctl don't install Broctl @@ -98,6 +100,7 @@ append_cache_entry PY_MOD_INSTALL_DIR PATH $prefix/lib/broctl append_cache_entry BRO_SCRIPT_INSTALL_PATH STRING $prefix/share/bro append_cache_entry BRO_ETC_INSTALL_DIR PATH $prefix/etc append_cache_entry ENABLE_DEBUG BOOL false +append_cache_entry ENABLE_PERFTOOLS BOOL false append_cache_entry ENABLE_PERFTOOLS_DEBUG BOOL false append_cache_entry BinPAC_SKIP_INSTALL BOOL true append_cache_entry BUILD_SHARED_LIBS BOOL true @@ -146,7 +149,11 @@ while [ $# -ne 0 ]; do --enable-mobile-ipv6) append_cache_entry ENABLE_MOBILE_IPV6 BOOL true ;; + --enable-perftools) + append_cache_entry ENABLE_PERFTOOLS BOOL true + ;; --enable-perftools-debug) + append_cache_entry ENABLE_PERFTOOLS BOOL true append_cache_entry ENABLE_PERFTOOLS_DEBUG BOOL true ;; --disable-broccoli) diff --git a/doc/logging.rst b/doc/logging.rst index cc6cb1e54d..7fb4205b9a 100644 --- a/doc/logging.rst +++ b/doc/logging.rst @@ -383,3 +383,4 @@ Bro supports the following output formats other than ASCII: :maxdepth: 1 logging-dataseries + logging-elasticsearch diff --git a/doc/scripts/builtins.rst b/doc/scripts/builtins.rst index 32908f71fd..0501067409 100644 --- a/doc/scripts/builtins.rst +++ b/doc/scripts/builtins.rst @@ -55,8 +55,8 @@ The Bro scripting language supports the following built-in types. A temporal type representing a relative time. An ``interval`` constant can be written as a numeric constant followed by a time - unit where the time unit is one of ``usec``, ``sec``, ``min``, - ``hr``, or ``day`` which respectively represent microseconds, + unit where the time unit is one of ``usec``, ``msec``, ``sec``, ``min``, + ``hr``, or ``day`` which respectively represent microseconds, milliseconds, seconds, minutes, hours, and days. Whitespace between the numeric constant and time unit is optional. Appending the letter "s" to the time unit in order to pluralize it is also optional (to no semantic @@ -95,14 +95,14 @@ The Bro scripting language supports the following built-in types. and embedded. In exact matching the ``==`` equality relational operator is used - with one :bro:type:`string` operand and one :bro:type:`pattern` - operand to check whether the full string exactly matches the - pattern. In this case, the ``^`` beginning-of-line and ``$`` - end-of-line anchors are redundant since pattern is implicitly - anchored to the beginning and end of the line to facilitate an exact - match. For example:: + with one :bro:type:`pattern` operand and one :bro:type:`string` + operand (order of operands does not matter) to check whether the full + string exactly matches the pattern. In exact matching, the ``^`` + beginning-of-line and ``$`` end-of-line anchors are redundant since + the pattern is implicitly anchored to the beginning and end of the + line to facilitate an exact match. For example:: - "foo" == /foo|bar/ + /foo|bar/ == "foo" yields true, while:: @@ -110,9 +110,9 @@ The Bro scripting language supports the following built-in types. yields false. The ``!=`` operator would yield the negation of ``==``. - In embedded matching the ``in`` operator is again used with one - :bro:type:`string` operand and one :bro:type:`pattern` operand - (which must be on the left-hand side), but tests whether the pattern + In embedded matching the ``in`` operator is used with one + :bro:type:`pattern` operand (which must be on the left-hand side) and + one :bro:type:`string` operand, but tests whether the pattern appears anywhere within the given string. For example:: /foo|bar/ in "foobar" diff --git a/doc/signatures.rst b/doc/signatures.rst index f65215eceb..36099ba40f 100644 --- a/doc/signatures.rst +++ b/doc/signatures.rst @@ -229,20 +229,10 @@ matched. The following context conditions are defined: confirming the match. If false is returned, no signature match is going to be triggered. The function has to be of type ``function cond(state: signature_state, data: string): bool``. Here, - ``content`` may contain the most recent content chunk available at + ``data`` may contain the most recent content chunk available at the time the signature was matched. If no such chunk is available, - ``content`` will be the empty string. ``signature_state`` is - defined as follows: - - .. code:: bro - - type signature_state: record { - id: string; # ID of the signature - conn: connection; # Current connection - is_orig: bool; # True if current endpoint is originator - payload_size: count; # Payload size of the first packet - }; - + ``data`` will be the empty string. See :bro:type:`signature_state` + for its definition. ``payload-size `` Compares the integer to the size of the payload of a packet. For diff --git a/scripts/base/frameworks/input/main.bro b/scripts/base/frameworks/input/main.bro index 55da6ae7ec..b5f44669c2 100644 --- a/scripts/base/frameworks/input/main.bro +++ b/scripts/base/frameworks/input/main.bro @@ -8,8 +8,16 @@ export { ## The default input reader used. Defaults to `READER_ASCII`. const default_reader = READER_ASCII &redef; + ## The default reader mode used. Defaults to `MANUAL`. const default_mode = MANUAL &redef; + ## Flag that controls if the input framework accepts records + ## that contain types that are not supported (at the moment + ## file and function). If true, the input framework will + ## warn in these cases, but continue. If false, it will + ## abort. Defaults to false (abort) + const accept_unsupported_types = F &redef; + ## TableFilter description type used for the `table` method. type TableDescription: record { ## Common definitions for tables and events diff --git a/scripts/base/frameworks/logging/main.bro b/scripts/base/frameworks/logging/main.bro index ccc65ddf67..bed76a1ae5 100644 --- a/scripts/base/frameworks/logging/main.bro +++ b/scripts/base/frameworks/logging/main.bro @@ -329,9 +329,9 @@ export { global run_rotation_postprocessor_cmd: function(info: RotationInfo, npath: string) : bool; ## The streams which are currently active and not disabled. - ## This set is not meant to be modified by users! Only use it for + ## This table is not meant to be modified by users! Only use it for ## examining which streams are active. - global active_streams: set[ID] = set(); + global active_streams: table[ID] of Stream = table(); } # We keep a script-level copy of all filters so that we can manipulate them. @@ -417,7 +417,7 @@ function create_stream(id: ID, stream: Stream) : bool if ( ! __create_stream(id, stream) ) return F; - add active_streams[id]; + active_streams[id] = stream; return add_default_filter(id); } diff --git a/scripts/base/frameworks/reporter/main.bro b/scripts/base/frameworks/reporter/main.bro index 3c19005364..edc5b1779a 100644 --- a/scripts/base/frameworks/reporter/main.bro +++ b/scripts/base/frameworks/reporter/main.bro @@ -1,5 +1,5 @@ -##! This framework is intended to create an output and filtering path for -##! internal messages/warnings/errors. It should typically be loaded to +##! This framework is intended to create an output and filtering path for +##! internal messages/warnings/errors. It should typically be loaded to ##! avoid Bro spewing internal messages to standard error and instead log ##! them to a file in a standard way. Note that this framework deals with ##! the handling of internally-generated reporter messages, for the @@ -13,11 +13,11 @@ export { redef enum Log::ID += { LOG }; ## An indicator of reporter message severity. - type Level: enum { + type Level: enum { ## Informational, not needing specific attention. - INFO, + INFO, ## Warning of a potential problem. - WARNING, + WARNING, ## A non-fatal error that should be addressed, but doesn't ## terminate program execution. ERROR @@ -36,24 +36,55 @@ export { ## Not all reporter messages will have locations in them though. location: string &log &optional; }; + + ## Tunable for sending reporter warning messages to STDERR. The option to + ## turn it off is presented here in case Bro is being run by some + ## external harness and shouldn't output anything to the console. + const warnings_to_stderr = T &redef; + + ## Tunable for sending reporter error messages to STDERR. The option to + ## turn it off is presented here in case Bro is being run by some + ## external harness and shouldn't output anything to the console. + const errors_to_stderr = T &redef; } +global stderr: file; + event bro_init() &priority=5 { Log::create_stream(Reporter::LOG, [$columns=Info]); + + if ( errors_to_stderr || warnings_to_stderr ) + stderr = open("/dev/stderr"); } -event reporter_info(t: time, msg: string, location: string) +event reporter_info(t: time, msg: string, location: string) &priority=-5 { Log::write(Reporter::LOG, [$ts=t, $level=INFO, $message=msg, $location=location]); } - -event reporter_warning(t: time, msg: string, location: string) + +event reporter_warning(t: time, msg: string, location: string) &priority=-5 { + if ( warnings_to_stderr ) + { + if ( t > double_to_time(0.0) ) + print stderr, fmt("WARNING: %.6f %s (%s)", t, msg, location); + else + print stderr, fmt("WARNING: %s (%s)", msg, location); + } + Log::write(Reporter::LOG, [$ts=t, $level=WARNING, $message=msg, $location=location]); } -event reporter_error(t: time, msg: string, location: string) +event reporter_error(t: time, msg: string, location: string) &priority=-5 { + if ( errors_to_stderr ) + { + if ( t > double_to_time(0.0) ) + print stderr, fmt("ERROR: %.6f %s (%s)", t, msg, location); + else + print stderr, fmt("ERROR: %s (%s)", msg, location); + } + Log::write(Reporter::LOG, [$ts=t, $level=ERROR, $message=msg, $location=location]); } diff --git a/scripts/policy/protocols/smtp/blocklists.bro b/scripts/policy/protocols/smtp/blocklists.bro index a3e75318bb..b1fb0e498d 100644 --- a/scripts/policy/protocols/smtp/blocklists.bro +++ b/scripts/policy/protocols/smtp/blocklists.bro @@ -1,3 +1,4 @@ +##! Watch for various SPAM blocklist URLs in SMTP error messages. @load base/protocols/smtp @@ -5,9 +6,11 @@ module SMTP; export { redef enum Notice::Type += { - ## Indicates that the server sent a reply mentioning an SMTP block list. + ## An SMTP server sent a reply mentioning an SMTP block list. Blocklist_Error_Message, - ## Indicates the client's address is seen in the block list error message. + ## The originator's address is seen in the block list error message. + ## This is useful to detect local hosts sending SPAM with a high + ## positive rate. Blocklist_Blocked_Host, }; @@ -52,7 +55,8 @@ event smtp_reply(c: connection, is_orig: bool, code: count, cmd: string, message = fmt("%s is on an SMTP block list", c$id$orig_h); } - NOTICE([$note=note, $conn=c, $msg=message, $sub=msg]); + NOTICE([$note=note, $conn=c, $msg=message, $sub=msg, + $identifier=cat(c$id$orig_h)]); } } } diff --git a/scripts/policy/tuning/logs-to-elasticsearch.bro b/scripts/policy/tuning/logs-to-elasticsearch.bro index 207a9acc04..2a4b70362a 100644 --- a/scripts/policy/tuning/logs-to-elasticsearch.bro +++ b/scripts/policy/tuning/logs-to-elasticsearch.bro @@ -8,13 +8,13 @@ export { ## Optionally ignore any :bro:type:`Log::ID` from being sent to ## ElasticSearch with this script. - const excluded_log_ids: set[string] = set("Communication::LOG") &redef; + const excluded_log_ids: set[Log::ID] &redef; ## If you want to explicitly only send certain :bro:type:`Log::ID` ## streams, add them to this set. If the set remains empty, all will ## be sent. The :bro:id:`LogElasticSearch::excluded_log_ids` option will remain in ## effect as well. - const send_logs: set[string] = set() &redef; + const send_logs: set[Log::ID] &redef; } event bro_init() &priority=-5 @@ -24,8 +24,8 @@ event bro_init() &priority=-5 for ( stream_id in Log::active_streams ) { - if ( fmt("%s", stream_id) in excluded_log_ids || - (|send_logs| > 0 && fmt("%s", stream_id) !in send_logs) ) + if ( stream_id in excluded_log_ids || + (|send_logs| > 0 && stream_id !in send_logs) ) next; local filter: Log::Filter = [$name = "default-es", diff --git a/src/File.cc b/src/File.cc index 20e845c09f..3b9f3be33b 100644 --- a/src/File.cc +++ b/src/File.cc @@ -138,11 +138,22 @@ BroFile::BroFile(FILE* arg_f, const char* arg_name, const char* arg_access) BroFile::BroFile(const char* arg_name, const char* arg_access, BroType* arg_t) { Init(); - + f = 0; name = copy_string(arg_name); access = copy_string(arg_access); t = arg_t ? arg_t : base_type(TYPE_STRING); - if ( ! Open() ) + + if ( streq(name, "/dev/stdin") ) + f = stdin; + else if ( streq(name, "/dev/stdout") ) + f = stdout; + else if ( streq(name, "/dev/stderr") ) + f = stderr; + + if ( f ) + is_open = 1; + + else if ( ! Open() ) { reporter->Error("cannot open %s: %s", name, strerror(errno)); is_open = 0; @@ -342,8 +353,8 @@ int BroFile::Close() FinishEncrypt(); - // Do not close stdout/stderr. - if ( f == stdout || f == stderr ) + // Do not close stdin/stdout/stderr. + if ( f == stdin || f == stdout || f == stderr ) return 0; if ( is_in_cache ) @@ -508,7 +519,7 @@ void BroFile::SetAttrs(Attributes* arg_attrs) if ( attrs->FindAttr(ATTR_RAW_OUTPUT) ) EnableRawOutput(); - + InstallRotateTimer(); } @@ -523,6 +534,10 @@ RecordVal* BroFile::Rotate() if ( ! is_open ) return 0; + // Do not rotate stdin/stdout/stderr. + if ( f == stdin || f == stdout || f == stderr ) + return 0; + if ( okay_to_manage && ! is_in_cache ) BringIntoCache(); diff --git a/src/RemoteSerializer.cc b/src/RemoteSerializer.cc index cfd20eba39..564ad2be68 100644 --- a/src/RemoteSerializer.cc +++ b/src/RemoteSerializer.cc @@ -2897,11 +2897,6 @@ void RemoteSerializer::GotID(ID* id, Val* val) (desc && *desc) ? desc : "not set"), current_peer); -#ifdef USE_PERFTOOLS_DEBUG - // May still be cached, but we don't care. - heap_checker->IgnoreObject(id); -#endif - Unref(id); return; } diff --git a/src/RuleCondition.cc b/src/RuleCondition.cc index 8852747cc4..410f6a1b3e 100644 --- a/src/RuleCondition.cc +++ b/src/RuleCondition.cc @@ -126,6 +126,23 @@ RuleConditionEval::RuleConditionEval(const char* func) rules_error("unknown identifier", func); return; } + + if ( id->Type()->Tag() == TYPE_FUNC ) + { + // Validate argument quantity and type. + FuncType* f = id->Type()->AsFuncType(); + + if ( f->YieldType()->Tag() != TYPE_BOOL ) + rules_error("eval function type must yield a 'bool'", func); + + TypeList tl; + tl.Append(internal_type("signature_state")->Ref()); + tl.Append(base_type(TYPE_STRING)); + + if ( ! f->CheckArgs(tl.Types()) ) + rules_error("eval function parameters must be a 'signature_state' " + "and a 'string' type", func); + } } bool RuleConditionEval::DoMatch(Rule* rule, RuleEndpointState* state, diff --git a/src/Stats.cc b/src/Stats.cc index c3035231e9..8d48c47a25 100644 --- a/src/Stats.cc +++ b/src/Stats.cc @@ -12,10 +12,10 @@ int killed_by_inactivity = 0; -uint32 tot_ack_events = 0; -uint32 tot_ack_bytes = 0; -uint32 tot_gap_events = 0; -uint32 tot_gap_bytes = 0; +uint64 tot_ack_events = 0; +uint64 tot_ack_bytes = 0; +uint64 tot_gap_events = 0; +uint64 tot_gap_bytes = 0; class ProfileTimer : public Timer { diff --git a/src/Stats.h b/src/Stats.h index eeebfe2213..a11d66828a 100644 --- a/src/Stats.h +++ b/src/Stats.h @@ -116,10 +116,10 @@ extern SampleLogger* sample_logger; extern int killed_by_inactivity; // Content gap statistics. -extern uint32 tot_ack_events; -extern uint32 tot_ack_bytes; -extern uint32 tot_gap_events; -extern uint32 tot_gap_bytes; +extern uint64 tot_ack_events; +extern uint64 tot_ack_bytes; +extern uint64 tot_gap_events; +extern uint64 tot_gap_bytes; // A TCPStateStats object tracks the distribution of TCP states for diff --git a/src/TCP.cc b/src/TCP.cc index 57e4449bf8..555adf1b57 100644 --- a/src/TCP.cc +++ b/src/TCP.cc @@ -46,6 +46,7 @@ TCP_Analyzer::TCP_Analyzer(Connection* conn) finished = 0; reassembling = 0; first_packet_seen = 0; + is_partial = 0; orig = new TCP_Endpoint(this, 1); resp = new TCP_Endpoint(this, 0); diff --git a/src/TCP_Reassembler.cc b/src/TCP_Reassembler.cc index fb67dba7ee..eb2709373c 100644 --- a/src/TCP_Reassembler.cc +++ b/src/TCP_Reassembler.cc @@ -20,10 +20,10 @@ const bool DEBUG_tcp_connection_close = false; const bool DEBUG_tcp_match_undelivered = false; static double last_gap_report = 0.0; -static uint32 last_ack_events = 0; -static uint32 last_ack_bytes = 0; -static uint32 last_gap_events = 0; -static uint32 last_gap_bytes = 0; +static uint64 last_ack_events = 0; +static uint64 last_ack_bytes = 0; +static uint64 last_gap_events = 0; +static uint64 last_gap_bytes = 0; TCP_Reassembler::TCP_Reassembler(Analyzer* arg_dst_analyzer, TCP_Analyzer* arg_tcp_analyzer, @@ -513,10 +513,10 @@ void TCP_Reassembler::AckReceived(int seq) if ( gap_report && gap_report_freq > 0.0 && dt >= gap_report_freq ) { - int devents = tot_ack_events - last_ack_events; - int dbytes = tot_ack_bytes - last_ack_bytes; - int dgaps = tot_gap_events - last_gap_events; - int dgap_bytes = tot_gap_bytes - last_gap_bytes; + uint64 devents = tot_ack_events - last_ack_events; + uint64 dbytes = tot_ack_bytes - last_ack_bytes; + uint64 dgaps = tot_gap_events - last_gap_events; + uint64 dgap_bytes = tot_gap_bytes - last_gap_bytes; RecordVal* r = new RecordVal(gap_info); r->Assign(0, new Val(devents, TYPE_COUNT)); diff --git a/src/Val.cc b/src/Val.cc index 8a8c2b18c0..79fa8a0c69 100644 --- a/src/Val.cc +++ b/src/Val.cc @@ -64,7 +64,7 @@ Val::~Val() Unref(type); #ifdef DEBUG - Unref(bound_id); + delete [] bound_id; #endif } diff --git a/src/Val.h b/src/Val.h index 2ca18e6131..c3ec5b04fb 100644 --- a/src/Val.h +++ b/src/Val.h @@ -347,13 +347,15 @@ public: #ifdef DEBUG // For debugging, we keep a reference to the global ID to which a // value has been bound *last*. - ID* GetID() const { return bound_id; } + ID* GetID() const + { + return bound_id ? global_scope()->Lookup(bound_id) : 0; + } + void SetID(ID* id) { - if ( bound_id ) - ::Unref(bound_id); - bound_id = id; - ::Ref(bound_id); + delete [] bound_id; + bound_id = id ? copy_string(id->Name()) : 0; } #endif @@ -401,8 +403,8 @@ protected: RecordVal* attribs; #ifdef DEBUG - // For debugging, we keep the ID to which a Val is bound. - ID* bound_id; + // For debugging, we keep the name of the ID to which a Val is bound. + const char* bound_id; #endif }; diff --git a/src/bro.bif b/src/bro.bif index 2a37429ad6..bc791d6858 100644 --- a/src/bro.bif +++ b/src/bro.bif @@ -3787,7 +3787,7 @@ static GeoIP* open_geoip_db(GeoIPDBTypes type) geoip = GeoIP_open_type(type, GEOIP_MEMORY_CACHE); if ( ! geoip ) - reporter->Warning("Failed to open GeoIP database: %s", + reporter->Info("Failed to open GeoIP database: %s", GeoIPDBFileName[type]); return geoip; } @@ -3827,7 +3827,7 @@ function lookup_location%(a: addr%) : geo_location if ( ! geoip ) builtin_error("Can't initialize GeoIP City/Country database"); else - reporter->Warning("Fell back to GeoIP Country database"); + reporter->Info("Fell back to GeoIP Country database"); } else have_city_db = true; diff --git a/src/input.bif b/src/input.bif index f494ef3b2f..199b665fa6 100644 --- a/src/input.bif +++ b/src/input.bif @@ -34,6 +34,10 @@ function Input::__force_update%(id: string%) : bool return new Val(res, TYPE_BOOL); %} +# Options for the input framework + +const accept_unsupported_types: bool; + # Options for Ascii Reader module InputAscii; diff --git a/src/input/Manager.cc b/src/input/Manager.cc index 64e54f9333..6eadb3aba8 100644 --- a/src/input/Manager.cc +++ b/src/input/Manager.cc @@ -388,6 +388,8 @@ bool Manager::CreateEventStream(RecordVal* fval) FuncType* etype = event->FType()->AsFuncType(); + bool allow_file_func = false; + if ( ! etype->IsEvent() ) { reporter->Error("stream event is a function, not an event"); @@ -453,6 +455,8 @@ bool Manager::CreateEventStream(RecordVal* fval) return false; } + allow_file_func = BifConst::Input::accept_unsupported_types; + } else @@ -461,7 +465,7 @@ bool Manager::CreateEventStream(RecordVal* fval) vector fieldsV; // vector, because UnrollRecordType needs it - bool status = !UnrollRecordType(&fieldsV, fields, ""); + bool status = (! UnrollRecordType(&fieldsV, fields, "", allow_file_func)); if ( status ) { @@ -609,12 +613,12 @@ bool Manager::CreateTableStream(RecordVal* fval) vector fieldsV; // vector, because we don't know the length beforehands - bool status = !UnrollRecordType(&fieldsV, idx, ""); + bool status = (! UnrollRecordType(&fieldsV, idx, "", false)); int idxfields = fieldsV.size(); if ( val ) // if we are not a set - status = status || !UnrollRecordType(&fieldsV, val, ""); + status = status || ! UnrollRecordType(&fieldsV, val, "", BifConst::Input::accept_unsupported_types); int valfields = fieldsV.size() - idxfields; @@ -772,15 +776,29 @@ bool Manager::RemoveStreamContinuation(ReaderFrontend* reader) return true; } -bool Manager::UnrollRecordType(vector *fields, - const RecordType *rec, const string& nameprepend) +bool Manager::UnrollRecordType(vector *fields, const RecordType *rec, + const string& nameprepend, bool allow_file_func) { - for ( int i = 0; i < rec->NumFields(); i++ ) { if ( ! IsCompatibleType(rec->FieldType(i)) ) - { + { + // If the field is a file or a function type + // and it is optional, we accept it nevertheless. + // This allows importing logfiles containing this + // stuff that we actually cannot read :) + if ( allow_file_func ) + { + if ( ( rec->FieldType(i)->Tag() == TYPE_FILE || + rec->FieldType(i)->Tag() == TYPE_FUNC ) && + rec->FieldDecl(i)->FindAttr(ATTR_OPTIONAL) ) + { + reporter->Info("Encountered incompatible type \"%s\" in table definition for ReaderFrontend. Ignoring field.", type_name(rec->FieldType(i)->Tag())); + continue; + } + } + reporter->Error("Incompatible type \"%s\" in table definition for ReaderFrontend", type_name(rec->FieldType(i)->Tag())); return false; } @@ -789,7 +807,7 @@ bool Manager::UnrollRecordType(vector *fields, { string prep = nameprepend + rec->FieldName(i) + "."; - if ( !UnrollRecordType(fields, rec->FieldType(i)->AsRecordType(), prep) ) + if ( !UnrollRecordType(fields, rec->FieldType(i)->AsRecordType(), prep, allow_file_func) ) { return false; } @@ -1044,9 +1062,7 @@ int Manager::SendEntryTable(Stream* i, const Value* const *vals) if ( ! updated ) { - // throw away. Hence - we quit. And remove the entry from the current dictionary... - // (but why should it be in there? assert this). - assert ( stream->currDict->RemoveEntry(idxhash) == 0 ); + // just quit and delete everything we created. delete idxhash; delete h; return stream->num_val_fields + stream->num_idx_fields; @@ -1212,7 +1228,7 @@ void Manager::EndCurrentSend(ReaderFrontend* reader) Ref(predidx); Ref(val); Ref(ev); - SendEvent(stream->event, 3, ev, predidx, val); + SendEvent(stream->event, 4, stream->description->Ref(), ev, predidx, val); } if ( predidx ) // if we have a stream or an event... @@ -1677,6 +1693,18 @@ RecordVal* Manager::ValueToRecordVal(const Value* const *vals, Val* fieldVal = 0; if ( request_type->FieldType(i)->Tag() == TYPE_RECORD ) fieldVal = ValueToRecordVal(vals, request_type->FieldType(i)->AsRecordType(), position); + else if ( request_type->FieldType(i)->Tag() == TYPE_FILE || + request_type->FieldType(i)->Tag() == TYPE_FUNC ) + { + // If those two unsupported types are encountered here, they have + // been let through by the type checking. + // That means that they are optional & the user agreed to ignore + // them and has been warned by reporter. + // Hence -> assign null to the field, done. + + // Better check that it really is optional. Uou never know. + assert(request_type->FieldDecl(i)->FindAttr(ATTR_OPTIONAL)); + } else { fieldVal = ValueToVal(vals[*position], request_type->FieldType(i)); @@ -1720,7 +1748,7 @@ int Manager::GetValueLength(const Value* val) { case TYPE_STRING: case TYPE_ENUM: { - length += val->val.string_val.length; + length += val->val.string_val.length + 1; break; } @@ -1820,7 +1848,10 @@ int Manager::CopyValue(char *data, const int startpos, const Value* val) case TYPE_ENUM: { memcpy(data+startpos, val->val.string_val.data, val->val.string_val.length); - return val->val.string_val.length; + // Add a \0 to the end. To be able to hash zero-length + // strings and differentiate from !present. + memset(data + startpos + val->val.string_val.length, 0, 1); + return val->val.string_val.length + 1; } case TYPE_ADDR: @@ -1911,13 +1942,15 @@ HashKey* Manager::HashValues(const int num_elements, const Value* const *vals) const Value* val = vals[i]; if ( val->present ) length += GetValueLength(val); + + // And in any case add 1 for the end-of-field-identifier. + length++; } - if ( length == 0 ) - { - reporter->Error("Input reader sent line where all elements are null values. Ignoring line"); + assert ( length >= num_elements ); + + if ( length == num_elements ) return NULL; - } int position = 0; char *data = (char*) malloc(length); @@ -1929,6 +1962,12 @@ HashKey* Manager::HashValues(const int num_elements, const Value* const *vals) const Value* val = vals[i]; if ( val->present ) position += CopyValue(data, position, val); + + memset(data + position, 1, 1); // Add end-of-field-marker. Does not really matter which value it is, + // it just has to be... something. + + position++; + } HashKey *key = new HashKey(data, length); diff --git a/src/input/Manager.h b/src/input/Manager.h index 1590042183..cc81df38b7 100644 --- a/src/input/Manager.h +++ b/src/input/Manager.h @@ -158,7 +158,7 @@ private: // Check if a record is made up of compatible types and return a list // of all fields that are in the record in order. Recursively unrolls // records - bool UnrollRecordType(vector *fields, const RecordType *rec, const string& nameprepend); + bool UnrollRecordType(vector *fields, const RecordType *rec, const string& nameprepend, bool allow_file_func); // Send events void SendEvent(EventHandlerPtr ev, const int numvals, ...); diff --git a/src/input/readers/Ascii.cc b/src/input/readers/Ascii.cc index fd936b07b6..e9cba27205 100644 --- a/src/input/readers/Ascii.cc +++ b/src/input/readers/Ascii.cc @@ -11,6 +11,7 @@ #include #include #include +#include using namespace input::reader; using threading::Value; @@ -209,6 +210,42 @@ bool Ascii::GetLine(string& str) return false; } +bool Ascii::CheckNumberError(const string& s, const char * end) + { + // Do this check first, before executing s.c_str() or similar. + // otherwise the value to which *end is pointing at the moment might + // be gone ... + bool endnotnull = (*end != '\0'); + + if ( s.length() == 0 ) + { + Error("Got empty string for number field"); + return true; + } + + if ( end == s.c_str() ) { + Error(Fmt("String '%s' contained no parseable number", s.c_str())); + return true; + } + + if ( endnotnull ) + Warning(Fmt("Number '%s' contained non-numeric trailing characters. Ignored trailing characters '%s'", s.c_str(), end)); + + if ( errno == EINVAL ) + { + Error(Fmt("String '%s' could not be converted to a number", s.c_str())); + return true; + } + + else if ( errno == ERANGE ) + { + Error(Fmt("Number '%s' out of supported range.", s.c_str())); + return true; + } + + return false; + } + Value* Ascii::EntryToVal(string s, FieldMapping field) { @@ -216,10 +253,13 @@ Value* Ascii::EntryToVal(string s, FieldMapping field) return new Value(field.type, false); Value* val = new Value(field.type, true); + char* end = 0; + errno = 0; switch ( field.type ) { case TYPE_ENUM: case TYPE_STRING: + s = get_unescaped_string(s); val->val.string_val.length = s.size(); val->val.string_val.data = copy_string(s.c_str()); break; @@ -238,27 +278,37 @@ Value* Ascii::EntryToVal(string s, FieldMapping field) break; case TYPE_INT: - val->val.int_val = atoi(s.c_str()); + val->val.int_val = strtoll(s.c_str(), &end, 10); + if ( CheckNumberError(s, end) ) + return 0; break; case TYPE_DOUBLE: case TYPE_TIME: case TYPE_INTERVAL: - val->val.double_val = atof(s.c_str()); + val->val.double_val = strtod(s.c_str(), &end); + if ( CheckNumberError(s, end) ) + return 0; break; case TYPE_COUNT: case TYPE_COUNTER: - val->val.uint_val = atoi(s.c_str()); + val->val.uint_val = strtoull(s.c_str(), &end, 10); + if ( CheckNumberError(s, end) ) + return 0; break; case TYPE_PORT: - val->val.port_val.port = atoi(s.c_str()); + val->val.port_val.port = strtoull(s.c_str(), &end, 10); + if ( CheckNumberError(s, end) ) + return 0; + val->val.port_val.proto = TRANSPORT_UNKNOWN; break; case TYPE_SUBNET: { + s = get_unescaped_string(s); size_t pos = s.find("/"); if ( pos == s.npos ) { @@ -266,7 +316,11 @@ Value* Ascii::EntryToVal(string s, FieldMapping field) return 0; } - int width = atoi(s.substr(pos+1).c_str()); + uint8_t width = (uint8_t) strtol(s.substr(pos+1).c_str(), &end, 10); + + if ( CheckNumberError(s, end) ) + return 0; + string addr = s.substr(0, pos); val->val.subnet_val.prefix = StringToAddr(addr); @@ -275,6 +329,7 @@ Value* Ascii::EntryToVal(string s, FieldMapping field) } case TYPE_ADDR: + s = get_unescaped_string(s); val->val.addr_val = StringToAddr(s); break; @@ -288,7 +343,10 @@ Value* Ascii::EntryToVal(string s, FieldMapping field) // how many entries do we have... unsigned int length = 1; for ( unsigned int i = 0; i < s.size(); i++ ) - if ( s[i] == ',' ) length++; + { + if ( s[i] == set_separator[0] ) + length++; + } unsigned int pos = 0; @@ -342,9 +400,24 @@ Value* Ascii::EntryToVal(string s, FieldMapping field) pos++; } + // Test if the string ends with a set_separator... or if the + // complete string is empty. In either of these cases we have + // to push an empty val on top of it. + if ( s.empty() || *s.rbegin() == set_separator[0] ) + { + lvals[pos] = EntryToVal("", field.subType()); + if ( lvals[pos] == 0 ) + { + Error("Error while trying to add empty set element"); + return 0; + } + + pos++; + } + if ( pos != length ) { - Error("Internal error while parsing set: did not find all elements"); + Error(Fmt("Internal error while parsing set: did not find all elements: %s", s.c_str())); return 0; } @@ -428,6 +501,7 @@ bool Ascii::DoUpdate() while ( GetLine(line ) ) { // split on tabs + bool error = false; istringstream splitstream(line); map stringfields; @@ -438,8 +512,6 @@ bool Ascii::DoUpdate() if ( ! getline(splitstream, s, separator[0]) ) break; - s = get_unescaped_string(s); - stringfields[pos] = s; pos++; } @@ -474,8 +546,9 @@ bool Ascii::DoUpdate() Value* val = EntryToVal(stringfields[(*fit).position], *fit); if ( val == 0 ) { - Error("Could not convert String value to Val"); - return false; + Error(Fmt("Could not convert line '%s' to Val. Ignoring line.", line.c_str())); + error = true; + break; } if ( (*fit).secondary_position != -1 ) @@ -492,6 +565,19 @@ bool Ascii::DoUpdate() fpos++; } + if ( error ) + { + // Encountered non-fatal error, ignoring line. But + // first, delete all successfully read fields and the + // array structure. + + for ( int i = 0; i < fpos; i++ ) + delete fields[fpos]; + + delete [] fields; + continue; + } + //printf("fpos: %d, second.num_fields: %d\n", fpos, (*it).second.num_fields); assert ( fpos == NumFields() ); diff --git a/src/input/readers/Ascii.h b/src/input/readers/Ascii.h index e1506cbe82..6e693fc74b 100644 --- a/src/input/readers/Ascii.h +++ b/src/input/readers/Ascii.h @@ -48,6 +48,7 @@ private: bool ReadHeader(bool useCached); bool GetLine(string& str); threading::Value* EntryToVal(string s, FieldMapping type); + bool CheckNumberError(const string& s, const char * end); ifstream* file; time_t mtime; diff --git a/src/logging/writers/Ascii.cc b/src/logging/writers/Ascii.cc index f6df3b9336..11b322f5a3 100644 --- a/src/logging/writers/Ascii.cc +++ b/src/logging/writers/Ascii.cc @@ -359,7 +359,7 @@ bool Ascii::DoWrite(int num_fields, const Field* const * fields, if ( ! safe_write(fd, bytes, len) ) goto write_error; - if ( IsBuf() ) + if ( ! IsBuf() ) fsync(fd); return true; diff --git a/src/logging/writers/DataSeries.cc b/src/logging/writers/DataSeries.cc index 7d3053e341..bc5a82ec54 100644 --- a/src/logging/writers/DataSeries.cc +++ b/src/logging/writers/DataSeries.cc @@ -243,8 +243,25 @@ bool DataSeries::OpenLog(string path) log_file->writeExtentLibrary(log_types); for( size_t i = 0; i < schema_list.size(); ++i ) - extents.insert(std::make_pair(schema_list[i].field_name, - GeneralField::create(log_series, schema_list[i].field_name))); + { + string fn = schema_list[i].field_name; + GeneralField* gf = 0; +#ifdef USE_PERFTOOLS_DEBUG + { + // GeneralField isn't cleaning up some results of xml parsing, reported + // here: https://github.com/dataseries/DataSeries/issues/1 + // Ignore for now to make leak tests pass. There's confidence that + // we do clean up the GeneralField* since the ExtentSeries dtor for + // member log_series would trigger an assert if dynamically allocated + // fields aren't deleted beforehand. + HeapLeakChecker::Disabler disabler; +#endif + gf = GeneralField::create(log_series, fn); +#ifdef USE_PERFTOOLS_DEBUG + } +#endif + extents.insert(std::make_pair(fn, gf)); + } if ( ds_extent_size < ROW_MIN ) { diff --git a/src/logging/writers/ElasticSearch.cc b/src/logging/writers/ElasticSearch.cc index e688686b35..cb3248a044 100644 --- a/src/logging/writers/ElasticSearch.cc +++ b/src/logging/writers/ElasticSearch.cc @@ -371,7 +371,11 @@ bool ElasticSearch::HTTPSend(CURL *handle) // The best (only?) way to disable that is to just use HTTP 1.0 curl_easy_setopt(handle, CURLOPT_HTTP_VERSION, CURL_HTTP_VERSION_1_0); - //curl_easy_setopt(handle, CURLOPT_TIMEOUT_MS, transfer_timeout); + // Some timeout options. These will need more attention later. + curl_easy_setopt(handle, CURLOPT_NOSIGNAL, 1); + curl_easy_setopt(handle, CURLOPT_CONNECTTIMEOUT_MS, transfer_timeout); + curl_easy_setopt(handle, CURLOPT_TIMEOUT_MS, transfer_timeout*2); + curl_easy_setopt(handle, CURLOPT_DNS_CACHE_TIMEOUT, 60*60); CURLcode return_code = curl_easy_perform(handle); diff --git a/src/main.cc b/src/main.cc index 407f67c9af..5999186240 100644 --- a/src/main.cc +++ b/src/main.cc @@ -337,6 +337,8 @@ void terminate_bro() delete log_mgr; delete thread_mgr; delete reporter; + + reporter = 0; } void termination_signal() @@ -380,6 +382,8 @@ static void bro_new_handler() int main(int argc, char** argv) { + std::set_new_handler(bro_new_handler); + brofiler.ReadStats(); bro_argc = argc; diff --git a/src/modp_numtoa.c b/src/modp_numtoa.c index 6deb8a70ed..2024f7c55b 100644 --- a/src/modp_numtoa.c +++ b/src/modp_numtoa.c @@ -56,7 +56,7 @@ void modp_uitoa10(uint32_t value, char* str) void modp_litoa10(int64_t value, char* str) { char* wstr=str; - unsigned long uvalue = (value < 0) ? -value : value; + uint64_t uvalue = (value < 0) ? -value : value; // Conversion. Number is reversed. do *wstr++ = (char)(48 + (uvalue % 10)); while(uvalue /= 10); diff --git a/src/scan.l b/src/scan.l index 645ce659cd..1b3d09f879 100644 --- a/src/scan.l +++ b/src/scan.l @@ -437,9 +437,7 @@ F RET_CONST(new Val(false, TYPE_BOOL)) } {D} { - // TODO: check if we can use strtoull instead of atol, - // and similarly for {HEX}. - RET_CONST(new Val(static_cast(atol(yytext)), + RET_CONST(new Val(static_cast(strtoul(yytext, (char**) NULL, 10)), TYPE_COUNT)) } {FLOAT} RET_CONST(new Val(atof(yytext), TYPE_DOUBLE)) @@ -483,7 +481,7 @@ F RET_CONST(new Val(false, TYPE_BOOL)) ({D}"."){3}{D} RET_CONST(new AddrVal(yytext)) -"0x"{HEX}+ RET_CONST(new Val(static_cast(strtol(yytext, 0, 16)), TYPE_COUNT)) +"0x"{HEX}+ RET_CONST(new Val(static_cast(strtoull(yytext, 0, 16)), TYPE_COUNT)) {H}("."{H})+ RET_CONST(dns_mgr->LookupHost(yytext)) diff --git a/src/util.cc b/src/util.cc index 2d981e952e..3b6fcac76f 100644 --- a/src/util.cc +++ b/src/util.cc @@ -1383,7 +1383,13 @@ void safe_close(int fd) void out_of_memory(const char* where) { - reporter->FatalError("out of memory in %s.\n", where); + fprintf(stderr, "out of memory in %s.\n", where); + + if ( reporter ) + // Guess that might fail here if memory is really tight ... + reporter->FatalError("out of memory in %s.\n", where); + + abort(); } void get_memory_usage(unsigned int* total, unsigned int* malloced) diff --git a/testing/btest/Baseline/analyzers.conn-size-cc/conn.log b/testing/btest/Baseline/analyzers.conn-size-cc/conn.log deleted file mode 100644 index 2f703cbcd6..0000000000 --- a/testing/btest/Baseline/analyzers.conn-size-cc/conn.log +++ /dev/null @@ -1,5 +0,0 @@ -1128727430.350788 ? 141.42.64.125 125.190.109.199 other 56729 12345 tcp ? ? S0 X 1 60 0 0 cc=1 -1144876538.705610 5.921003 169.229.147.203 239.255.255.253 other 49370 427 udp 147 ? S0 X 3 231 0 0 -1144876599.397603 0.815763 192.150.186.169 194.64.249.244 http 53063 80 tcp 377 445 SF X 6 677 5 713 -1144876709.032670 9.000191 169.229.147.43 239.255.255.253 other 49370 427 udp 196 ? S0 X 4 308 0 0 -1144876697.068273 0.000650 192.150.186.169 192.150.186.15 icmp-unreach 3 3 icmp 56 ? OTH X 2 112 0 0 diff --git a/testing/btest/Baseline/analyzers.conn-size/conn.log b/testing/btest/Baseline/analyzers.conn-size/conn.log deleted file mode 100644 index 8129bc37f8..0000000000 --- a/testing/btest/Baseline/analyzers.conn-size/conn.log +++ /dev/null @@ -1,5 +0,0 @@ -1128727430.350788 ? 141.42.64.125 125.190.109.199 other 56729 12345 tcp ? ? S0 X 1 60 0 0 -1144876538.705610 5.921003 169.229.147.203 239.255.255.253 other 49370 427 udp 147 ? S0 X 3 231 0 0 -1144876599.397603 0.815763 192.150.186.169 194.64.249.244 http 53063 80 tcp 377 445 SF X 6 697 5 713 -1144876709.032670 9.000191 169.229.147.43 239.255.255.253 other 49370 427 udp 196 ? S0 X 4 308 0 0 -1144876697.068273 0.000650 192.150.186.169 192.150.186.15 icmp-unreach 3 3 icmp 56 ? OTH X 2 112 0 0 diff --git a/testing/btest/Baseline/bifs.analyzer_name/out b/testing/btest/Baseline/bifs.analyzer_name/out new file mode 100644 index 0000000000..84613e9dd1 --- /dev/null +++ b/testing/btest/Baseline/bifs.analyzer_name/out @@ -0,0 +1 @@ +PIA_TCP diff --git a/testing/btest/Baseline/bifs.capture_state_updates/out b/testing/btest/Baseline/bifs.capture_state_updates/out new file mode 100644 index 0000000000..62a6e3c9df --- /dev/null +++ b/testing/btest/Baseline/bifs.capture_state_updates/out @@ -0,0 +1 @@ +T diff --git a/testing/btest/Baseline/bifs.entropy_test/out b/testing/btest/Baseline/bifs.entropy_test/out new file mode 100644 index 0000000000..08a09de4e4 --- /dev/null +++ b/testing/btest/Baseline/bifs.entropy_test/out @@ -0,0 +1,2 @@ +[entropy=4.715374, chi_square=591.981818, mean=75.472727, monte_carlo_pi=4.0, serial_correlation=-0.11027] +[entropy=2.083189, chi_square=3906.018182, mean=69.054545, monte_carlo_pi=4.0, serial_correlation=0.849402] diff --git a/testing/btest/Baseline/bifs.global_sizes/out b/testing/btest/Baseline/bifs.global_sizes/out new file mode 100644 index 0000000000..76c40b297a --- /dev/null +++ b/testing/btest/Baseline/bifs.global_sizes/out @@ -0,0 +1 @@ +found bro_init diff --git a/testing/btest/Baseline/bifs.identify_data/out b/testing/btest/Baseline/bifs.identify_data/out new file mode 100644 index 0000000000..1cadefbf6e --- /dev/null +++ b/testing/btest/Baseline/bifs.identify_data/out @@ -0,0 +1,4 @@ +ASCII text, with no line terminators +text/plain; charset=us-ascii +PNG image +image/png; charset=binary diff --git a/testing/btest/Baseline/bifs.is_local_interface/out b/testing/btest/Baseline/bifs.is_local_interface/out new file mode 100644 index 0000000000..328bff6687 --- /dev/null +++ b/testing/btest/Baseline/bifs.is_local_interface/out @@ -0,0 +1,4 @@ +T +F +F +T diff --git a/testing/btest/Baseline/bifs.reading_traces/out1 b/testing/btest/Baseline/bifs.reading_traces/out1 new file mode 100644 index 0000000000..cf84443e49 --- /dev/null +++ b/testing/btest/Baseline/bifs.reading_traces/out1 @@ -0,0 +1 @@ +F diff --git a/testing/btest/Baseline/bifs.reading_traces/out2 b/testing/btest/Baseline/bifs.reading_traces/out2 new file mode 100644 index 0000000000..62a6e3c9df --- /dev/null +++ b/testing/btest/Baseline/bifs.reading_traces/out2 @@ -0,0 +1 @@ +T diff --git a/testing/btest/Baseline/bifs.strftime/out b/testing/btest/Baseline/bifs.strftime/out new file mode 100644 index 0000000000..b32393b332 --- /dev/null +++ b/testing/btest/Baseline/bifs.strftime/out @@ -0,0 +1,4 @@ +1970-01-01 00:00:00 +000000 19700101 +1973-11-29 21:33:09 +213309 19731129 diff --git a/testing/btest/Baseline/core.leaks.basic-cluster/manager-1.metrics.log b/testing/btest/Baseline/core.leaks.basic-cluster/manager-1.metrics.log index 42fcd6a526..cb1bd5af01 100644 --- a/testing/btest/Baseline/core.leaks.basic-cluster/manager-1.metrics.log +++ b/testing/btest/Baseline/core.leaks.basic-cluster/manager-1.metrics.log @@ -3,8 +3,10 @@ #empty_field (empty) #unset_field - #path metrics +#open 2012-07-20-01-50-41 #fields ts metric_id filter_name index.host index.str index.network value #types time enum string addr string subnet count -1331256494.591966 TEST_METRIC foo-bar 6.5.4.3 - - 4 -1331256494.591966 TEST_METRIC foo-bar 7.2.1.5 - - 2 -1331256494.591966 TEST_METRIC foo-bar 1.2.3.4 - - 6 +1342749041.601712 TEST_METRIC foo-bar 6.5.4.3 - - 4 +1342749041.601712 TEST_METRIC foo-bar 7.2.1.5 - - 2 +1342749041.601712 TEST_METRIC foo-bar 1.2.3.4 - - 6 +#close 2012-07-20-01-50-49 diff --git a/testing/btest/Baseline/core.leaks.remote/sender.test.failure.log b/testing/btest/Baseline/core.leaks.remote/sender.test.failure.log index 5a26f322f4..71e1d18c73 100644 --- a/testing/btest/Baseline/core.leaks.remote/sender.test.failure.log +++ b/testing/btest/Baseline/core.leaks.remote/sender.test.failure.log @@ -3,8 +3,10 @@ #empty_field (empty) #unset_field - #path test.failure +#open 2012-07-20-01-50-18 #fields t id.orig_h id.orig_p id.resp_h id.resp_p status country #types time addr port addr port string string -1331256472.375609 1.2.3.4 1234 2.3.4.5 80 failure US -1331256472.375609 1.2.3.4 1234 2.3.4.5 80 failure UK -1331256472.375609 1.2.3.4 1234 2.3.4.5 80 failure MX +1342749018.970682 1.2.3.4 1234 2.3.4.5 80 failure US +1342749018.970682 1.2.3.4 1234 2.3.4.5 80 failure UK +1342749018.970682 1.2.3.4 1234 2.3.4.5 80 failure MX +#close 2012-07-20-01-50-18 diff --git a/testing/btest/Baseline/core.leaks.remote/sender.test.log b/testing/btest/Baseline/core.leaks.remote/sender.test.log index 9d2ba26f48..bc3dac5a1a 100644 --- a/testing/btest/Baseline/core.leaks.remote/sender.test.log +++ b/testing/btest/Baseline/core.leaks.remote/sender.test.log @@ -3,10 +3,12 @@ #empty_field (empty) #unset_field - #path test +#open 2012-07-20-01-50-18 #fields t id.orig_h id.orig_p id.resp_h id.resp_p status country #types time addr port addr port string string -1331256472.375609 1.2.3.4 1234 2.3.4.5 80 success unknown -1331256472.375609 1.2.3.4 1234 2.3.4.5 80 failure US -1331256472.375609 1.2.3.4 1234 2.3.4.5 80 failure UK -1331256472.375609 1.2.3.4 1234 2.3.4.5 80 success BR -1331256472.375609 1.2.3.4 1234 2.3.4.5 80 failure MX +1342749018.970682 1.2.3.4 1234 2.3.4.5 80 success unknown +1342749018.970682 1.2.3.4 1234 2.3.4.5 80 failure US +1342749018.970682 1.2.3.4 1234 2.3.4.5 80 failure UK +1342749018.970682 1.2.3.4 1234 2.3.4.5 80 success BR +1342749018.970682 1.2.3.4 1234 2.3.4.5 80 failure MX +#close 2012-07-20-01-50-18 diff --git a/testing/btest/Baseline/core.leaks.remote/sender.test.success.log b/testing/btest/Baseline/core.leaks.remote/sender.test.success.log index 1b2ed452a0..f0b26454b4 100644 --- a/testing/btest/Baseline/core.leaks.remote/sender.test.success.log +++ b/testing/btest/Baseline/core.leaks.remote/sender.test.success.log @@ -3,7 +3,9 @@ #empty_field (empty) #unset_field - #path test.success +#open 2012-07-20-01-50-18 #fields t id.orig_h id.orig_p id.resp_h id.resp_p status country #types time addr port addr port string string -1331256472.375609 1.2.3.4 1234 2.3.4.5 80 success unknown -1331256472.375609 1.2.3.4 1234 2.3.4.5 80 success BR +1342749018.970682 1.2.3.4 1234 2.3.4.5 80 success unknown +1342749018.970682 1.2.3.4 1234 2.3.4.5 80 success BR +#close 2012-07-20-01-50-18 diff --git a/testing/btest/Baseline/core.reporter-error-in-handler/output b/testing/btest/Baseline/core.reporter-error-in-handler/output index 83b310ab61..b20b1b2292 100644 --- a/testing/btest/Baseline/core.reporter-error-in-handler/output +++ b/testing/btest/Baseline/core.reporter-error-in-handler/output @@ -1,2 +1,3 @@ -error in /da/home/robin/bro/master/testing/btest/.tmp/core.reporter-error-in-handler/reporter-error-in-handler.bro, line 22: no such index (a[2]) +error in /home/jsiwek/bro/testing/btest/.tmp/core.reporter-error-in-handler/reporter-error-in-handler.bro, line 22: no such index (a[2]) +ERROR: no such index (a[1]) (/home/jsiwek/bro/testing/btest/.tmp/core.reporter-error-in-handler/reporter-error-in-handler.bro, line 28) 1st error printed on script level diff --git a/testing/btest/Baseline/core.reporter-runtime-error/output b/testing/btest/Baseline/core.reporter-runtime-error/output index 59bcc3ac9b..5a03f5feb2 100644 --- a/testing/btest/Baseline/core.reporter-runtime-error/output +++ b/testing/btest/Baseline/core.reporter-runtime-error/output @@ -1 +1,2 @@ -error in /da/home/robin/bro/master/testing/btest/.tmp/core.reporter-runtime-error/reporter-runtime-error.bro, line 12: no such index (a[1]) +error in /home/jsiwek/bro/testing/btest/.tmp/core.reporter-runtime-error/reporter-runtime-error.bro, line 12: no such index (a[1]) +ERROR: no such index (a[2]) (/home/jsiwek/bro/testing/btest/.tmp/core.reporter-runtime-error/reporter-runtime-error.bro, line 9) diff --git a/testing/btest/Baseline/core.reporter/logger-test.log b/testing/btest/Baseline/core.reporter/logger-test.log index 6f7ba1d8c7..5afd904b63 100644 --- a/testing/btest/Baseline/core.reporter/logger-test.log +++ b/testing/btest/Baseline/core.reporter/logger-test.log @@ -1,6 +1,6 @@ -reporter_info|init test-info|/da/home/robin/bro/master/testing/btest/.tmp/core.reporter/reporter.bro, line 8|0.000000 -reporter_warning|init test-warning|/da/home/robin/bro/master/testing/btest/.tmp/core.reporter/reporter.bro, line 9|0.000000 -reporter_error|init test-error|/da/home/robin/bro/master/testing/btest/.tmp/core.reporter/reporter.bro, line 10|0.000000 -reporter_info|done test-info|/da/home/robin/bro/master/testing/btest/.tmp/core.reporter/reporter.bro, line 15|0.000000 -reporter_warning|done test-warning|/da/home/robin/bro/master/testing/btest/.tmp/core.reporter/reporter.bro, line 16|0.000000 -reporter_error|done test-error|/da/home/robin/bro/master/testing/btest/.tmp/core.reporter/reporter.bro, line 17|0.000000 +reporter_info|init test-info|/home/jsiwek/bro/testing/btest/.tmp/core.reporter/reporter.bro, line 8|0.000000 +reporter_warning|init test-warning|/home/jsiwek/bro/testing/btest/.tmp/core.reporter/reporter.bro, line 9|0.000000 +reporter_error|init test-error|/home/jsiwek/bro/testing/btest/.tmp/core.reporter/reporter.bro, line 10|0.000000 +reporter_info|done test-info|/home/jsiwek/bro/testing/btest/.tmp/core.reporter/reporter.bro, line 15|0.000000 +reporter_warning|done test-warning|/home/jsiwek/bro/testing/btest/.tmp/core.reporter/reporter.bro, line 16|0.000000 +reporter_error|done test-error|/home/jsiwek/bro/testing/btest/.tmp/core.reporter/reporter.bro, line 17|0.000000 diff --git a/testing/btest/Baseline/core.reporter/output b/testing/btest/Baseline/core.reporter/output index 2735adc931..f2c59259c2 100644 --- a/testing/btest/Baseline/core.reporter/output +++ b/testing/btest/Baseline/core.reporter/output @@ -1,3 +1,7 @@ -/da/home/robin/bro/master/testing/btest/.tmp/core.reporter/reporter.bro, line 52: pre test-info -warning in /da/home/robin/bro/master/testing/btest/.tmp/core.reporter/reporter.bro, line 53: pre test-warning -error in /da/home/robin/bro/master/testing/btest/.tmp/core.reporter/reporter.bro, line 54: pre test-error +/home/jsiwek/bro/testing/btest/.tmp/core.reporter/reporter.bro, line 52: pre test-info +warning in /home/jsiwek/bro/testing/btest/.tmp/core.reporter/reporter.bro, line 53: pre test-warning +error in /home/jsiwek/bro/testing/btest/.tmp/core.reporter/reporter.bro, line 54: pre test-error +WARNING: init test-warning (/home/jsiwek/bro/testing/btest/.tmp/core.reporter/reporter.bro, line 9) +ERROR: init test-error (/home/jsiwek/bro/testing/btest/.tmp/core.reporter/reporter.bro, line 10) +WARNING: done test-warning (/home/jsiwek/bro/testing/btest/.tmp/core.reporter/reporter.bro, line 16) +ERROR: done test-error (/home/jsiwek/bro/testing/btest/.tmp/core.reporter/reporter.bro, line 17) diff --git a/testing/btest/Baseline/coverage.bare-mode-errors/unique_errors_no_elasticsearch b/testing/btest/Baseline/coverage.bare-mode-errors/unique_errors_no_elasticsearch new file mode 100644 index 0000000000..e95f88e74b --- /dev/null +++ b/testing/btest/Baseline/coverage.bare-mode-errors/unique_errors_no_elasticsearch @@ -0,0 +1 @@ +error: unknown writer type requested diff --git a/testing/btest/Baseline/scripts.base.frameworks.input.bignumber/out b/testing/btest/Baseline/scripts.base.frameworks.input.bignumber/out new file mode 100644 index 0000000000..8b95ed8b19 --- /dev/null +++ b/testing/btest/Baseline/scripts.base.frameworks.input.bignumber/out @@ -0,0 +1,4 @@ +{ +[9223372036854775800] = [c=18446744073709551612], +[-9223372036854775800] = [c=18446744073709551612] +} diff --git a/testing/btest/Baseline/scripts.base.frameworks.input.empty-values-hashing/out b/testing/btest/Baseline/scripts.base.frameworks.input.empty-values-hashing/out new file mode 100644 index 0000000000..474ef45cc2 --- /dev/null +++ b/testing/btest/Baseline/scripts.base.frameworks.input.empty-values-hashing/out @@ -0,0 +1,155 @@ +============PREDICATE============ +Input::EVENT_NEW +[i=1] +[s=, ss=TEST] +============PREDICATE============ +Input::EVENT_NEW +[i=2] +[s=, ss=] +============EVENT============ +Description +[source=../input.log, reader=Input::READER_ASCII, mode=Input::REREAD, name=ssh, destination={ +[2] = [s=, ss=], +[1] = [s=, ss=TEST] +}, idx=, val=, want_record=T, ev=line +{ +print A::outfile, ============EVENT============; +print A::outfile, Description; +print A::outfile, A::description; +print A::outfile, Type; +print A::outfile, A::tpe; +print A::outfile, Left; +print A::outfile, A::left; +print A::outfile, Right; +print A::outfile, A::right; +}, pred=anonymous-function +{ +print A::outfile, ============PREDICATE============; +print A::outfile, A::typ; +print A::outfile, A::left; +print A::outfile, A::right; +return (T); +}, config={ + +}] +Type +Input::EVENT_NEW +Left +[i=1] +Right +[s=, ss=TEST] +============EVENT============ +Description +[source=../input.log, reader=Input::READER_ASCII, mode=Input::REREAD, name=ssh, destination={ +[2] = [s=, ss=], +[1] = [s=, ss=TEST] +}, idx=, val=, want_record=T, ev=line +{ +print A::outfile, ============EVENT============; +print A::outfile, Description; +print A::outfile, A::description; +print A::outfile, Type; +print A::outfile, A::tpe; +print A::outfile, Left; +print A::outfile, A::left; +print A::outfile, Right; +print A::outfile, A::right; +}, pred=anonymous-function +{ +print A::outfile, ============PREDICATE============; +print A::outfile, A::typ; +print A::outfile, A::left; +print A::outfile, A::right; +return (T); +}, config={ + +}] +Type +Input::EVENT_NEW +Left +[i=2] +Right +[s=, ss=] +==========SERVERS============ +{ +[2] = [s=, ss=], +[1] = [s=, ss=TEST] +} +============PREDICATE============ +Input::EVENT_CHANGED +[i=1] +[s=TEST, ss=] +============PREDICATE============ +Input::EVENT_CHANGED +[i=2] +[s=TEST, ss=TEST] +============EVENT============ +Description +[source=../input.log, reader=Input::READER_ASCII, mode=Input::REREAD, name=ssh, destination={ +[2] = [s=TEST, ss=TEST], +[1] = [s=TEST, ss=] +}, idx=, val=, want_record=T, ev=line +{ +print A::outfile, ============EVENT============; +print A::outfile, Description; +print A::outfile, A::description; +print A::outfile, Type; +print A::outfile, A::tpe; +print A::outfile, Left; +print A::outfile, A::left; +print A::outfile, Right; +print A::outfile, A::right; +}, pred=anonymous-function +{ +print A::outfile, ============PREDICATE============; +print A::outfile, A::typ; +print A::outfile, A::left; +print A::outfile, A::right; +return (T); +}, config={ + +}] +Type +Input::EVENT_CHANGED +Left +[i=1] +Right +[s=, ss=TEST] +============EVENT============ +Description +[source=../input.log, reader=Input::READER_ASCII, mode=Input::REREAD, name=ssh, destination={ +[2] = [s=TEST, ss=TEST], +[1] = [s=TEST, ss=] +}, idx=, val=, want_record=T, ev=line +{ +print A::outfile, ============EVENT============; +print A::outfile, Description; +print A::outfile, A::description; +print A::outfile, Type; +print A::outfile, A::tpe; +print A::outfile, Left; +print A::outfile, A::left; +print A::outfile, Right; +print A::outfile, A::right; +}, pred=anonymous-function +{ +print A::outfile, ============PREDICATE============; +print A::outfile, A::typ; +print A::outfile, A::left; +print A::outfile, A::right; +return (T); +}, config={ + +}] +Type +Input::EVENT_CHANGED +Left +[i=2] +Right +[s=, ss=] +==========SERVERS============ +{ +[2] = [s=TEST, ss=TEST], +[1] = [s=TEST, ss=] +} +done diff --git a/testing/btest/Baseline/scripts.base.frameworks.input.invalidnumbers/.stderrwithoutfirstline b/testing/btest/Baseline/scripts.base.frameworks.input.invalidnumbers/.stderrwithoutfirstline new file mode 100644 index 0000000000..3ef51e40f2 --- /dev/null +++ b/testing/btest/Baseline/scripts.base.frameworks.input.invalidnumbers/.stderrwithoutfirstline @@ -0,0 +1,8 @@ +error: ../input.log/Input::READER_ASCII: Number '12129223372036854775800' out of supported range. +error: ../input.log/Input::READER_ASCII: Could not convert line '12129223372036854775800 121218446744073709551612' to Val. Ignoring line. +warning: ../input.log/Input::READER_ASCII: Number '9223372036854775801TEXTHERE' contained non-numeric trailing characters. Ignored trailing characters 'TEXTHERE' +warning: ../input.log/Input::READER_ASCII: Number '1Justtext' contained non-numeric trailing characters. Ignored trailing characters 'Justtext' +error: ../input.log/Input::READER_ASCII: String 'Justtext' contained no parseable number +error: ../input.log/Input::READER_ASCII: Could not convert line 'Justtext 1' to Val. Ignoring line. +received termination signal +>>> diff --git a/testing/btest/Baseline/scripts.base.frameworks.input.invalidnumbers/out b/testing/btest/Baseline/scripts.base.frameworks.input.invalidnumbers/out new file mode 100644 index 0000000000..56b2736006 --- /dev/null +++ b/testing/btest/Baseline/scripts.base.frameworks.input.invalidnumbers/out @@ -0,0 +1,4 @@ +{ +[9223372036854775800] = [c=4], +[9223372036854775801] = [c=1] +} diff --git a/testing/btest/Baseline/scripts.base.frameworks.input.predicaterefusesecondsamerecord/out b/testing/btest/Baseline/scripts.base.frameworks.input.predicaterefusesecondsamerecord/out new file mode 100644 index 0000000000..f752ff451a --- /dev/null +++ b/testing/btest/Baseline/scripts.base.frameworks.input.predicaterefusesecondsamerecord/out @@ -0,0 +1,3 @@ +{ +[1.228.83.33] = [asn=9318 HANARO-AS Hanaro Telecom Inc., severity=medium, confidence=95, detecttime=1342569600.0] +} diff --git a/testing/btest/Baseline/scripts.base.frameworks.input.reread/out b/testing/btest/Baseline/scripts.base.frameworks.input.reread/out index 8b55ced2ac..538a6dec18 100644 --- a/testing/btest/Baseline/scripts.base.frameworks.input.reread/out +++ b/testing/btest/Baseline/scripts.base.frameworks.input.reread/out @@ -1084,7 +1084,7 @@ BB } ============PREDICATE============ Input::EVENT_REMOVED -[i=-43] +[i=-44] [b=F, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={ 2, 4, @@ -1096,6 +1096,21 @@ AA, BB }, se={ +}, vc=[10, 20, 30], ve=[]] +============PREDICATE============ +Input::EVENT_REMOVED +[i=-42] +[b=T, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={ +2, +4, +1, +3 +}, ss={ +CC, +AA, +BB +}, se={ + }, vc=[10, 20, 30], ve=[]] ============PREDICATE============ Input::EVENT_REMOVED @@ -1111,21 +1126,6 @@ AA, BB }, se={ -}, vc=[10, 20, 30], ve=[]] -============PREDICATE============ -Input::EVENT_REMOVED -[i=-44] -[b=F, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={ -2, -4, -1, -3 -}, ss={ -CC, -AA, -BB -}, se={ - }, vc=[10, 20, 30], ve=[]] ============PREDICATE============ Input::EVENT_REMOVED @@ -1159,25 +1159,7 @@ BB }, vc=[10, 20, 30], ve=[]] ============PREDICATE============ Input::EVENT_REMOVED -[i=-42] -[b=T, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={ -2, -4, -1, -3 -}, ss={ -CC, -AA, -BB -}, se={ - -}, vc=[10, 20, 30], ve=[]] -============EVENT============ -Description -Input::EVENT_REMOVED -Type [i=-43] -Left [b=F, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={ 2, 4, @@ -1190,14 +1172,10 @@ BB }, se={ }, vc=[10, 20, 30], ve=[]] -Right ============EVENT============ Description -Input::EVENT_REMOVED -Type -[i=-46] -Left -[b=F, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={ +[source=../input.log, reader=Input::READER_ASCII, mode=Input::REREAD, name=ssh, destination={ +[-48] = [b=F, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={ 2, 4, 1, @@ -1209,13 +1187,32 @@ BB }, se={ }, vc=[10, 20, 30], ve=[]] -Right -============EVENT============ -Description -Input::EVENT_REMOVED +}, idx=, val=, want_record=T, ev=line +{ +print A::outfile, ============EVENT============; +print A::outfile, Description; +print A::outfile, A::description; +print A::outfile, Type; +print A::outfile, A::tpe; +print A::outfile, Left; +print A::outfile, A::left; +print A::outfile, Right; +print A::outfile, A::right; +}, pred=anonymous-function +{ +print A::outfile, ============PREDICATE============; +print A::outfile, A::typ; +print A::outfile, A::left; +print A::outfile, A::right; +return (T); +}, config={ + +}] Type +Input::EVENT_REMOVED +Left [i=-44] -Left +Right [b=F, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={ 2, 4, @@ -1228,14 +1225,10 @@ BB }, se={ }, vc=[10, 20, 30], ve=[]] -Right ============EVENT============ Description -Input::EVENT_REMOVED -Type -[i=-47] -Left -[b=F, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={ +[source=../input.log, reader=Input::READER_ASCII, mode=Input::REREAD, name=ssh, destination={ +[-48] = [b=F, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={ 2, 4, 1, @@ -1247,32 +1240,32 @@ BB }, se={ }, vc=[10, 20, 30], ve=[]] -Right -============EVENT============ -Description -Input::EVENT_REMOVED -Type -[i=-45] -Left -[b=F, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={ -2, -4, -1, -3 -}, ss={ -CC, -AA, -BB -}, se={ +}, idx=, val=, want_record=T, ev=line +{ +print A::outfile, ============EVENT============; +print A::outfile, Description; +print A::outfile, A::description; +print A::outfile, Type; +print A::outfile, A::tpe; +print A::outfile, Left; +print A::outfile, A::left; +print A::outfile, Right; +print A::outfile, A::right; +}, pred=anonymous-function +{ +print A::outfile, ============PREDICATE============; +print A::outfile, A::typ; +print A::outfile, A::left; +print A::outfile, A::right; +return (T); +}, config={ -}, vc=[10, 20, 30], ve=[]] -Right -============EVENT============ -Description -Input::EVENT_REMOVED +}] Type +Input::EVENT_REMOVED +Left [i=-42] -Left +Right [b=T, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={ 2, 4, @@ -1285,7 +1278,218 @@ BB }, se={ }, vc=[10, 20, 30], ve=[]] +============EVENT============ +Description +[source=../input.log, reader=Input::READER_ASCII, mode=Input::REREAD, name=ssh, destination={ +[-48] = [b=F, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={ +2, +4, +1, +3 +}, ss={ +CC, +AA, +BB +}, se={ + +}, vc=[10, 20, 30], ve=[]] +}, idx=, val=, want_record=T, ev=line +{ +print A::outfile, ============EVENT============; +print A::outfile, Description; +print A::outfile, A::description; +print A::outfile, Type; +print A::outfile, A::tpe; +print A::outfile, Left; +print A::outfile, A::left; +print A::outfile, Right; +print A::outfile, A::right; +}, pred=anonymous-function +{ +print A::outfile, ============PREDICATE============; +print A::outfile, A::typ; +print A::outfile, A::left; +print A::outfile, A::right; +return (T); +}, config={ + +}] +Type +Input::EVENT_REMOVED +Left +[i=-46] Right +[b=F, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={ +2, +4, +1, +3 +}, ss={ +CC, +AA, +BB +}, se={ + +}, vc=[10, 20, 30], ve=[]] +============EVENT============ +Description +[source=../input.log, reader=Input::READER_ASCII, mode=Input::REREAD, name=ssh, destination={ +[-48] = [b=F, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={ +2, +4, +1, +3 +}, ss={ +CC, +AA, +BB +}, se={ + +}, vc=[10, 20, 30], ve=[]] +}, idx=, val=, want_record=T, ev=line +{ +print A::outfile, ============EVENT============; +print A::outfile, Description; +print A::outfile, A::description; +print A::outfile, Type; +print A::outfile, A::tpe; +print A::outfile, Left; +print A::outfile, A::left; +print A::outfile, Right; +print A::outfile, A::right; +}, pred=anonymous-function +{ +print A::outfile, ============PREDICATE============; +print A::outfile, A::typ; +print A::outfile, A::left; +print A::outfile, A::right; +return (T); +}, config={ + +}] +Type +Input::EVENT_REMOVED +Left +[i=-47] +Right +[b=F, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={ +2, +4, +1, +3 +}, ss={ +CC, +AA, +BB +}, se={ + +}, vc=[10, 20, 30], ve=[]] +============EVENT============ +Description +[source=../input.log, reader=Input::READER_ASCII, mode=Input::REREAD, name=ssh, destination={ +[-48] = [b=F, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={ +2, +4, +1, +3 +}, ss={ +CC, +AA, +BB +}, se={ + +}, vc=[10, 20, 30], ve=[]] +}, idx=, val=, want_record=T, ev=line +{ +print A::outfile, ============EVENT============; +print A::outfile, Description; +print A::outfile, A::description; +print A::outfile, Type; +print A::outfile, A::tpe; +print A::outfile, Left; +print A::outfile, A::left; +print A::outfile, Right; +print A::outfile, A::right; +}, pred=anonymous-function +{ +print A::outfile, ============PREDICATE============; +print A::outfile, A::typ; +print A::outfile, A::left; +print A::outfile, A::right; +return (T); +}, config={ + +}] +Type +Input::EVENT_REMOVED +Left +[i=-45] +Right +[b=F, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={ +2, +4, +1, +3 +}, ss={ +CC, +AA, +BB +}, se={ + +}, vc=[10, 20, 30], ve=[]] +============EVENT============ +Description +[source=../input.log, reader=Input::READER_ASCII, mode=Input::REREAD, name=ssh, destination={ +[-48] = [b=F, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={ +2, +4, +1, +3 +}, ss={ +CC, +AA, +BB +}, se={ + +}, vc=[10, 20, 30], ve=[]] +}, idx=, val=, want_record=T, ev=line +{ +print A::outfile, ============EVENT============; +print A::outfile, Description; +print A::outfile, A::description; +print A::outfile, Type; +print A::outfile, A::tpe; +print A::outfile, Left; +print A::outfile, A::left; +print A::outfile, Right; +print A::outfile, A::right; +}, pred=anonymous-function +{ +print A::outfile, ============PREDICATE============; +print A::outfile, A::typ; +print A::outfile, A::left; +print A::outfile, A::right; +return (T); +}, config={ + +}] +Type +Input::EVENT_REMOVED +Left +[i=-43] +Right +[b=F, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={ +2, +4, +1, +3 +}, ss={ +CC, +AA, +BB +}, se={ + +}, vc=[10, 20, 30], ve=[]] ==========SERVERS============ { [-48] = [b=F, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={ diff --git a/testing/btest/Baseline/scripts.base.frameworks.input.set/out b/testing/btest/Baseline/scripts.base.frameworks.input.set/out new file mode 100644 index 0000000000..998244cf3f --- /dev/null +++ b/testing/btest/Baseline/scripts.base.frameworks.input.set/out @@ -0,0 +1,7 @@ +{ +192.168.17.7, +192.168.17.42, +192.168.17.14, +192.168.17.1, +192.168.17.2 +} diff --git a/testing/btest/Baseline/scripts.base.frameworks.input.setseparator/out b/testing/btest/Baseline/scripts.base.frameworks.input.setseparator/out new file mode 100644 index 0000000000..d0e0f53310 --- /dev/null +++ b/testing/btest/Baseline/scripts.base.frameworks.input.setseparator/out @@ -0,0 +1,10 @@ +{ +[1] = [s={ +b, +e, +d, +c, +f, +a +}, ss=[1, 2, 3, 4, 5, 6]] +} diff --git a/testing/btest/Baseline/scripts.base.frameworks.input.setspecialcases/out b/testing/btest/Baseline/scripts.base.frameworks.input.setspecialcases/out new file mode 100644 index 0000000000..62229f7f37 --- /dev/null +++ b/testing/btest/Baseline/scripts.base.frameworks.input.setspecialcases/out @@ -0,0 +1,23 @@ +{ +[2] = [s={ +, +testing +}, s=[testing, , testing]], +[4] = [s={ +, +testing +}, s=[testing, ]], +[6] = [s={ + +}, s=[]], +[1] = [s={ +testing,testing,testing, +}, s=[testing,testing,testing,]], +[5] = [s={ + +}, s=[, , , ]], +[3] = [s={ +, +testing +}, s=[, testing]] +} diff --git a/testing/btest/Baseline/scripts.base.frameworks.input.unsupported_types/out b/testing/btest/Baseline/scripts.base.frameworks.input.unsupported_types/out new file mode 100644 index 0000000000..7ef82cf368 --- /dev/null +++ b/testing/btest/Baseline/scripts.base.frameworks.input.unsupported_types/out @@ -0,0 +1,14 @@ +{ +[-42] = [fi=, b=T, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={ +2, +4, +1, +3 +}, ss={ +CC, +AA, +BB +}, se={ + +}, vc=[10, 20, 30], ve=[]] +} diff --git a/testing/btest/Baseline/scripts.base.frameworks.reporter.disable-stderr/.stderr b/testing/btest/Baseline/scripts.base.frameworks.reporter.disable-stderr/.stderr new file mode 100644 index 0000000000..e69de29bb2 diff --git a/testing/btest/Baseline/scripts.base.frameworks.reporter.disable-stderr/reporter.log b/testing/btest/Baseline/scripts.base.frameworks.reporter.disable-stderr/reporter.log new file mode 100644 index 0000000000..144c094b2f --- /dev/null +++ b/testing/btest/Baseline/scripts.base.frameworks.reporter.disable-stderr/reporter.log @@ -0,0 +1,10 @@ +#separator \x09 +#set_separator , +#empty_field (empty) +#unset_field - +#path reporter +#open 2012-08-10-20-09-16 +#fields ts level message location +#types time enum string string +0.000000 Reporter::ERROR no such index (test[3]) /da/home/robin/bro/master/testing/btest/.tmp/scripts.base.frameworks.reporter.disable-stderr/disable-stderr.bro, line 12 +#close 2012-08-10-20-09-16 diff --git a/testing/btest/Baseline/scripts.base.frameworks.reporter.stderr/.stderr b/testing/btest/Baseline/scripts.base.frameworks.reporter.stderr/.stderr new file mode 100644 index 0000000000..78af1e7a73 --- /dev/null +++ b/testing/btest/Baseline/scripts.base.frameworks.reporter.stderr/.stderr @@ -0,0 +1 @@ +ERROR: no such index (test[3]) (/blah/testing/btest/.tmp/scripts.base.frameworks.reporter.stderr/stderr.bro, line 9) diff --git a/testing/btest/Baseline/scripts.base.frameworks.reporter.stderr/reporter.log b/testing/btest/Baseline/scripts.base.frameworks.reporter.stderr/reporter.log new file mode 100644 index 0000000000..b314bc45c3 --- /dev/null +++ b/testing/btest/Baseline/scripts.base.frameworks.reporter.stderr/reporter.log @@ -0,0 +1,10 @@ +#separator \x09 +#set_separator , +#empty_field (empty) +#unset_field - +#path reporter +#open 2012-08-10-20-09-23 +#fields ts level message location +#types time enum string string +0.000000 Reporter::ERROR no such index (test[3]) /da/home/robin/bro/master/testing/btest/.tmp/scripts.base.frameworks.reporter.stderr/stderr.bro, line 9 +#close 2012-08-10-20-09-23 diff --git a/testing/btest/Baseline/signatures.bad-eval-condition/.stderr b/testing/btest/Baseline/signatures.bad-eval-condition/.stderr new file mode 100644 index 0000000000..c4de35ffe9 --- /dev/null +++ b/testing/btest/Baseline/signatures.bad-eval-condition/.stderr @@ -0,0 +1,2 @@ +error: Error in signature (./blah.sig:6): eval function parameters must be a 'signature_state' and a 'string' type (mark_conn) + diff --git a/testing/btest/Baseline/signatures.eval-condition/conn.log b/testing/btest/Baseline/signatures.eval-condition/conn.log new file mode 100644 index 0000000000..a803f74320 --- /dev/null +++ b/testing/btest/Baseline/signatures.eval-condition/conn.log @@ -0,0 +1,14 @@ +#separator \x09 +#set_separator , +#empty_field (empty) +#unset_field - +#path conn +#open 2012-08-23-16-41-23 +#fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p proto service duration orig_bytes resp_bytes conn_state local_orig missed_bytes history orig_pkts orig_ip_bytes resp_pkts resp_ip_bytes tunnel_parents +#types time string addr port addr port enum string interval count count string bool count string count count count count table[string] +1329843175.736107 arKYeMETxOg 141.142.220.235 37604 199.233.217.249 56666 tcp ftp-data 0.112432 0 342 SF - 0 ShAdfFa 4 216 4 562 (empty) +1329843179.871641 k6kgXLOoSKl 141.142.220.235 59378 199.233.217.249 56667 tcp ftp-data 0.111218 0 77 SF - 0 ShAdfFa 4 216 4 297 (empty) +1329843194.151526 nQcgTWjvg4c 199.233.217.249 61920 141.142.220.235 33582 tcp ftp-data 0.056211 342 0 SF - 0 ShADaFf 5 614 3 164 (empty) +1329843197.783443 j4u32Pc5bif 199.233.217.249 61918 141.142.220.235 37835 tcp ftp-data 0.056005 77 0 SF - 0 ShADaFf 5 349 3 164 (empty) +1329843161.968492 UWkUyAuUGXf 141.142.220.235 50003 199.233.217.249 21 tcp ftp,blah 38.055625 180 3146 SF - 0 ShAdDfFa 38 2164 25 4458 (empty) +#close 2012-08-23-16-41-23 diff --git a/testing/btest/Baseline/core.load-sigs/output b/testing/btest/Baseline/signatures.load-sigs/output similarity index 100% rename from testing/btest/Baseline/core.load-sigs/output rename to testing/btest/Baseline/signatures.load-sigs/output diff --git a/testing/btest/analyzers/conn-size-cc.bro b/testing/btest/analyzers/conn-size-cc.bro deleted file mode 100644 index 0ba7977cf5..0000000000 --- a/testing/btest/analyzers/conn-size-cc.bro +++ /dev/null @@ -1,2 +0,0 @@ -# @TEST-EXEC: bro -C -r ${TRACES}/conn-size.trace tcp udp icmp report_conn_size_analyzer=T -# @TEST-EXEC: btest-diff conn.log diff --git a/testing/btest/analyzers/conn-size.bro b/testing/btest/analyzers/conn-size.bro deleted file mode 100644 index 0ba7977cf5..0000000000 --- a/testing/btest/analyzers/conn-size.bro +++ /dev/null @@ -1,2 +0,0 @@ -# @TEST-EXEC: bro -C -r ${TRACES}/conn-size.trace tcp udp icmp report_conn_size_analyzer=T -# @TEST-EXEC: btest-diff conn.log diff --git a/testing/btest/bifs/analyzer_name.bro b/testing/btest/bifs/analyzer_name.bro new file mode 100644 index 0000000000..034344f5c4 --- /dev/null +++ b/testing/btest/bifs/analyzer_name.bro @@ -0,0 +1,9 @@ +# +# @TEST-EXEC: bro %INPUT >out +# @TEST-EXEC: btest-diff out + +event bro_init() + { + local a = 1; + print analyzer_name(a); + } diff --git a/testing/btest/bifs/bro_version.bro b/testing/btest/bifs/bro_version.bro new file mode 100644 index 0000000000..7465cbc0f5 --- /dev/null +++ b/testing/btest/bifs/bro_version.bro @@ -0,0 +1,9 @@ +# +# @TEST-EXEC: bro %INPUT + +event bro_init() + { + local a = bro_version(); + if ( |a| == 0 ) + exit(1); + } diff --git a/testing/btest/bifs/capture_state_updates.bro b/testing/btest/bifs/capture_state_updates.bro new file mode 100644 index 0000000000..3abfdffdc1 --- /dev/null +++ b/testing/btest/bifs/capture_state_updates.bro @@ -0,0 +1,9 @@ +# +# @TEST-EXEC: bro %INPUT >out +# @TEST-EXEC: btest-diff out +# @TEST-EXEC: test -f testfile + +event bro_init() + { + print capture_state_updates("testfile"); + } diff --git a/testing/btest/bifs/checkpoint_state.bro b/testing/btest/bifs/checkpoint_state.bro new file mode 100644 index 0000000000..2a66bd1729 --- /dev/null +++ b/testing/btest/bifs/checkpoint_state.bro @@ -0,0 +1,10 @@ +# +# @TEST-EXEC: bro %INPUT +# @TEST-EXEC: test -f .state/state.bst + +event bro_init() + { + local a = checkpoint_state(); + if ( a != T ) + exit(1); + } diff --git a/testing/btest/bifs/current_analyzer.bro b/testing/btest/bifs/current_analyzer.bro new file mode 100644 index 0000000000..45b495c046 --- /dev/null +++ b/testing/btest/bifs/current_analyzer.bro @@ -0,0 +1,11 @@ +# +# @TEST-EXEC: bro %INPUT + +event bro_init() + { + local a = current_analyzer(); + if ( a != 0 ) + exit(1); + + # TODO: add a test for non-zero return value + } diff --git a/testing/btest/bifs/current_time.bro b/testing/btest/bifs/current_time.bro new file mode 100644 index 0000000000..5d16df396d --- /dev/null +++ b/testing/btest/bifs/current_time.bro @@ -0,0 +1,9 @@ +# +# @TEST-EXEC: bro %INPUT + +event bro_init() + { + local a = current_time(); + if ( a <= double_to_time(0) ) + exit(1); + } diff --git a/testing/btest/bifs/entropy_test.bro b/testing/btest/bifs/entropy_test.bro new file mode 100644 index 0000000000..ca01c79ed7 --- /dev/null +++ b/testing/btest/bifs/entropy_test.bro @@ -0,0 +1,24 @@ +# +# @TEST-EXEC: bro %INPUT >out +# @TEST-EXEC: btest-diff out + +event bro_init() + { + local a = "dh3Hie02uh^s#Sdf9L3frd243h$d78r2G4cM6*Q05d(7rh46f!0|4-f"; + if ( entropy_test_init(1) != T ) + exit(1); + + if ( entropy_test_add(1, a) != T ) + exit(1); + + print entropy_test_finish(1); + + local b = "0011000aaabbbbcccc000011111000000000aaaabbbbcccc0000000"; + if ( entropy_test_init(2) != T ) + exit(1); + + if ( entropy_test_add(2, b) != T ) + exit(1); + + print entropy_test_finish(2); + } diff --git a/testing/btest/bifs/get_matcher_stats.bro b/testing/btest/bifs/get_matcher_stats.bro new file mode 100644 index 0000000000..baee49fe1e --- /dev/null +++ b/testing/btest/bifs/get_matcher_stats.bro @@ -0,0 +1,9 @@ +# +# @TEST-EXEC: bro %INPUT + +event bro_init() + { + local a = get_matcher_stats(); + if ( a$matchers == 0 ) + exit(1); + } diff --git a/testing/btest/bifs/gethostname.bro b/testing/btest/bifs/gethostname.bro new file mode 100644 index 0000000000..97af719745 --- /dev/null +++ b/testing/btest/bifs/gethostname.bro @@ -0,0 +1,9 @@ +# +# @TEST-EXEC: bro %INPUT + +event bro_init() + { + local a = gethostname(); + if ( |a| == 0 ) + exit(1); + } diff --git a/testing/btest/bifs/getpid.bro b/testing/btest/bifs/getpid.bro new file mode 100644 index 0000000000..98edc19a44 --- /dev/null +++ b/testing/btest/bifs/getpid.bro @@ -0,0 +1,9 @@ +# +# @TEST-EXEC: bro %INPUT + +event bro_init() + { + local a = getpid(); + if ( a == 0 ) + exit(1); + } diff --git a/testing/btest/bifs/global_sizes.bro b/testing/btest/bifs/global_sizes.bro new file mode 100644 index 0000000000..4862db318b --- /dev/null +++ b/testing/btest/bifs/global_sizes.bro @@ -0,0 +1,16 @@ +# +# @TEST-EXEC: bro %INPUT >out +# @TEST-EXEC: btest-diff out + +event bro_init() + { + local a = global_sizes(); + for ( i in a ) + { + # the table is quite large, so just look for one item we expect + if ( i == "bro_init" ) + print "found bro_init"; + + } + + } diff --git a/testing/btest/bifs/identify_data.bro b/testing/btest/bifs/identify_data.bro new file mode 100644 index 0000000000..39f289d40b --- /dev/null +++ b/testing/btest/bifs/identify_data.bro @@ -0,0 +1,16 @@ +# +# @TEST-EXEC: bro %INPUT | sed 's/PNG image data/PNG image/g' >out +# @TEST-EXEC: btest-diff out + +event bro_init() + { + # plain text + local a = "This is a test"; + print identify_data(a, F); + print identify_data(a, T); + + # PNG image + local b = "\x89\x50\x4e\x47\x0d\x0a\x1a\x0a"; + print identify_data(b, F); + print identify_data(b, T); + } diff --git a/testing/btest/bifs/is_local_interface.bro b/testing/btest/bifs/is_local_interface.bro new file mode 100644 index 0000000000..8befdca385 --- /dev/null +++ b/testing/btest/bifs/is_local_interface.bro @@ -0,0 +1,11 @@ +# +# @TEST-EXEC: bro %INPUT >out +# @TEST-EXEC: btest-diff out + +event bro_init() + { + print is_local_interface(127.0.0.1); + print is_local_interface(1.2.3.4); + print is_local_interface([2607::a:b:c:d]); + print is_local_interface([::1]); + } diff --git a/testing/btest/bifs/reading_traces.bro b/testing/btest/bifs/reading_traces.bro new file mode 100644 index 0000000000..fc83c50ccb --- /dev/null +++ b/testing/btest/bifs/reading_traces.bro @@ -0,0 +1,10 @@ + +# @TEST-EXEC: bro %INPUT >out1 +# @TEST-EXEC: btest-diff out1 +# @TEST-EXEC: bro -r $TRACES/web.trace %INPUT >out2 +# @TEST-EXEC: btest-diff out2 + +event bro_init() + { + print reading_traces(); + } diff --git a/testing/btest/bifs/resource_usage.bro b/testing/btest/bifs/resource_usage.bro new file mode 100644 index 0000000000..35f5b020d6 --- /dev/null +++ b/testing/btest/bifs/resource_usage.bro @@ -0,0 +1,9 @@ +# +# @TEST-EXEC: bro %INPUT + +event bro_init() + { + local a = resource_usage(); + if ( a$version != bro_version() ) + exit(1); + } diff --git a/testing/btest/bifs/strftime.bro b/testing/btest/bifs/strftime.bro new file mode 100644 index 0000000000..31f9538632 --- /dev/null +++ b/testing/btest/bifs/strftime.bro @@ -0,0 +1,17 @@ +# +# @TEST-EXEC: bro %INPUT >out +# @TEST-EXEC: btest-diff out + +event bro_init() + { + local f1 = "%Y-%m-%d %H:%M:%S"; + local f2 = "%H%M%S %Y%m%d"; + + local a = double_to_time(0); + print strftime(f1, a); + print strftime(f2, a); + + a = double_to_time(123456789); + print strftime(f1, a); + print strftime(f2, a); + } diff --git a/testing/btest/btest.cfg b/testing/btest/btest.cfg index 4c4074ee24..d86b45d8a9 100644 --- a/testing/btest/btest.cfg +++ b/testing/btest/btest.cfg @@ -1,5 +1,5 @@ [btest] -TestDirs = doc bifs language core scripts istate coverage +TestDirs = doc bifs language core scripts istate coverage signatures TmpDir = %(testbase)s/.tmp BaselineDir = %(testbase)s/Baseline IgnoreDirs = .svn CVS .tmp diff --git a/testing/btest/core/leaks/basic-cluster.bro b/testing/btest/core/leaks/basic-cluster.bro index f5b40c1104..319368bc6e 100644 --- a/testing/btest/core/leaks/basic-cluster.bro +++ b/testing/btest/core/leaks/basic-cluster.bro @@ -1,21 +1,22 @@ # Needs perftools support. # +# @TEST-SERIALIZE: comm # @TEST-GROUP: leaks - +# # @TEST-REQUIRES: bro --help 2>&1 | grep -q mem-leaks - +# # @TEST-EXEC: btest-bg-run manager-1 HEAP_CHECK_DUMP_DIRECTORY=. HEAPCHECK=local BROPATH=$BROPATH:.. CLUSTER_NODE=manager-1 bro -m %INPUT # @TEST-EXEC: btest-bg-run proxy-1 HEAP_CHECK_DUMP_DIRECTORY=. HEAPCHECK=local BROPATH=$BROPATH:.. CLUSTER_NODE=proxy-1 bro -m %INPUT # @TEST-EXEC: sleep 1 # @TEST-EXEC: btest-bg-run worker-1 HEAP_CHECK_DUMP_DIRECTORY=. HEAPCHECK=local BROPATH=$BROPATH:.. CLUSTER_NODE=worker-1 bro -m -r $TRACES/web.trace --pseudo-realtime %INPUT # @TEST-EXEC: btest-bg-run worker-2 HEAP_CHECK_DUMP_DIRECTORY=. HEAPCHECK=local BROPATH=$BROPATH:.. CLUSTER_NODE=worker-2 bro -m -r $TRACES/web.trace --pseudo-realtime %INPUT -# @TEST-EXEC: btest-bg-wait -k 30 +# @TEST-EXEC: btest-bg-wait 60 # @TEST-EXEC: btest-diff manager-1/metrics.log @TEST-START-FILE cluster-layout.bro redef Cluster::nodes = { - ["manager-1"] = [$node_type=Cluster::MANAGER, $ip=127.0.0.1, $p=37757/tcp, $workers=set("worker-1")], - ["proxy-1"] = [$node_type=Cluster::PROXY, $ip=127.0.0.1, $p=37758/tcp, $manager="manager-1", $workers=set("worker-1")], + ["manager-1"] = [$node_type=Cluster::MANAGER, $ip=127.0.0.1, $p=37757/tcp, $workers=set("worker-1", "worker-2")], + ["proxy-1"] = [$node_type=Cluster::PROXY, $ip=127.0.0.1, $p=37758/tcp, $manager="manager-1", $workers=set("worker-1", "worker-2")], ["worker-1"] = [$node_type=Cluster::WORKER, $ip=127.0.0.1, $p=37760/tcp, $manager="manager-1", $proxy="proxy-1", $interface="eth0"], ["worker-2"] = [$node_type=Cluster::WORKER, $ip=127.0.0.1, $p=37761/tcp, $manager="manager-1", $proxy="proxy-1", $interface="eth1"], }; @@ -32,11 +33,51 @@ event bro_init() &priority=5 Metrics::add_filter(TEST_METRIC, [$name="foo-bar", $break_interval=3secs]); - - if ( Cluster::local_node_type() == Cluster::WORKER ) + } + +event remote_connection_closed(p: event_peer) + { + terminate(); + } + +global ready_for_data: event(); + +redef Cluster::manager2worker_events += /ready_for_data/; + +@if ( Cluster::local_node_type() == Cluster::WORKER ) + +event ready_for_data() + { + Metrics::add_data(TEST_METRIC, [$host=1.2.3.4], 3); + Metrics::add_data(TEST_METRIC, [$host=6.5.4.3], 2); + Metrics::add_data(TEST_METRIC, [$host=7.2.1.5], 1); + } + +@endif + +@if ( Cluster::local_node_type() == Cluster::MANAGER ) + +global n = 0; +global peer_count = 0; + +event Metrics::log_metrics(rec: Metrics::Info) + { + n = n + 1; + if ( n == 3 ) { - Metrics::add_data(TEST_METRIC, [$host=1.2.3.4], 3); - Metrics::add_data(TEST_METRIC, [$host=6.5.4.3], 2); - Metrics::add_data(TEST_METRIC, [$host=7.2.1.5], 1); + terminate_communication(); + terminate(); } } + +event remote_connection_handshake_done(p: event_peer) + { + print p; + peer_count = peer_count + 1; + if ( peer_count == 3 ) + { + event ready_for_data(); + } + } + +@endif diff --git a/testing/btest/core/leaks/remote.bro b/testing/btest/core/leaks/remote.bro index f888d8f6ee..41bbaec076 100644 --- a/testing/btest/core/leaks/remote.bro +++ b/testing/btest/core/leaks/remote.bro @@ -1,20 +1,23 @@ # Needs perftools support. # +# @TEST-SERIALIZE: comm # @TEST-GROUP: leaks # # @TEST-REQUIRES: bro --help 2>&1 | grep -q mem-leaks # -# @TEST-EXEC: btest-bg-run sender HEAP_CHECK_DUMP_DIRECTORY=. HEAPCHECK=local bro -m --pseudo-realtime %INPUT ../sender.bro +# @TEST-EXEC: btest-bg-run sender HEAP_CHECK_DUMP_DIRECTORY=. HEAPCHECK=local bro -b -m --pseudo-realtime %INPUT ../sender.bro # @TEST-EXEC: sleep 1 -# @TEST-EXEC: btest-bg-run receiver HEAP_CHECK_DUMP_DIRECTORY=. HEAPCHECK=local bro -m --pseudo-realtime %INPUT ../receiver.bro +# @TEST-EXEC: btest-bg-run receiver HEAP_CHECK_DUMP_DIRECTORY=. HEAPCHECK=local bro -b -m --pseudo-realtime %INPUT ../receiver.bro # @TEST-EXEC: sleep 1 -# @TEST-EXEC: btest-bg-wait -k 10 +# @TEST-EXEC: btest-bg-wait 30 # @TEST-EXEC: btest-diff sender/test.log # @TEST-EXEC: btest-diff sender/test.failure.log # @TEST-EXEC: btest-diff sender/test.success.log -# @TEST-EXEC: cmp receiver/test.log sender/test.log -# @TEST-EXEC: cmp receiver/test.failure.log sender/test.failure.log -# @TEST-EXEC: cmp receiver/test.success.log sender/test.success.log +# @TEST-EXEC: ( cd sender && for i in *.log; do cat $i | $SCRIPTS/diff-remove-timestamps >c.$i; done ) +# @TEST-EXEC: ( cd receiver && for i in *.log; do cat $i | $SCRIPTS/diff-remove-timestamps >c.$i; done ) +# @TEST-EXEC: cmp receiver/c.test.log sender/c.test.log +# @TEST-EXEC: cmp receiver/c.test.failure.log sender/c.test.failure.log +# @TEST-EXEC: cmp receiver/c.test.success.log sender/c.test.success.log # This is the common part loaded by both sender and receiver. module Test; @@ -43,10 +46,10 @@ event bro_init() @TEST-START-FILE sender.bro -module Test; - @load frameworks/communication/listen +module Test; + function fail(rec: Log): bool { return rec$status != "success"; @@ -68,14 +71,27 @@ event remote_connection_handshake_done(p: event_peer) Log::write(Test::LOG, [$t=network_time(), $id=cid, $status="failure", $country="MX"]); disconnect(p); } + +event remote_connection_closed(p: event_peer) + { + terminate(); + } + @TEST-END-FILE @TEST-START-FILE receiver.bro ##### +@load base/frameworks/communication + redef Communication::nodes += { ["foo"] = [$host = 127.0.0.1, $connect=T, $request_logs=T] }; +event remote_connection_closed(p: event_peer) + { + terminate(); + } + @TEST-END-FILE diff --git a/testing/btest/coverage/bare-mode-errors.test b/testing/btest/coverage/bare-mode-errors.test index 21e7d4f4a9..635726841b 100644 --- a/testing/btest/coverage/bare-mode-errors.test +++ b/testing/btest/coverage/bare-mode-errors.test @@ -8,6 +8,7 @@ # @TEST-SERIALIZE: comm # # @TEST-EXEC: test -d $DIST/scripts -# @TEST-EXEC: for script in `find $DIST/scripts -name \*\.bro -not -path '*/site/*'`; do echo $script; if echo "$script" | egrep -q 'communication/listen|controllee'; then rm -rf load_attempt .bgprocs; btest-bg-run load_attempt bro -b $script; btest-bg-wait -k 2; cat load_attempt/.stderr >>allerrors; else bro -b $script 2>>allerrors; fi done || exit 0 +# @TEST-EXEC: for script in `find $DIST/scripts/ -name \*\.bro -not -path '*/site/*'`; do echo $script; if echo "$script" | egrep -q 'communication/listen|controllee'; then rm -rf load_attempt .bgprocs; btest-bg-run load_attempt bro -b $script; btest-bg-wait -k 2; cat load_attempt/.stderr >>allerrors; else bro -b $script 2>>allerrors; fi done || exit 0 # @TEST-EXEC: cat allerrors | grep -v "received termination signal" | sort | uniq > unique_errors -# @TEST-EXEC: btest-diff unique_errors +# @TEST-EXEC: if [ $(grep -c CURL_INCLUDE_DIR-NOTFOUND $BUILD/CMakeCache.txt) -ne 0 ]; then cp unique_errors unique_errors_no_elasticsearch; fi +# @TEST-EXEC: if [ $(grep -c CURL_INCLUDE_DIR-NOTFOUND $BUILD/CMakeCache.txt) -ne 0 ]; then btest-diff unique_errors_no_elasticsearch; else btest-diff unique_errors; fi diff --git a/testing/btest/scripts/base/frameworks/input/bignumber.bro b/testing/btest/scripts/base/frameworks/input/bignumber.bro new file mode 100644 index 0000000000..250f84bbb2 --- /dev/null +++ b/testing/btest/scripts/base/frameworks/input/bignumber.bro @@ -0,0 +1,45 @@ +# (uses listen.bro just to ensure input sources are more reliably fully-read). +# @TEST-SERIALIZE: comm +# +# @TEST-EXEC: btest-bg-run bro bro -b %INPUT +# @TEST-EXEC: btest-bg-wait -k 5 +# @TEST-EXEC: btest-diff out + +@TEST-START-FILE input.log +#separator \x09 +#fields i c +#types int count +9223372036854775800 18446744073709551612 +-9223372036854775800 18446744073709551612 +@TEST-END-FILE + +@load frameworks/communication/listen + +global outfile: file; + +module A; + +type Idx: record { + i: int; +}; + +type Val: record { + c: count; +}; + +global servers: table[int] of Val = table(); + +event bro_init() + { + outfile = open("../out"); + # first read in the old stuff into the table... + Input::add_table([$source="../input.log", $name="ssh", $idx=Idx, $val=Val, $destination=servers]); + Input::remove("ssh"); + } + +event Input::update_finished(name: string, source:string) + { + print outfile, servers; + close(outfile); + terminate(); + } diff --git a/testing/btest/scripts/base/frameworks/input/empty-values-hashing.bro b/testing/btest/scripts/base/frameworks/input/empty-values-hashing.bro new file mode 100644 index 0000000000..b66febba82 --- /dev/null +++ b/testing/btest/scripts/base/frameworks/input/empty-values-hashing.bro @@ -0,0 +1,89 @@ +# (uses listen.bro just to ensure input sources are more reliably fully-read). +# @TEST-SERIALIZE: comm +# +# @TEST-EXEC: cp input1.log input.log +# @TEST-EXEC: btest-bg-run bro bro -b %INPUT +# @TEST-EXEC: sleep 2 +# @TEST-EXEC: cp input2.log input.log +# @TEST-EXEC: btest-bg-wait -k 5 +# @TEST-EXEC: btest-diff out + +@TEST-START-FILE input1.log +#separator \x09 +#fields i s ss +#types int sting string +1 - TEST +2 - - +@TEST-END-FILE +@TEST-START-FILE input2.log +#separator \x09 +#fields i s ss +#types int sting string +1 TEST - +2 TEST TEST +@TEST-END-FILE + +@load frameworks/communication/listen + + +module A; + +type Idx: record { + i: int; +}; + +type Val: record { + s: string; + ss: string; +}; + +global servers: table[int] of Val = table(); + +global outfile: file; + +global try: count; + +event line(description: Input::TableDescription, tpe: Input::Event, left: Idx, right: Val) + { + print outfile, "============EVENT============"; + print outfile, "Description"; + print outfile, description; + print outfile, "Type"; + print outfile, tpe; + print outfile, "Left"; + print outfile, left; + print outfile, "Right"; + print outfile, right; + } + +event bro_init() + { + outfile = open("../out"); + try = 0; + # first read in the old stuff into the table... + Input::add_table([$source="../input.log", $mode=Input::REREAD, $name="ssh", $idx=Idx, $val=Val, $destination=servers, $ev=line, + $pred(typ: Input::Event, left: Idx, right: Val) = { + print outfile, "============PREDICATE============"; + print outfile, typ; + print outfile, left; + print outfile, right; + return T; + } + ]); + } + + +event Input::update_finished(name: string, source: string) + { + print outfile, "==========SERVERS============"; + print outfile, servers; + + try = try + 1; + if ( try == 2 ) + { + print outfile, "done"; + close(outfile); + Input::remove("input"); + terminate(); + } + } diff --git a/testing/btest/scripts/base/frameworks/input/invalidnumbers.bro b/testing/btest/scripts/base/frameworks/input/invalidnumbers.bro new file mode 100644 index 0000000000..3c755f1d08 --- /dev/null +++ b/testing/btest/scripts/base/frameworks/input/invalidnumbers.bro @@ -0,0 +1,48 @@ +# (uses listen.bro just to ensure input sources are more reliably fully-read). +# @TEST-SERIALIZE: comm +# +# @TEST-EXEC: btest-bg-run bro bro -b %INPUT +# @TEST-EXEC: btest-bg-wait -k 5 +# @TEST-EXEC: btest-diff out +# @TEST-EXEC: sed 1d .stderr > .stderrwithoutfirstline +# @TEST-EXEC: TEST_DIFF_CANONIFIER=$SCRIPTS/diff-remove-abspath btest-diff .stderrwithoutfirstline + +@TEST-START-FILE input.log +#separator \x09 +#fields i c +#types int count +12129223372036854775800 121218446744073709551612 +9223372036854775801TEXTHERE 1Justtext +Justtext 1 +9223372036854775800 -18446744073709551612 +@TEST-END-FILE + +@load frameworks/communication/listen + +global outfile: file; + +module A; + +type Idx: record { + i: int; +}; + +type Val: record { + c: count; +}; + +global servers: table[int] of Val = table(); + +event bro_init() + { + outfile = open("../out"); + # first read in the old stuff into the table... + Input::add_table([$source="../input.log", $name="ssh", $idx=Idx, $val=Val, $destination=servers]); + Input::remove("ssh"); + } + +event Input::update_finished(name: string, source:string) + { + print outfile, servers; + terminate(); + } diff --git a/testing/btest/scripts/base/frameworks/input/predicate.bro b/testing/btest/scripts/base/frameworks/input/predicate.bro index 2cda6f5fb9..fcd986c9a6 100644 --- a/testing/btest/scripts/base/frameworks/input/predicate.bro +++ b/testing/btest/scripts/base/frameworks/input/predicate.bro @@ -35,7 +35,7 @@ type Val: record { b: bool; }; -global servers: table[int] of Val = table(); +global servers: table[int] of bool = table(); event bro_init() { diff --git a/testing/btest/scripts/base/frameworks/input/predicaterefusesecondsamerecord.bro b/testing/btest/scripts/base/frameworks/input/predicaterefusesecondsamerecord.bro new file mode 100644 index 0000000000..d572b30090 --- /dev/null +++ b/testing/btest/scripts/base/frameworks/input/predicaterefusesecondsamerecord.bro @@ -0,0 +1,56 @@ +# (uses listen.bro just to ensure input sources are more reliably fully-read). +# @TEST-SERIALIZE: comm +# +# @TEST-EXEC: btest-bg-run bro bro -b %INPUT +# @TEST-EXEC: btest-bg-wait -k 5 +# @TEST-EXEC: btest-diff out + +# Ok, this one tests a fun case. +# Input file contains two lines mapping to the same index, but with different values, +# where the predicate accepts the first one and refuses the second one. +# Desired result -> first entry stays. + +@TEST-START-FILE input.log +#fields restriction guid severity confidence detecttime address protocol portlist asn prefix rir cc impact description alternativeid_restriction alternativeid +need-to-know 8c864306-d21a-37b1-8705-746a786719bf medium 65 1342656000 1.0.17.227 - - 2519 VECTANT VECTANT Ltd. 1.0.16.0/23 apnic JP spam infrastructure spamming public http://reputation.alienvault.com/reputation.generic +need-to-know 8c864306-d21a-37b1-8705-746a786719bf medium 95 1342569600 1.228.83.33 6 25 9318 HANARO-AS Hanaro Telecom Inc. 1.224.0.0/13 apnic KR spam infrastructure direct ube sources, spam operations & spam services public http://www.spamhaus.org/query/bl?ip=1.228.83.33 +need-to-know 8c864306-d21a-37b1-8705-746a786719bf medium 65 1342656000 1.228.83.33 - - 9318 HANARO-AS Hanaro Telecom Inc. 1.224.0.0/13 apnic KR spam infrastructure spamming;malware domain public http://reputation.alienvault.com/reputation.generic +@TEST-END-FILE + +@load frameworks/communication/listen + +global outfile: file; + +redef InputAscii::empty_field = "EMPTY"; + +module A; + +type Idx: record { + address: addr; +}; + +type Val: record { + asn: string; + severity: string; + confidence: count; + detecttime: time; +}; + +global servers: table[addr] of Val = table(); + +event bro_init() + { + outfile = open("../out"); + # first read in the old stuff into the table... + Input::add_table([$source="../input.log", $name="input", $idx=Idx, $val=Val, $destination=servers, + $pred(typ: Input::Event, left: Idx, right: Val) = { if ( right$confidence > 90 ) { return T; } return F; } + ]); + Input::remove("input"); + } + +event Input::update_finished(name: string, source: string) + { + print outfile, servers; + close(outfile); + terminate(); + } diff --git a/testing/btest/scripts/base/frameworks/input/set.bro b/testing/btest/scripts/base/frameworks/input/set.bro new file mode 100644 index 0000000000..5215523ee3 --- /dev/null +++ b/testing/btest/scripts/base/frameworks/input/set.bro @@ -0,0 +1,46 @@ +# (uses listen.bro just to ensure input sources are more reliably fully-read). +# @TEST-SERIALIZE: comm +# +# @TEST-EXEC: btest-bg-run bro bro -b %INPUT +# @TEST-EXEC: btest-bg-wait -k 5 +# @TEST-EXEC: TEST_DIFF_CANONIFIER=$SCRIPTS/diff-sort btest-diff out + +@TEST-START-FILE input.log +#separator \x09 +#fields ip +#types addr +192.168.17.1 +192.168.17.2 +192.168.17.7 +192.168.17.14 +192.168.17.42 +@TEST-END-FILE + +@load frameworks/communication/listen + +global outfile: file; + +redef InputAscii::empty_field = "EMPTY"; + +module A; + +type Idx: record { + ip: addr; +}; + +global servers: set[addr] = set(); + +event bro_init() + { + outfile = open("../out"); + # first read in the old stuff into the table... + Input::add_table([$source="../input.log", $name="ssh", $idx=Idx, $destination=servers]); + Input::remove("ssh"); + } + +event Input::update_finished(name: string, source:string) + { + print outfile, servers; + close(outfile); + terminate(); + } diff --git a/testing/btest/scripts/base/frameworks/input/setseparator.bro b/testing/btest/scripts/base/frameworks/input/setseparator.bro new file mode 100644 index 0000000000..44b9d08d54 --- /dev/null +++ b/testing/btest/scripts/base/frameworks/input/setseparator.bro @@ -0,0 +1,46 @@ +# (uses listen.bro just to ensure input sources are more reliably fully-read). +# @TEST-SERIALIZE: comm +# +# @TEST-EXEC: btest-bg-run bro bro -b %INPUT +# @TEST-EXEC: btest-bg-wait -k 5 +# @TEST-EXEC: TEST_DIFF_CANONIFIER=$SCRIPTS/diff-sort btest-diff out + +@TEST-START-FILE input.log +#separator \x09 +#fields i s ss +1 a|b|c|d|e|f 1|2|3|4|5|6 +@TEST-END-FILE + +redef InputAscii::set_separator = "|"; + +@load frameworks/communication/listen + +global outfile: file; + +module A; + +type Idx: record { + i: int; +}; + +type Val: record { + s: set[string]; + ss:vector of count; +}; + +global servers: table[int] of Val = table(); + +event bro_init() + { + outfile = open("../out"); + # first read in the old stuff into the table... + Input::add_table([$source="../input.log", $name="ssh", $idx=Idx, $val=Val, $destination=servers]); + Input::remove("ssh"); + } + +event Input::update_finished(name: string, source:string) + { + print outfile, servers; + close(outfile); + terminate(); + } diff --git a/testing/btest/scripts/base/frameworks/input/setspecialcases.bro b/testing/btest/scripts/base/frameworks/input/setspecialcases.bro new file mode 100644 index 0000000000..239bdfe7e7 --- /dev/null +++ b/testing/btest/scripts/base/frameworks/input/setspecialcases.bro @@ -0,0 +1,50 @@ +# (uses listen.bro just to ensure input sources are more reliably fully-read). +# @TEST-SERIALIZE: comm +# +# @TEST-EXEC: btest-bg-run bro bro -b %INPUT +# @TEST-EXEC: btest-bg-wait -k 5 +# @TEST-EXEC: TEST_DIFF_CANONIFIER=$SCRIPTS/diff-sort btest-diff out + +@TEST-START-FILE input.log +#separator \x09 +#fields i s ss +1 testing\x2ctesting\x2ctesting\x2c testing\x2ctesting\x2ctesting\x2c +2 testing,,testing testing,,testing +3 ,testing ,testing +4 testing, testing, +5 ,,, ,,, +6 +@TEST-END-FILE + + +@load frameworks/communication/listen + +global outfile: file; + +module A; + +type Idx: record { + i: int; +}; + +type Val: record { + s: set[string]; + s: vector of string; +}; + +global servers: table[int] of Val = table(); + +event bro_init() + { + outfile = open("../out"); + # first read in the old stuff into the table... + Input::add_table([$source="../input.log", $name="ssh", $idx=Idx, $val=Val, $destination=servers]); + Input::remove("ssh"); + } + +event Input::update_finished(name: string, source:string) + { + print outfile, servers; + close(outfile); + terminate(); + } diff --git a/testing/btest/scripts/base/frameworks/input/unsupported_types.bro b/testing/btest/scripts/base/frameworks/input/unsupported_types.bro new file mode 100644 index 0000000000..7affa4065d --- /dev/null +++ b/testing/btest/scripts/base/frameworks/input/unsupported_types.bro @@ -0,0 +1,64 @@ +# (uses listen.bro just to ensure input sources are more reliably fully-read). +# @TEST-SERIALIZE: comm +# +# @TEST-EXEC: btest-bg-run bro bro -b %INPUT +# @TEST-EXEC: btest-bg-wait -k 5 +# @TEST-EXEC: btest-diff out + +@TEST-START-FILE input.log +#separator \x09 +#path ssh +#fields fi b i e c p sn a d t iv s sc ss se vc ve f +#types file bool int enum count port subnet addr double time interval string table table table vector vector func +whatever T -42 SSH::LOG 21 123 10.0.0.0/24 1.2.3.4 3.14 1315801931.273616 100.000000 hurz 2,4,1,3 CC,AA,BB EMPTY 10,20,30 EMPTY SSH::foo\x0a{ \x0aif (0 < SSH::i) \x0a\x09return (Foo);\x0aelse\x0a\x09return (Bar);\x0a\x0a} +@TEST-END-FILE + +@load base/protocols/ssh +@load frameworks/communication/listen + +global outfile: file; + +redef InputAscii::empty_field = "EMPTY"; +redef Input::accept_unsupported_types = T; + +module A; + +type Idx: record { + i: int; +}; + +type Val: record { + fi: file &optional; + b: bool; + e: Log::ID; + c: count; + p: port; + sn: subnet; + a: addr; + d: double; + t: time; + iv: interval; + s: string; + sc: set[count]; + ss: set[string]; + se: set[string]; + vc: vector of int; + ve: vector of int; +}; + +global servers: table[int] of Val = table(); + +event bro_init() + { + outfile = open("../out"); + # first read in the old stuff into the table... + Input::add_table([$source="../input.log", $name="ssh", $idx=Idx, $val=Val, $destination=servers]); + Input::remove("ssh"); + } + +event Input::update_finished(name: string, source:string) + { + print outfile, servers; + close(outfile); + terminate(); + } diff --git a/testing/btest/scripts/base/frameworks/logging/remote.bro b/testing/btest/scripts/base/frameworks/logging/remote.bro index 48683148f5..ba577cc92b 100644 --- a/testing/btest/scripts/base/frameworks/logging/remote.bro +++ b/testing/btest/scripts/base/frameworks/logging/remote.bro @@ -1,10 +1,10 @@ # @TEST-SERIALIZE: comm # -# @TEST-EXEC: btest-bg-run sender bro --pseudo-realtime %INPUT ../sender.bro +# @TEST-EXEC: btest-bg-run sender bro -b --pseudo-realtime %INPUT ../sender.bro # @TEST-EXEC: sleep 1 -# @TEST-EXEC: btest-bg-run receiver bro --pseudo-realtime %INPUT ../receiver.bro +# @TEST-EXEC: btest-bg-run receiver bro -b --pseudo-realtime %INPUT ../receiver.bro # @TEST-EXEC: sleep 1 -# @TEST-EXEC: btest-bg-wait -k 10 +# @TEST-EXEC: btest-bg-wait 15 # @TEST-EXEC: btest-diff sender/test.log # @TEST-EXEC: btest-diff sender/test.failure.log # @TEST-EXEC: btest-diff sender/test.success.log @@ -41,10 +41,10 @@ event bro_init() @TEST-START-FILE sender.bro -module Test; - @load frameworks/communication/listen +module Test; + function fail(rec: Log): bool { return rec$status != "success"; @@ -66,14 +66,27 @@ event remote_connection_handshake_done(p: event_peer) Log::write(Test::LOG, [$t=network_time(), $id=cid, $status="failure", $country="MX"]); disconnect(p); } + +event remote_connection_closed(p: event_peer) + { + terminate(); + } + @TEST-END-FILE @TEST-START-FILE receiver.bro ##### +@load base/frameworks/communication + redef Communication::nodes += { ["foo"] = [$host = 127.0.0.1, $connect=T, $request_logs=T] }; +event remote_connection_closed(p: event_peer) + { + terminate(); + } + @TEST-END-FILE diff --git a/testing/btest/scripts/base/frameworks/logging/rotate-custom.bro b/testing/btest/scripts/base/frameworks/logging/rotate-custom.bro index 07fc8cef7c..c0f0ef8643 100644 --- a/testing/btest/scripts/base/frameworks/logging/rotate-custom.bro +++ b/testing/btest/scripts/base/frameworks/logging/rotate-custom.bro @@ -1,7 +1,7 @@ # # @TEST-EXEC: bro -b -r ${TRACES}/rotation.trace %INPUT | egrep "test|test2" | sort >out.tmp # @TEST-EXEC: cat out.tmp pp.log | sort >out -# @TEST-EXEC: for i in `ls test*.log | sort`; do printf '> %s\n' $i; cat $i; done | sort | uniq >>out +# @TEST-EXEC: for i in `ls test*.log | sort`; do printf '> %s\n' $i; cat $i; done | sort | $SCRIPTS/diff-remove-timestamps | uniq >>out # @TEST-EXEC: btest-diff out # @TEST-EXEC: TEST_DIFF_CANONIFIER=$SCRIPTS/diff-sort btest-diff .stderr diff --git a/testing/btest/scripts/base/frameworks/metrics/basic-cluster.bro b/testing/btest/scripts/base/frameworks/metrics/basic-cluster.bro index 09479b7a2f..89ae5bf79f 100644 --- a/testing/btest/scripts/base/frameworks/metrics/basic-cluster.bro +++ b/testing/btest/scripts/base/frameworks/metrics/basic-cluster.bro @@ -5,13 +5,13 @@ # @TEST-EXEC: sleep 1 # @TEST-EXEC: btest-bg-run worker-1 BROPATH=$BROPATH:.. CLUSTER_NODE=worker-1 bro %INPUT # @TEST-EXEC: btest-bg-run worker-2 BROPATH=$BROPATH:.. CLUSTER_NODE=worker-2 bro %INPUT -# @TEST-EXEC: btest-bg-wait -k 10 +# @TEST-EXEC: btest-bg-wait 30 # @TEST-EXEC: btest-diff manager-1/metrics.log @TEST-START-FILE cluster-layout.bro redef Cluster::nodes = { - ["manager-1"] = [$node_type=Cluster::MANAGER, $ip=127.0.0.1, $p=37757/tcp, $workers=set("worker-1")], - ["proxy-1"] = [$node_type=Cluster::PROXY, $ip=127.0.0.1, $p=37758/tcp, $manager="manager-1", $workers=set("worker-1")], + ["manager-1"] = [$node_type=Cluster::MANAGER, $ip=127.0.0.1, $p=37757/tcp, $workers=set("worker-1", "worker-2")], + ["proxy-1"] = [$node_type=Cluster::PROXY, $ip=127.0.0.1, $p=37758/tcp, $manager="manager-1", $workers=set("worker-1", "worker-2")], ["worker-1"] = [$node_type=Cluster::WORKER, $ip=127.0.0.1, $p=37760/tcp, $manager="manager-1", $proxy="proxy-1", $interface="eth0"], ["worker-2"] = [$node_type=Cluster::WORKER, $ip=127.0.0.1, $p=37761/tcp, $manager="manager-1", $proxy="proxy-1", $interface="eth1"], }; @@ -28,11 +28,51 @@ event bro_init() &priority=5 Metrics::add_filter(TEST_METRIC, [$name="foo-bar", $break_interval=3secs]); - - if ( Cluster::local_node_type() == Cluster::WORKER ) + } + +event remote_connection_closed(p: event_peer) + { + terminate(); + } + +global ready_for_data: event(); + +redef Cluster::manager2worker_events += /ready_for_data/; + +@if ( Cluster::local_node_type() == Cluster::WORKER ) + +event ready_for_data() + { + Metrics::add_data(TEST_METRIC, [$host=1.2.3.4], 3); + Metrics::add_data(TEST_METRIC, [$host=6.5.4.3], 2); + Metrics::add_data(TEST_METRIC, [$host=7.2.1.5], 1); + } + +@endif + +@if ( Cluster::local_node_type() == Cluster::MANAGER ) + +global n = 0; +global peer_count = 0; + +event Metrics::log_metrics(rec: Metrics::Info) + { + n = n + 1; + if ( n == 3 ) { - Metrics::add_data(TEST_METRIC, [$host=1.2.3.4], 3); - Metrics::add_data(TEST_METRIC, [$host=6.5.4.3], 2); - Metrics::add_data(TEST_METRIC, [$host=7.2.1.5], 1); + terminate_communication(); + terminate(); } } + +event remote_connection_handshake_done(p: event_peer) + { + print p; + peer_count = peer_count + 1; + if ( peer_count == 3 ) + { + event ready_for_data(); + } + } + +@endif diff --git a/testing/btest/scripts/base/frameworks/metrics/cluster-intermediate-update.bro b/testing/btest/scripts/base/frameworks/metrics/cluster-intermediate-update.bro index 654e42976a..db2c7e9f5d 100644 --- a/testing/btest/scripts/base/frameworks/metrics/cluster-intermediate-update.bro +++ b/testing/btest/scripts/base/frameworks/metrics/cluster-intermediate-update.bro @@ -5,7 +5,7 @@ # @TEST-EXEC: sleep 1 # @TEST-EXEC: btest-bg-run worker-1 BROPATH=$BROPATH:.. CLUSTER_NODE=worker-1 bro %INPUT # @TEST-EXEC: btest-bg-run worker-2 BROPATH=$BROPATH:.. CLUSTER_NODE=worker-2 bro %INPUT -# @TEST-EXEC: btest-bg-wait -k 10 +# @TEST-EXEC: btest-bg-wait 20 # @TEST-EXEC: btest-diff manager-1/notice.log @TEST-START-FILE cluster-layout.bro @@ -37,6 +37,21 @@ event bro_init() &priority=5 $log=T]); } +event remote_connection_closed(p: event_peer) + { + terminate(); + } + +@if ( Cluster::local_node_type() == Cluster::MANAGER ) + +event Notice::log_notice(rec: Notice::Info) + { + terminate_communication(); + terminate(); + } + +@endif + @if ( Cluster::local_node_type() == Cluster::WORKER ) event do_metrics(i: count) diff --git a/testing/btest/scripts/base/frameworks/reporter/disable-stderr.bro b/testing/btest/scripts/base/frameworks/reporter/disable-stderr.bro new file mode 100644 index 0000000000..b1afb99b5c --- /dev/null +++ b/testing/btest/scripts/base/frameworks/reporter/disable-stderr.bro @@ -0,0 +1,13 @@ +# @TEST-EXEC: bro %INPUT +# @TEST-EXEC: TEST_DIFF_CANONIFIER=$SCRIPTS/diff-remove-abspath btest-diff .stderr +# @TEST-EXEC: TEST_DIFF_CANONIFIER="$SCRIPTS/diff-remove-abspath | $SCRIPTS/diff-remove-timestamps" btest-diff reporter.log + +redef Reporter::warnings_to_stderr = F; +redef Reporter::errors_to_stderr = F; + +global test: table[count] of string = {}; + +event bro_init() + { + print test[3]; + } diff --git a/testing/btest/scripts/base/frameworks/reporter/stderr.bro b/testing/btest/scripts/base/frameworks/reporter/stderr.bro new file mode 100644 index 0000000000..ef01c9fdf9 --- /dev/null +++ b/testing/btest/scripts/base/frameworks/reporter/stderr.bro @@ -0,0 +1,10 @@ +# @TEST-EXEC: bro %INPUT +# @TEST-EXEC: TEST_DIFF_CANONIFIER=$SCRIPTS/diff-remove-abspath btest-diff .stderr +# @TEST-EXEC: TEST_DIFF_CANONIFIER="$SCRIPTS/diff-remove-abspath | $SCRIPTS/diff-remove-timestamps" btest-diff reporter.log + +global test: table[count] of string = {}; + +event bro_init() + { + print test[3]; + } diff --git a/testing/btest/signatures/bad-eval-condition.bro b/testing/btest/signatures/bad-eval-condition.bro new file mode 100644 index 0000000000..34997b1124 --- /dev/null +++ b/testing/btest/signatures/bad-eval-condition.bro @@ -0,0 +1,22 @@ +# @TEST-EXEC-FAIL: bro -r $TRACES/ftp-ipv4.trace %INPUT +# @TEST-EXEC: btest-diff .stderr + +@load-sigs blah.sig + +@TEST-START-FILE blah.sig +signature blah + { + ip-proto == tcp + src-port == 21 + payload /.*/ + eval mark_conn + } +@TEST-END-FILE + +# wrong function signature for use with signature 'eval' conditions +# needs to be reported +function mark_conn(state: signature_state): bool + { + add state$conn$service["blah"]; + return T; + } diff --git a/testing/btest/signatures/eval-condition.bro b/testing/btest/signatures/eval-condition.bro new file mode 100644 index 0000000000..f3f1171da6 --- /dev/null +++ b/testing/btest/signatures/eval-condition.bro @@ -0,0 +1,20 @@ +# @TEST-EXEC: bro -r $TRACES/ftp-ipv4.trace %INPUT +# @TEST-EXEC: btest-diff conn.log + +@load-sigs blah.sig + +@TEST-START-FILE blah.sig +signature blah + { + ip-proto == tcp + src-port == 21 + payload /.*/ + eval mark_conn + } +@TEST-END-FILE + +function mark_conn(state: signature_state, data: string): bool + { + add state$conn$service["blah"]; + return T; + } diff --git a/testing/btest/core/load-sigs.bro b/testing/btest/signatures/load-sigs.bro similarity index 100% rename from testing/btest/core/load-sigs.bro rename to testing/btest/signatures/load-sigs.bro diff --git a/testing/scripts/diff-remove-mime-types b/testing/scripts/diff-remove-mime-types index fb447a9989..b8cc3d1e6d 100755 --- a/testing/scripts/diff-remove-mime-types +++ b/testing/scripts/diff-remove-mime-types @@ -3,20 +3,27 @@ # A diff canonifier that removes all MIME types because libmagic output # can differ between installations. -BEGIN { FS="\t"; OFS="\t"; column = -1; } +BEGIN { FS="\t"; OFS="\t"; type_col = -1; desc_col = -1 } /^#fields/ { for ( i = 2; i < NF; ++i ) + { if ( $i == "mime_type" ) - column = i-1; + type_col = i-1; + if ( $i == "mime_desc" ) + desc_col = i-1; + } } -column >= 0 { - if ( $column != "-" ) +function remove_mime (n) { + if ( n >= 0 && $n != "-" ) # Mark that it's set, but ignore content. - $column = "+"; + $n = "+" } +remove_mime(type_col) +remove_mime(desc_col) + { print; }