diff --git a/CHANGES b/CHANGES index 2d528beefb..1f2df6ff0d 100644 --- a/CHANGES +++ b/CHANGES @@ -1,4 +1,48 @@ +2.0-145 | 2012-03-09 15:10:35 -0800 + + * Remove the match expression. 'match' and 'using' are no longer + keywords. Addressed #753. (Jon Siwek) + +2.0-143 | 2012-03-09 15:07:42 -0800 + + * Fix a BRO_PROFILER_FILE/mkstemp portability issue. Addresses #794. + (Jon Siwek) + +2.0-139 | 2012-03-02 09:33:04 -0800 + + * Changes to how script coverage integrates with test suites. (Jon Siwek) + + - BRO_PROFILER_FILE now passes .X* templated filenames to mkstemp + for generating unique coverage state files. + + - Rearranging Makefile targets. The general rule is that if the + all/brief target fails out due to a test failure, then the dependent + coverage target won't run, but can still be invoked directly later. + (e.g. make brief || make coverage) + + * Standardized on the &default function for SSL constants. (Seth + Hall) + + * Adding btest group "leaks" to leak tests. (Robin Sommer) + + * Adding btest group "comm" to communication tests for parallelizing + execution with new btest version. (Robin Sommer) + + * Sorting all output for diffing in the external tests. (Robin + Sommer) + + * Cleaned up dead code from the old SSL analyzers. Reported by + Julien Sentier. (Seth Hall) + + * Update/add tests for broccoli IPv6 addr/subnet support. Addresses + #448. (Jon Siwek) + + * Remove connection compressor. Addresses #559. (Jon Siwek) + + * Refactor IP_Hdr class ctors. Addresses #532. (Jon Siwek) + + 2.0-121 | 2012-02-24 16:34:17 -0800 * A number of smaller memory fixes and code cleanups. (Julien diff --git a/NEWS b/NEWS index 045ba62a7a..15b834b040 100644 --- a/NEWS +++ b/NEWS @@ -21,6 +21,13 @@ Bro 2.1 such that at the scripting layer, the name resolution can yield a set with both IPv4 and IPv6 addresses. +- The connection compressor was already deprecated in 2.0 and has now + been removed from the code base. + +- We removed the "match" statement, which was no longer used by any of + the default scripts, nor was it likely to be used by anybody anytime + soon. With that, "match" and "using" are no longer reserved keywords. + TODO: Extend. Bro 2.0 diff --git a/VERSION b/VERSION index 2027efd988..c4c99acc07 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -2.0-121 +2.0-145 diff --git a/scripts/base/protocols/ssl/consts.bro b/scripts/base/protocols/ssl/consts.bro index 9d16ab18ba..ab130c4318 100644 --- a/scripts/base/protocols/ssl/consts.bro +++ b/scripts/base/protocols/ssl/consts.bro @@ -13,7 +13,7 @@ export { [TLSv10] = "TLSv10", [TLSv11] = "TLSv11", [TLSv12] = "TLSv12", - } &default="UNKNOWN"; + } &default=function(i: count):string { return fmt("unknown-%d", i); }; ## Mapping between numeric codes and human readable strings for alert ## levels. @@ -535,7 +535,7 @@ export { [SSL_RSA_FIPS_WITH_3DES_EDE_CBC_SHA] = "SSL_RSA_FIPS_WITH_3DES_EDE_CBC_SHA", [SSL_RSA_FIPS_WITH_DES_CBC_SHA_2] = "SSL_RSA_FIPS_WITH_DES_CBC_SHA_2", [SSL_RSA_FIPS_WITH_3DES_EDE_CBC_SHA_2] = "SSL_RSA_FIPS_WITH_3DES_EDE_CBC_SHA_2", - } &default="UNKNOWN"; + } &default=function(i: count):string { return fmt("unknown-%d", i); }; ## Mapping between the constants and string values for SSL/TLS errors. const x509_errors: table[count] of string = { @@ -573,6 +573,6 @@ export { [31] = "keyusage no certsign", [32] = "unable to get crl issuer", [33] = "unhandled critical extension", - }; + } &default=function(i: count):string { return fmt("unknown-%d", i); }; } diff --git a/src/Brofiler.cc b/src/Brofiler.cc index 783d027761..c9a3505069 100644 --- a/src/Brofiler.cc +++ b/src/Brofiler.cc @@ -1,4 +1,5 @@ #include +#include #include #include #include "Brofiler.h" @@ -48,10 +49,27 @@ bool Brofiler::WriteStats() char* bf = getenv("BRO_PROFILER_FILE"); if ( ! bf ) return false; - FILE* f = fopen(bf, "w"); + FILE* f; + const char* p = strstr(bf, ".XXXXXX"); + + if ( p && ! p[7] ) + { + int fd = mkstemp(bf); + if ( fd == -1 ) + { + reporter->Error("Failed to generate unique file name from BRO_PROFILER_FILE: %s", bf); + return false; + } + f = fdopen(fd, "w"); + } + else + { + f = fopen(bf, "w"); + } + if ( ! f ) { - reporter->Error("Failed to open BRO_PROFILER_FILE destination '%s' for writing\n", bf); + reporter->Error("Failed to open BRO_PROFILER_FILE destination '%s' for writing", bf); return false; } diff --git a/src/Brofiler.h b/src/Brofiler.h index edbe1e932c..22e5808bf6 100644 --- a/src/Brofiler.h +++ b/src/Brofiler.h @@ -26,7 +26,9 @@ public: /** * Combines usage stats from current run with any read from ReadStats(), * then writes information to file pointed to by environment variable - * BRO_PROFILER_FILE. + * BRO_PROFILER_FILE. If the value of that env. variable ends with + * ".XXXXXX" (exactly 6 X's), then it is first passed through mkstemp + * to get a unique file. * * @return: true when usage info is written, otherwise false. */ diff --git a/src/Expr.cc b/src/Expr.cc index 45f363a559..58f5db3fd1 100644 --- a/src/Expr.cc +++ b/src/Expr.cc @@ -3633,151 +3633,6 @@ bool FieldAssignExpr::DoUnserialize(UnserialInfo* info) return true; } -RecordMatchExpr::RecordMatchExpr(Expr* op1 /* record to match */, - Expr* op2 /* cases to match against */) -: BinaryExpr(EXPR_MATCH, op1, op2) - { - BroType* result_type = 0; - - // Make sure the second argument is of a suitable type. - if ( ! op2->Type()->IsSet() ) - { - ExprError("matching must be done against a set of match records"); - return; - } - - type_list* elt_types = op2->Type()->AsSetType()->Indices()->Types(); - - if ( ! elt_types->length() || - (*elt_types)[0]->Tag() != TYPE_RECORD ) - { - ExprError("matching must be done against a set of match records"); - return; - } - - RecordType* case_rec_type = (*elt_types)[0]->AsRecordType(); - - // NOTE: The "result" and "pred" field names are hardcoded here. - result_field_index = case_rec_type->FieldOffset("result"); - - if ( result_field_index < 0 ) - { - ExprError("match records must have a $result field"); - return; - } - - result_type = case_rec_type->FieldType("result")->Ref(); - - // Check that pred exists, and that the first argument matches it. - if ( (pred_field_index = case_rec_type->FieldOffset("pred")) < 0 || - case_rec_type->FieldType("pred")->Tag() != TYPE_FUNC ) - { - ExprError("match records must have a $pred' field of function type"); - return; - } - - FuncType* pred_type = case_rec_type->FieldType("pred")->AsFuncType(); - type_list* pred_arg_types = pred_type->ArgTypes()->Types(); - if ( pred_arg_types->length() != 1 || - ! check_and_promote_expr(op1, (*pred_arg_types)[0]) ) - ExprError("record to match does not have the same type as predicate argument"); - - // NOTE: The "priority" field name is hardcoded here. - if ( (priority_field_index = case_rec_type->FieldOffset("priority")) >= 0 && - ! IsArithmetic(case_rec_type->FieldType("priority")->Tag()) ) - ExprError("$priority field must have a numeric type"); - - SetType(result_type); - } - -void RecordMatchExpr::ExprDescribe(ODesc* d) const - { - if ( d->IsReadable() ) - { - d->Add("match "); - op1->Describe(d); - d->Add(" using "); - op2->Describe(d); - } - } - -Val* RecordMatchExpr::Fold(Val* v1, Val* v2) const - { - TableVal* match_set = v2->AsTableVal(); - if ( ! match_set ) - Internal("non-table in RecordMatchExpr"); - - Val* return_val = 0; - double highest_priority = -1e100; - - ListVal* match_recs = match_set->ConvertToList(TYPE_ANY); - for ( int i = 0; i < match_recs->Length(); ++i ) - { - val_list args(1); - args.append(v1->Ref()); - - double this_priority = 0; - - // ### Get rid of the double Index if TYPE_ANY->TYPE_RECORD. - Val* v = match_recs->Index(i)->AsListVal()->Index(0); - - const RecordVal* match_rec = v->AsRecordVal(); - if ( ! match_rec ) - Internal("Element of match set is not a record"); - - if ( priority_field_index >= 0 ) - { - this_priority = - match_rec->Lookup(priority_field_index)->CoerceToDouble(); - if ( this_priority <= highest_priority ) - { - Unref(v1); - continue; - } - } - - // No try/catch here; we pass exceptions upstream. - Val* pred_val = - match_rec->Lookup(pred_field_index)->AsFunc()->Call(&args); - bool is_zero = pred_val->IsZero(); - Unref(pred_val); - - if ( ! is_zero ) - { - Val* new_return_val = - match_rec->Lookup(result_field_index); - - Unref(return_val); - return_val = new_return_val->Ref(); - - if ( priority_field_index >= 0 ) - highest_priority = this_priority; - else - break; - } - } - - Unref(match_recs); - - return return_val; - } - -IMPLEMENT_SERIAL(RecordMatchExpr, SER_RECORD_MATCH_EXPR); - -bool RecordMatchExpr::DoSerialize(SerialInfo* info) const - { - DO_SERIALIZE(SER_RECORD_MATCH_EXPR, BinaryExpr); - return SERIALIZE(pred_field_index) && SERIALIZE(result_field_index) && - SERIALIZE(priority_field_index); - } - -bool RecordMatchExpr::DoUnserialize(UnserialInfo* info) - { - DO_UNSERIALIZE(BinaryExpr); - return UNSERIALIZE(&pred_field_index) && UNSERIALIZE(&result_field_index) && - UNSERIALIZE(&priority_field_index); - } - ArithCoerceExpr::ArithCoerceExpr(Expr* arg_op, TypeTag t) : UnaryExpr(EXPR_ARITH_COERCE, arg_op) { diff --git a/src/Expr.h b/src/Expr.h index 8676a1ad7e..f0798359c2 100644 --- a/src/Expr.h +++ b/src/Expr.h @@ -823,32 +823,6 @@ protected: string field_name; }; -class RecordMatchExpr : public BinaryExpr { -public: - RecordMatchExpr(Expr* op1 /* record to match */, - Expr* op2 /* cases to match against */); - -protected: - friend class Expr; - RecordMatchExpr() - { - pred_field_index = result_field_index = - priority_field_index = 0; - } - - virtual Val* Fold(Val* v1, Val* v2) const; - void ExprDescribe(ODesc*) const; - - DECLARE_SERIAL(RecordMatchExpr); - - // The following are used to hold the field offset of - // $pred, $result, $priority, so the names only need to - // be looked up at compile-time. - int pred_field_index; - int result_field_index; - int priority_field_index; -}; - class ArithCoerceExpr : public UnaryExpr { public: ArithCoerceExpr(Expr* op, TypeTag t); diff --git a/src/SerialTypes.h b/src/SerialTypes.h index 0ba48f89a9..c47ff19298 100644 --- a/src/SerialTypes.h +++ b/src/SerialTypes.h @@ -125,7 +125,7 @@ SERIAL_EXPR(FIELD_EXPR, 22) SERIAL_EXPR(HAS_FIELD_EXPR, 23) SERIAL_EXPR(RECORD_CONSTRUCTOR_EXPR, 24) SERIAL_EXPR(FIELD_ASSIGN_EXPR, 25) -SERIAL_EXPR(RECORD_MATCH_EXPR, 26) +// There used to be a SERIAL_EXPR(RECORD_MATCH_EXPR, 26) here SERIAL_EXPR(ARITH_COERCE_EXPR, 27) SERIAL_EXPR(RECORD_COERCE_EXPR, 28) SERIAL_EXPR(FLATTEN_EXPR, 29) diff --git a/src/Serializer.h b/src/Serializer.h index cd1199a340..e7396cb7f8 100644 --- a/src/Serializer.h +++ b/src/Serializer.h @@ -125,7 +125,7 @@ protected: // This will be increased whenever there is an incompatible change // in the data format. - static const uint32 DATA_FORMAT_VERSION = 21; + static const uint32 DATA_FORMAT_VERSION = 22; ChunkedIO* io; diff --git a/src/parse.y b/src/parse.y index 1b05171ecf..f78003f08b 100644 --- a/src/parse.y +++ b/src/parse.y @@ -10,7 +10,7 @@ %token TOK_CONSTANT TOK_COPY TOK_COUNT TOK_COUNTER TOK_DEFAULT TOK_DELETE %token TOK_DOUBLE TOK_ELSE TOK_ENUM TOK_EVENT TOK_EXPORT TOK_FILE TOK_FOR %token TOK_FUNCTION TOK_GLOBAL TOK_ID TOK_IF TOK_INT -%token TOK_INTERVAL TOK_LIST TOK_LOCAL TOK_MODULE TOK_MATCH +%token TOK_INTERVAL TOK_LIST TOK_LOCAL TOK_MODULE %token TOK_NEXT TOK_OF TOK_PATTERN TOK_PATTERN_TEXT %token TOK_PORT TOK_PRINT TOK_RECORD TOK_REDEF %token TOK_REMOVE_FROM TOK_RETURN TOK_SCHEDULE TOK_SET @@ -33,7 +33,7 @@ %left ',' '|' %right '=' TOK_ADD_TO TOK_REMOVE_FROM -%right '?' ':' TOK_USING +%right '?' ':' %left TOK_OR %left TOK_AND %nonassoc '<' '>' TOK_LE TOK_GE TOK_EQ TOK_NE @@ -504,12 +504,6 @@ expr: $$ = new VectorConstructorExpr($3); } - | TOK_MATCH expr TOK_USING expr - { - set_location(@1, @4); - $$ = new RecordMatchExpr($2, $4); - } - | expr '(' opt_expr_list ')' { set_location(@1, @4); diff --git a/src/scan.l b/src/scan.l index 4914783c44..5bb97e4314 100644 --- a/src/scan.l +++ b/src/scan.l @@ -273,7 +273,6 @@ int return TOK_INT; interval return TOK_INTERVAL; list return TOK_LIST; local return TOK_LOCAL; -match return TOK_MATCH; module return TOK_MODULE; next return TOK_NEXT; of return TOK_OF; @@ -295,7 +294,6 @@ timeout return TOK_TIMEOUT; timer return TOK_TIMER; type return TOK_TYPE; union return TOK_UNION; -using return TOK_USING; vector return TOK_VECTOR; when return TOK_WHEN; diff --git a/testing/Makefile b/testing/Makefile index f65d5a1fef..1c82580ec4 100644 --- a/testing/Makefile +++ b/testing/Makefile @@ -12,6 +12,7 @@ make-brief: @for repo in $(DIRS); do (cd $$repo && make brief ); done coverage: + @for repo in $(DIRS); do (cd $$repo && echo "Coverage for '$$repo' dir:" && make coverage); done @test -f btest/coverage.log && cp btest/coverage.log `mktemp brocov.tmp.XXX` || true @for f in external/*/coverage.log; do test -f $$f && cp $$f `mktemp brocov.tmp.XXX` || true; done @echo "Complete test suite code coverage:" diff --git a/testing/btest/Baseline/language.match-test/output b/testing/btest/Baseline/language.match-test/output deleted file mode 100644 index 5ee7ba029d..0000000000 --- a/testing/btest/Baseline/language.match-test/output +++ /dev/null @@ -1,3 +0,0 @@ -default -it's big -it's really big diff --git a/testing/btest/Baseline/language.match-test2/output b/testing/btest/Baseline/language.match-test2/output deleted file mode 100644 index 0cfbf08886..0000000000 --- a/testing/btest/Baseline/language.match-test2/output +++ /dev/null @@ -1 +0,0 @@ -2 diff --git a/testing/btest/Makefile b/testing/btest/Makefile index a2ca30609a..caf0a786f7 100644 --- a/testing/btest/Makefile +++ b/testing/btest/Makefile @@ -2,16 +2,23 @@ DIAG=diag.log BTEST=../../aux/btest/btest -all: cleanup - # Showing all tests. - @$(BTEST) -f $(DIAG) - @../scripts/coverage-calc ".tmp/script-coverage*" coverage.log `pwd`/../../scripts +all: cleanup btest-verbose coverage -brief: cleanup - # Brief output showing only failed tests. +# Showing all tests. +btest-verbose: + @$(BTEST) -f $(DIAG) + +brief: cleanup btest-brief coverage + +# Brief output showing only failed tests. +btest-brief: @$(BTEST) -b -f $(DIAG) + +coverage: @../scripts/coverage-calc ".tmp/script-coverage*" coverage.log `pwd`/../../scripts cleanup: @rm -f $(DIAG) @rm -f .tmp/script-coverage* + +.PHONY: all btest-verbose brief btest-brief coverage cleanup diff --git a/testing/btest/btest.cfg b/testing/btest/btest.cfg index 2126e733e7..6afbde1ddb 100644 --- a/testing/btest/btest.cfg +++ b/testing/btest/btest.cfg @@ -18,4 +18,4 @@ DIST=%(testbase)s/../.. BUILD=%(testbase)s/../../build TEST_DIFF_CANONIFIER=$SCRIPTS/diff-canonifier TMPDIR=%(testbase)s/.tmp -BRO_PROFILER_FILE=%(testbase)s/.tmp/script-coverage +BRO_PROFILER_FILE=%(testbase)s/.tmp/script-coverage.XXXXXX diff --git a/testing/btest/language/match-test.bro b/testing/btest/language/match-test.bro deleted file mode 100644 index 9352d0f39f..0000000000 --- a/testing/btest/language/match-test.bro +++ /dev/null @@ -1,20 +0,0 @@ -# @TEST-EXEC: bro %INPUT >output 2>&1 -# @TEST-EXEC: btest-diff output - -global match_stuff = { - [$pred(a: count) = { return a > 5; }, - $result = "it's big", - $priority = 2], - - [$pred(a: count) = { return a > 15; }, - $result = "it's really big", - $priority = 3], - - [$pred(a: count) = { return T; }, - $result = "default", - $priority = 0], -}; - -print match 0 using match_stuff; -print match 10 using match_stuff; -print match 20 using match_stuff; diff --git a/testing/btest/language/match-test2.bro b/testing/btest/language/match-test2.bro deleted file mode 100644 index f1c120adf2..0000000000 --- a/testing/btest/language/match-test2.bro +++ /dev/null @@ -1,51 +0,0 @@ -# @TEST-EXEC: bro %INPUT >output 2>&1 -# @TEST-EXEC: btest-diff output - -type fakealert : record { - alert: string; -}; - - -type match_rec : record { - result : count; - pred : function(rec : fakealert) : bool; - priority: count; -}; - - -#global test_set : set[int] = -#{ -#1, 2, 3 -#}; - -global match_set : set[match_rec] = -{ - [$result = 1, $pred(a: fakealert) = { return T; }, $priority = 8 ], - [$result = 2, $pred(a: fakealert) = { return T; }, $priority = 9 ] -}; - -global al : fakealert; - -#global testset : set[fakealert] = -#{ -# [$alert="hithere"] -#}; - - -type nonalert: record { - alert : string; - pred : function(a : int) : int; -}; - -#global na : nonalert; -#na$alert = "5"; - -#al$alert = "hithere2"; -#if (al in testset) -# print 1; -#else -# print 0; - - -al$alert = "hi"; -print (match al using match_set); diff --git a/testing/external/Makefile b/testing/external/Makefile index 994d8962c0..b705734003 100644 --- a/testing/external/Makefile +++ b/testing/external/Makefile @@ -24,3 +24,7 @@ push: status: @for repo in $(REPOS); do ( cd $$repo && echo '>>' $$repo && git status -bs && echo ); done +coverage: + @for repo in $(REPOS); do (cd $$repo && echo "Coverage for '$$repo' repo:" && make coverage); done + +.PHONY: all brief init pull push status coverage diff --git a/testing/external/subdir-btest.cfg b/testing/external/subdir-btest.cfg index 7b1d59cb07..c4e74f99fa 100644 --- a/testing/external/subdir-btest.cfg +++ b/testing/external/subdir-btest.cfg @@ -17,4 +17,4 @@ TRACES=%(testbase)s/Traces SCRIPTS=%(testbase)s/../scripts DIST=%(testbase)s/../../.. BUILD=%(testbase)s/../../../build -BRO_PROFILER_FILE=%(testbase)s/.tmp/script-coverage +BRO_PROFILER_FILE=%(testbase)s/.tmp/script-coverage.XXXXXX diff --git a/testing/scripts/btest-bg-run b/testing/scripts/btest-bg-run deleted file mode 100755 index 64a38b9759..0000000000 --- a/testing/scripts/btest-bg-run +++ /dev/null @@ -1,7 +0,0 @@ -#! /usr/bin/env bash - -# This is a wrapper script to btest's real btest-bg-run. It's used -# when collecting Bro script coverage statistics so that two independent -# Bro processing don't try to write those usage statistics to the same file. - -BRO_PROFILER_FILE=`mktemp $TMPDIR/script-coverage.XXXX` $BTEST_PATH/btest-bg-run $@