diff --git a/CHANGES b/CHANGES index 3e2b575dd6..84f1076046 100644 --- a/CHANGES +++ b/CHANGES @@ -1,4 +1,31 @@ +2.5-310 | 2017-09-21 09:10:21 -0700 + + * fix interaction of gridftp scripts with other thresholds. (Justin Azoff) + +2.5-307 | 2017-09-20 10:51:09 -0500 + + * BIT-1846: Updating broctl submodule to include fix for symlinking + issue (Jon Siwek) + +2.5-306 | 2017-09-18 14:43:42 -0700 + + * Make strerror_r portable, supporting XSI/gnu versions. (Thomas Petersen) + + * Prevent crash when calling bro -U. (Thomas Petersen) + + * Remove annoying error message from connsize bifs. (Johanna Amann) + + * Add test to verify that log rotation works with gzipped logs (Daniel Thayer) + + * Fix ascii writer to not discard a ".gz" file extension. (Daniel Thayer) + + When Bro writes a compressed log, it uses a file extension of ".gz". + However, upon log rotation the ascii writer script function + "default_rotation_postprocessor_func" was discarding the ".gz" + file extension. Fixed so that the correct file extension is + preserved after rotation. (Daniel Thayer) + 2.5-297 | 2017-09-11 09:26:33 -0700 * Fix small OCSP parser bug; serial numbers were not passed to events diff --git a/VERSION b/VERSION index a5ed34e608..3f3f014028 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -2.5-297 +2.5-310 diff --git a/aux/broctl b/aux/broctl index 1ab5ed3d3b..e960be2c19 160000 --- a/aux/broctl +++ b/aux/broctl @@ -1 +1 @@ -Subproject commit 1ab5ed3d3b0f2a3ff231de77816a697d55abccb8 +Subproject commit e960be2c192a02f1244ebca3ec31ca57d64e23dc diff --git a/scripts/base/frameworks/logging/writers/ascii.bro b/scripts/base/frameworks/logging/writers/ascii.bro index bbf11c26e7..6f2b03aafd 100644 --- a/scripts/base/frameworks/logging/writers/ascii.bro +++ b/scripts/base/frameworks/logging/writers/ascii.bro @@ -79,9 +79,12 @@ export { # runs the writer's default postprocessor command on it. function default_rotation_postprocessor_func(info: Log::RotationInfo) : bool { + # If the filename has a ".gz" extension, then keep it. + local gz = info$fname[-3:] == ".gz" ? ".gz" : ""; + # Move file to name including both opening and closing time. - local dst = fmt("%s.%s.log", info$path, - strftime(Log::default_rotation_date_format, info$open)); + local dst = fmt("%s.%s.log%s", info$path, + strftime(Log::default_rotation_date_format, info$open), gz); system(fmt("/bin/mv %s %s", info$fname, dst)); diff --git a/scripts/base/protocols/ftp/gridftp.bro b/scripts/base/protocols/ftp/gridftp.bro index 68be66d53a..38f6d8186c 100644 --- a/scripts/base/protocols/ftp/gridftp.bro +++ b/scripts/base/protocols/ftp/gridftp.bro @@ -75,6 +75,9 @@ event ConnThreshold::bytes_threshold_crossed(c: connection, threshold: count, is if ( threshold < size_threshold || "gridftp-data" in c$service || c$duration > max_time ) return; + if ( ! data_channel_initial_criteria(c) ) + return; + add c$service["gridftp-data"]; event GridFTP::data_channel_detected(c); diff --git a/src/Expr.cc b/src/Expr.cc index 9927ca52ec..bea43ff7c4 100644 --- a/src/Expr.cc +++ b/src/Expr.cc @@ -4351,9 +4351,8 @@ Val* InExpr::Fold(Val* v1, Val* v2) const const BroString* s1 = v1->AsString(); const BroString* s2 = v2->AsString(); - // Could do better here - either roll our own, to deal with - // NULs, and/or Boyer-Moore if done repeatedly. - return new Val(strstr(s2->CheckString(), s1->CheckString()) != 0, TYPE_BOOL); + // Could do better here e.g. Boyer-Moore if done repeatedly. + return new Val(strstr_n(s2->Len(), s2->Bytes(), s1->Len(), reinterpret_cast(s1->CheckString())) != -1, TYPE_BOOL); } if ( v1->Type()->Tag() == TYPE_ADDR && diff --git a/src/File.cc b/src/File.cc index 7c4a21d5e8..e0e0d63332 100644 --- a/src/File.cc +++ b/src/File.cc @@ -302,7 +302,7 @@ FILE* BroFile::BringIntoCache() if ( ! f ) { - strerror_r(errno, buf, sizeof(buf)); + bro_strerror_r(errno, buf, sizeof(buf)); reporter->Error("can't open %s: %s", name, buf); f = fopen("/dev/null", "w"); @@ -313,7 +313,7 @@ FILE* BroFile::BringIntoCache() return f; } - strerror_r(errno, buf, sizeof(buf)); + bro_strerror_r(errno, buf, sizeof(buf)); reporter->Error("can't open /dev/null: %s", buf); return 0; } @@ -323,7 +323,7 @@ FILE* BroFile::BringIntoCache() if ( fseek(f, position, SEEK_SET) < 0 ) { - strerror_r(errno, buf, sizeof(buf)); + bro_strerror_r(errno, buf, sizeof(buf)); reporter->Error("reopen seek failed: %s", buf); } @@ -413,7 +413,7 @@ void BroFile::Suspend() if ( (position = ftell(f)) < 0 ) { char buf[256]; - strerror_r(errno, buf, sizeof(buf)); + bro_strerror_r(errno, buf, sizeof(buf)); reporter->Error("ftell failed: %s", buf); position = 0; } diff --git a/src/Flare.cc b/src/Flare.cc index 5df6d663aa..87dc946955 100644 --- a/src/Flare.cc +++ b/src/Flare.cc @@ -16,7 +16,7 @@ Flare::Flare() static void bad_pipe_op(const char* which) { char buf[256]; - strerror_r(errno, buf, sizeof(buf)); + bro_strerror_r(errno, buf, sizeof(buf)); reporter->FatalErrorWithCore("unexpected pipe %s failure: %s", which, buf); } diff --git a/src/Pipe.cc b/src/Pipe.cc index 3f60409fdb..3775ca705d 100644 --- a/src/Pipe.cc +++ b/src/Pipe.cc @@ -12,7 +12,7 @@ using namespace bro; static void pipe_fail(int eno) { char tmp[256]; - strerror_r(eno, tmp, sizeof(tmp)); + bro_strerror_r(eno, tmp, sizeof(tmp)); reporter->FatalError("Pipe failure: %s", tmp); } diff --git a/src/PolicyFile.cc b/src/PolicyFile.cc index bd41c15e9d..22f09e6970 100644 --- a/src/PolicyFile.cc +++ b/src/PolicyFile.cc @@ -84,7 +84,7 @@ bool LoadPolicyFileText(const char* policy_filename) if ( fstat(fileno(f), &st) != 0 ) { char buf[256]; - strerror_r(errno, buf, sizeof(buf)); + bro_strerror_r(errno, buf, sizeof(buf)); reporter->Error("fstat failed on %s: %s", policy_filename, buf); fclose(f); return false; diff --git a/src/analyzer/protocol/conn-size/functions.bif b/src/analyzer/protocol/conn-size/functions.bif index a05359a17b..225e9db913 100644 --- a/src/analyzer/protocol/conn-size/functions.bif +++ b/src/analyzer/protocol/conn-size/functions.bif @@ -5,10 +5,7 @@ static analyzer::Analyzer* GetConnsizeAnalyzer(Val* cid) { Connection* c = sessions->FindConnection(cid); if ( ! c ) - { - reporter->Error("cannot find connection"); return 0; - } analyzer::Analyzer* a = c->FindAnalyzer("CONNSIZE"); if ( ! a ) diff --git a/src/analyzer/protocol/ssl/tls-handshake-protocol.pac b/src/analyzer/protocol/ssl/tls-handshake-protocol.pac index 6a1988111e..bd155d644d 100644 --- a/src/analyzer/protocol/ssl/tls-handshake-protocol.pac +++ b/src/analyzer/protocol/ssl/tls-handshake-protocol.pac @@ -487,7 +487,7 @@ type SSLExtension(rec: HandshakeRecord) = record { EXT_SIGNATURE_ALGORITHMS -> signature_algorithm: SignatureAlgorithm(rec)[] &until($element == 0 || $element != 0); EXT_SIGNED_CERTIFICATE_TIMESTAMP -> certificate_timestamp: SignedCertificateTimestampList(rec)[] &until($element == 0 || $element != 0); EXT_KEY_SHARE -> key_share: KeyShare(rec)[] &until($element == 0 || $element != 0); - EXT_SUPPORTED_VERSIONS -> supported_versions: SupportedVersions(rec)[] &until($element == 0 || $element != 0); + EXT_SUPPORTED_VERSIONS -> supported_versions_selector: SupportedVersionsSelector(rec, data_len)[] &until($element == 0 || $element != 0); EXT_PSK_KEY_EXCHANGE_MODES -> psk_key_exchange_modes: PSKKeyExchangeModes(rec)[] &until($element == 0 || $element != 0); default -> data: bytestring &restofdata; }; @@ -495,6 +495,11 @@ type SSLExtension(rec: HandshakeRecord) = record { %include tls-handshake-signed_certificate_timestamp.pac +type SupportedVersionsSelector(rec: HandshakeRecord, data_len: uint16) = case rec.is_orig of { + true -> a: SupportedVersions(rec); + false -> b: bytestring &length=data_len &transient; +} + type SupportedVersions(rec: HandshakeRecord) = record { length: uint8; versions: uint16[] &until($input.length() == 0); diff --git a/src/analyzer/protocol/tcp/TCP_Endpoint.cc b/src/analyzer/protocol/tcp/TCP_Endpoint.cc index 7c359623f3..c3175ec9f5 100644 --- a/src/analyzer/protocol/tcp/TCP_Endpoint.cc +++ b/src/analyzer/protocol/tcp/TCP_Endpoint.cc @@ -229,7 +229,7 @@ int TCP_Endpoint::DataSent(double t, uint64 seq, int len, int caplen, if ( fwrite(data, 1, len, f) < unsigned(len) ) { char buf[256]; - strerror_r(errno, buf, sizeof(buf)); + bro_strerror_r(errno, buf, sizeof(buf)); reporter->Error("TCP contents write failed: %s", buf); if ( contents_file_write_failure ) diff --git a/src/file_analysis/analyzer/extract/Extract.cc b/src/file_analysis/analyzer/extract/Extract.cc index c758414a6e..f936a5156b 100644 --- a/src/file_analysis/analyzer/extract/Extract.cc +++ b/src/file_analysis/analyzer/extract/Extract.cc @@ -20,7 +20,7 @@ Extract::Extract(RecordVal* args, File* file, const string& arg_filename, { fd = 0; char buf[128]; - strerror_r(errno, buf, sizeof(buf)); + bro_strerror_r(errno, buf, sizeof(buf)); reporter->Error("cannot open %s: %s", filename.c_str(), buf); } } diff --git a/src/input/readers/raw/Raw.cc b/src/input/readers/raw/Raw.cc index ae1f0939a8..27d8b0c685 100644 --- a/src/input/readers/raw/Raw.cc +++ b/src/input/readers/raw/Raw.cc @@ -90,7 +90,7 @@ bool Raw::SetFDFlags(int fd, int cmd, int flags) return true; char buf[256]; - strerror_r(errno, buf, sizeof(buf)); + bro_strerror_r(errno, buf, sizeof(buf)); Error(Fmt("failed to set fd flags: %s", buf)); return false; } @@ -197,7 +197,7 @@ bool Raw::Execute() else { char buf[256]; - strerror_r(errno, buf, sizeof(buf)); + bro_strerror_r(errno, buf, sizeof(buf)); Warning(Fmt("Could not set child process group: %s", buf)); } } @@ -293,7 +293,7 @@ bool Raw::OpenInput() if ( fseek(file.get(), pos, whence) < 0 ) { char buf[256]; - strerror_r(errno, buf, sizeof(buf)); + bro_strerror_r(errno, buf, sizeof(buf)); Error(Fmt("Seek failed in init: %s", buf)); } } diff --git a/src/logging/writers/ascii/Ascii.cc b/src/logging/writers/ascii/Ascii.cc index dec1689df4..baaba22665 100644 --- a/src/logging/writers/ascii/Ascii.cc +++ b/src/logging/writers/ascii/Ascii.cc @@ -414,7 +414,7 @@ bool Ascii::DoRotate(const char* rotated_path, double open, double close, bool t if ( rename(fname.c_str(), nname.c_str()) != 0 ) { char buf[256]; - strerror_r(errno, buf, sizeof(buf)); + bro_strerror_r(errno, buf, sizeof(buf)); Error(Fmt("failed to rename %s to %s: %s", fname.c_str(), nname.c_str(), buf)); FinishedRotation(); diff --git a/src/threading/BasicThread.cc b/src/threading/BasicThread.cc index d63b307470..3b6f5d6532 100644 --- a/src/threading/BasicThread.cc +++ b/src/threading/BasicThread.cc @@ -98,7 +98,7 @@ const char* BasicThread::Strerror(int err) if ( ! strerr_buffer ) strerr_buffer = new char[256]; - strerror_r(err, strerr_buffer, 256); + bro_strerror_r(err, strerr_buffer, 256); return strerr_buffer; } diff --git a/src/util.cc b/src/util.cc index acfcb19573..a2f0cb8c94 100644 --- a/src/util.cc +++ b/src/util.cc @@ -1012,7 +1012,7 @@ FILE* open_file(const string& path, const string& mode) if ( ! rval ) { char buf[256]; - strerror_r(errno, buf, sizeof(buf)); + bro_strerror_r(errno, buf, sizeof(buf)); reporter->Error("Failed to open file %s: %s", filename, buf); } @@ -1396,9 +1396,13 @@ void _set_processing_status(const char* status) if ( fd < 0 ) { char buf[256]; - strerror_r(errno, buf, sizeof(buf)); - reporter->Error("Failed to open process status file '%s': %s", - proc_status_file, buf); + bro_strerror_r(errno, buf, sizeof(buf)); + if ( reporter ) + reporter->Error("Failed to open process status file '%s': %s", + proc_status_file, buf); + else + fprintf(stderr, "Failed to open process status file '%s': %s\n", + proc_status_file, buf); errno = old_errno; return; } @@ -1612,7 +1616,7 @@ void safe_close(int fd) if ( close(fd) < 0 && errno != EINTR ) { char buf[128]; - strerror_r(errno, buf, sizeof(buf)); + bro_strerror_r(errno, buf, sizeof(buf)); fprintf(stderr, "safe_close error %d: %s\n", errno, buf); abort(); } @@ -1745,3 +1749,24 @@ std::string canonify_name(const std::string& name) return nname; } + +static void strerror_r_helper(char* result, char* buf, size_t buflen) + { + // Seems the GNU flavor of strerror_r may return a pointer to a static + // string. So try to copy as much as possible into desired buffer. + auto len = strlen(result); + strncpy(buf, result, buflen); + + if ( len >= buflen ) + buf[buflen - 1] = 0; + } + +static void strerror_r_helper(int result, char* buf, size_t buflen) + { /* XSI flavor of strerror_r, no-op. */ } + +void bro_strerror_r(int bro_errno, char* buf, size_t buflen) + { + auto res = strerror_r(bro_errno, buf, buflen); + // GNU vs. XSI flavors make it harder to use strerror_r. + strerror_r_helper(res, buf, buflen); + } diff --git a/src/util.h b/src/util.h index a2c1b78db3..30ef8a61da 100644 --- a/src/util.h +++ b/src/util.h @@ -516,4 +516,10 @@ struct CompareString */ std::string canonify_name(const std::string& name); +/** + * Reentrant version of strerror(). Takes care of the difference between the + * XSI-compliant and the GNU-specific version of strerror_r(). + */ +void bro_strerror_r(int bro_errno, char* buf, size_t buflen); + #endif diff --git a/testing/btest/Baseline/scripts.base.files.data_event.basic/.stderr b/testing/btest/Baseline/scripts.base.files.data_event.basic/.stderr new file mode 100644 index 0000000000..e69de29bb2 diff --git a/testing/btest/Baseline/scripts.base.files.data_event.basic/.stdout b/testing/btest/Baseline/scripts.base.files.data_event.basic/.stdout new file mode 100644 index 0000000000..ddfdf71f06 --- /dev/null +++ b/testing/btest/Baseline/scripts.base.files.data_event.basic/.stdout @@ -0,0 +1,3 @@ +Found +Found +Found diff --git a/testing/btest/Baseline/scripts.base.protocols.ssl.tls13-experiment/.stdout b/testing/btest/Baseline/scripts.base.protocols.ssl.tls13-experiment/.stdout new file mode 100644 index 0000000000..0b7bcb5742 --- /dev/null +++ b/testing/btest/Baseline/scripts.base.protocols.ssl.tls13-experiment/.stdout @@ -0,0 +1 @@ +7e01 diff --git a/testing/btest/Baseline/scripts.base.protocols.ssl.tls13-experiment/ssl.log b/testing/btest/Baseline/scripts.base.protocols.ssl.tls13-experiment/ssl.log new file mode 100644 index 0000000000..c88237dd18 --- /dev/null +++ b/testing/btest/Baseline/scripts.base.protocols.ssl.tls13-experiment/ssl.log @@ -0,0 +1,10 @@ +#separator \x09 +#set_separator , +#empty_field (empty) +#unset_field - +#path ssl +#open 2017-09-10-05-23-15 +#fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p version cipher curve server_name resumed last_alert next_protocol established cert_chain_fuids client_cert_chain_fuids subject issuer client_subject client_issuer +#types time string addr port addr port string string string string bool string string bool vector[string] vector[string] string string string string +1505019126.007778 CHhAvVGS1DHFjwGM9 192.168.0.2 62873 104.196.219.53 443 TLSv12 TLS_AES_128_GCM_SHA256 x25519 tls.ctf.network T - - T - - - - - - +#close 2017-09-10-05-23-16 diff --git a/testing/btest/Traces/tls/chrome-63.0.3211.0-canary-tls_experiment.pcap b/testing/btest/Traces/tls/chrome-63.0.3211.0-canary-tls_experiment.pcap new file mode 100644 index 0000000000..2b8040b109 Binary files /dev/null and b/testing/btest/Traces/tls/chrome-63.0.3211.0-canary-tls_experiment.pcap differ diff --git a/testing/btest/scripts/base/files/data_event/basic.bro b/testing/btest/scripts/base/files/data_event/basic.bro new file mode 100644 index 0000000000..2877155ebb --- /dev/null +++ b/testing/btest/scripts/base/files/data_event/basic.bro @@ -0,0 +1,20 @@ +# Just a very basic test to check if ANALYZER_DATA_EVENT works. +# Also check if "in" works with binary data. +# @TEST-EXEC: bro -r $TRACES/pe/pe.trace %INPUT +# @TEST-EXEC: btest-diff .stdout +# @TEST-EXEC: btest-diff .stderr + +event stream_data(f: fa_file, data: string) + { + if ( "Windows" in data ) + { + print "Found"; + } + } + +event file_new (f: fa_file) + { + Files::add_analyzer(f, Files::ANALYZER_DATA_EVENT, + [$stream_event=stream_data]); + } + diff --git a/testing/btest/scripts/base/frameworks/logging/ascii-gz-rotate.bro b/testing/btest/scripts/base/frameworks/logging/ascii-gz-rotate.bro new file mode 100644 index 0000000000..2a1c388322 --- /dev/null +++ b/testing/btest/scripts/base/frameworks/logging/ascii-gz-rotate.bro @@ -0,0 +1,25 @@ +# Test that log rotation works with compressed logs. +# +# @TEST-EXEC: bro -b %INPUT +# @TEST-EXEC: gunzip test.*.log.gz +# + +module Test; + +export { + redef enum Log::ID += { LOG }; + + type Log: record { + s: string; + } &log; +} + +redef Log::default_rotation_interval = 1hr; +redef LogAscii::gzip_level = 1; + +event bro_init() +{ + Log::create_stream(Test::LOG, [$columns=Log]); + + Log::write(Test::LOG, [$s="testing"]); +} diff --git a/testing/btest/scripts/base/protocols/ssl/tls13-experiment.test b/testing/btest/scripts/base/protocols/ssl/tls13-experiment.test new file mode 100644 index 0000000000..d49cd928f6 --- /dev/null +++ b/testing/btest/scripts/base/protocols/ssl/tls13-experiment.test @@ -0,0 +1,16 @@ +# @TEST-EXEC: bro -C -r $TRACES/tls/chrome-63.0.3211.0-canary-tls_experiment.pcap %INPUT +# @TEST-EXEC: btest-diff ssl.log +# @TEST-EXEC: btest-diff .stdout + +# This is a trace that uses a completely non-standard way of establishing TLS 1.3; this seems +# to be an undocumented extension where the TLS version is negotiated via the server sending back +# an supported_versions extension (which, according to the RFC is strictly prohibited). +# +# This only seems to happen with Chrome talking to google servers. We do not recognize this as +# TLS 1.3, but we do not abort when encountering traffic like this. + +event ssl_extension(c: connection, is_orig: bool, code: count, val: string) + { + if ( ! is_orig && code == 43 ) + print bytestring_to_hexstr(val); + }