From b73809d54f960c9e50dd7651ec512f4a16b498eb Mon Sep 17 00:00:00 2001 From: Matthias Vallentin Date: Thu, 27 Sep 2012 12:18:25 -0700 Subject: [PATCH 1/4] Fix compile issues with older versions of libcurl. Older versions of libcurl do not offer *_MS timeout constants, which causes the build to fail. For sub-second timeout specification, we now fall back to hard-coded timeouts in older libcurl version. --- src/logging/writers/ElasticSearch.cc | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/src/logging/writers/ElasticSearch.cc b/src/logging/writers/ElasticSearch.cc index cb3248a044..24489314ec 100644 --- a/src/logging/writers/ElasticSearch.cc +++ b/src/logging/writers/ElasticSearch.cc @@ -48,7 +48,7 @@ ElasticSearch::ElasticSearch(WriterFrontend* frontend) : WriterBackend(frontend) last_send = current_time(); failing = false; - transfer_timeout = BifConst::LogElasticSearch::transfer_timeout * 1000; + transfer_timeout = static_cast(BifConst::LogElasticSearch::transfer_timeout) * 1000; curl_handle = HTTPSetup(); } @@ -373,8 +373,21 @@ bool ElasticSearch::HTTPSend(CURL *handle) // Some timeout options. These will need more attention later. curl_easy_setopt(handle, CURLOPT_NOSIGNAL, 1); +#if LIBCURL_VERSION_NUM > 0x071002 curl_easy_setopt(handle, CURLOPT_CONNECTTIMEOUT_MS, transfer_timeout); curl_easy_setopt(handle, CURLOPT_TIMEOUT_MS, transfer_timeout*2); +#else + if ( transfer_timeout > 1000 ) + { + curl_easy_setopt(handle, CURLOPT_CONNECTTIMEOUT, transfer_timeout/1000); + curl_easy_setopt(handle, CURLOPT_TIMEOUT, transfer_timeout/2000); + } + else + { + curl_easy_setopt(handle, CURLOPT_CONNECTTIMEOUT, 2); + curl_easy_setopt(handle, CURLOPT_TIMEOUT, 1); + } +#endif curl_easy_setopt(handle, CURLOPT_DNS_CACHE_TIMEOUT, 60*60); CURLcode return_code = curl_easy_perform(handle); From 1ce76da90f4aa032da601e80e339518622272457 Mon Sep 17 00:00:00 2001 From: Matthias Vallentin Date: Thu, 27 Sep 2012 16:25:05 -0700 Subject: [PATCH 2/4] Use second granularity for ElasticSearch timeouts. Since the millisecond resolution cannot be harnessed universally and is not supported by older version of libcurl, we will allow only specifications at the granularity of seconds. This commit also fixes a typing issue that causes that prevented the ElasticSearch timeout to work in the first place: curl_easy_setopt requires a long but was given a uint64_t. --- .../logging/writers/elasticsearch.bro | 5 +++-- src/logging/writers/ElasticSearch.cc | 19 +++---------------- src/logging/writers/ElasticSearch.h | 2 +- 3 files changed, 7 insertions(+), 19 deletions(-) diff --git a/scripts/base/frameworks/logging/writers/elasticsearch.bro b/scripts/base/frameworks/logging/writers/elasticsearch.bro index b0e8fac40e..1cb1c3f83f 100644 --- a/scripts/base/frameworks/logging/writers/elasticsearch.bro +++ b/scripts/base/frameworks/logging/writers/elasticsearch.bro @@ -26,8 +26,9 @@ export { ## e.g. prefix = "bro\_" would create types of bro_dns, bro_software, etc. const type_prefix = "" &redef; - ## The time before an ElasticSearch transfer will timeout. - ## This is not working! + ## The time before an ElasticSearch transfer will timeout. Time + ## specifications less than seconds result in a timeout value of 0, which + ## means "no timeout." const transfer_timeout = 2secs; ## The batch size is the number of messages that will be queued up before diff --git a/src/logging/writers/ElasticSearch.cc b/src/logging/writers/ElasticSearch.cc index 24489314ec..393d52c188 100644 --- a/src/logging/writers/ElasticSearch.cc +++ b/src/logging/writers/ElasticSearch.cc @@ -48,7 +48,7 @@ ElasticSearch::ElasticSearch(WriterFrontend* frontend) : WriterBackend(frontend) last_send = current_time(); failing = false; - transfer_timeout = static_cast(BifConst::LogElasticSearch::transfer_timeout) * 1000; + transfer_timeout = static_cast(BifConst::LogElasticSearch::transfer_timeout); curl_handle = HTTPSetup(); } @@ -373,21 +373,8 @@ bool ElasticSearch::HTTPSend(CURL *handle) // Some timeout options. These will need more attention later. curl_easy_setopt(handle, CURLOPT_NOSIGNAL, 1); -#if LIBCURL_VERSION_NUM > 0x071002 - curl_easy_setopt(handle, CURLOPT_CONNECTTIMEOUT_MS, transfer_timeout); - curl_easy_setopt(handle, CURLOPT_TIMEOUT_MS, transfer_timeout*2); -#else - if ( transfer_timeout > 1000 ) - { - curl_easy_setopt(handle, CURLOPT_CONNECTTIMEOUT, transfer_timeout/1000); - curl_easy_setopt(handle, CURLOPT_TIMEOUT, transfer_timeout/2000); - } - else - { - curl_easy_setopt(handle, CURLOPT_CONNECTTIMEOUT, 2); - curl_easy_setopt(handle, CURLOPT_TIMEOUT, 1); - } -#endif + curl_easy_setopt(handle, CURLOPT_CONNECTTIMEOUT, transfer_timeout); + curl_easy_setopt(handle, CURLOPT_TIMEOUT, transfer_timeout); curl_easy_setopt(handle, CURLOPT_DNS_CACHE_TIMEOUT, 60*60); CURLcode return_code = curl_easy_perform(handle); diff --git a/src/logging/writers/ElasticSearch.h b/src/logging/writers/ElasticSearch.h index 0e88bf3e88..fef0a00ffd 100644 --- a/src/logging/writers/ElasticSearch.h +++ b/src/logging/writers/ElasticSearch.h @@ -68,7 +68,7 @@ private: string path; string index_prefix; - uint64 transfer_timeout; + long transfer_timeout; bool failing; uint64 batch_size; From 4cbf4e3cafb6e4e071970cfeb625f7029354d3d5 Mon Sep 17 00:00:00 2001 From: Bernhard Amann Date: Mon, 1 Oct 2012 13:04:40 -0700 Subject: [PATCH 3/4] Small but important fix for the input framework. BroStrings were constructed without a final \0 - which means that strings read by the input framework are unusable by basically all internal functions (like to_count). the basic test now also checks this. Thanks at Sheharbano for noticing this. --- src/input/Manager.cc | 2 +- .../Baseline/scripts.base.frameworks.input.basic/out | 3 ++- testing/btest/scripts/base/frameworks/input/basic.bro | 8 +++++--- 3 files changed, 8 insertions(+), 5 deletions(-) diff --git a/src/input/Manager.cc b/src/input/Manager.cc index 6eadb3aba8..83e9dc9bc5 100644 --- a/src/input/Manager.cc +++ b/src/input/Manager.cc @@ -2007,7 +2007,7 @@ Val* Manager::ValueToVal(const Value* val, BroType* request_type) case TYPE_STRING: { - BroString *s = new BroString((const u_char*)val->val.string_val.data, val->val.string_val.length, 0); + BroString *s = new BroString((const u_char*)val->val.string_val.data, val->val.string_val.length, 1); return new StringVal(s); } diff --git a/testing/btest/Baseline/scripts.base.frameworks.input.basic/out b/testing/btest/Baseline/scripts.base.frameworks.input.basic/out index ebac1866b6..c456298062 100644 --- a/testing/btest/Baseline/scripts.base.frameworks.input.basic/out +++ b/testing/btest/Baseline/scripts.base.frameworks.input.basic/out @@ -1,5 +1,5 @@ { -[-42] = [b=T, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={ +[-42] = [b=T, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, ns=4242, sc={ 2, 4, 1, @@ -12,3 +12,4 @@ BB }, vc=[10, 20, 30], ve=[]] } +4242 diff --git a/testing/btest/scripts/base/frameworks/input/basic.bro b/testing/btest/scripts/base/frameworks/input/basic.bro index df2ab676b8..faab303534 100644 --- a/testing/btest/scripts/base/frameworks/input/basic.bro +++ b/testing/btest/scripts/base/frameworks/input/basic.bro @@ -8,9 +8,9 @@ @TEST-START-FILE input.log #separator \x09 #path ssh -#fields b i e c p sn a d t iv s sc ss se vc ve f -#types bool int enum count port subnet addr double time interval string table table table vector vector func -T -42 SSH::LOG 21 123 10.0.0.0/24 1.2.3.4 3.14 1315801931.273616 100.000000 hurz 2,4,1,3 CC,AA,BB EMPTY 10,20,30 EMPTY SSH::foo\x0a{ \x0aif (0 < SSH::i) \x0a\x09return (Foo);\x0aelse\x0a\x09return (Bar);\x0a\x0a} +#fields b i e c p sn a d t iv s sc ss se vc ve ns +#types bool int enum count port subnet addr double time interval string table table table vector vector string +T -42 SSH::LOG 21 123 10.0.0.0/24 1.2.3.4 3.14 1315801931.273616 100.000000 hurz 2,4,1,3 CC,AA,BB EMPTY 10,20,30 EMPTY 4242 @TEST-END-FILE @load base/protocols/ssh @@ -37,6 +37,7 @@ type Val: record { t: time; iv: interval; s: string; + ns: string; sc: set[count]; ss: set[string]; se: set[string]; @@ -57,6 +58,7 @@ event bro_init() event Input::update_finished(name: string, source:string) { print outfile, servers; + print outfile, to_count(servers[-42]$ns); # try to actually use a string. If null-termination is wrong this will fail. close(outfile); terminate(); } From b4b7a384dcb038060f3e33fc5bbd36708e8ff1f5 Mon Sep 17 00:00:00 2001 From: Robin Sommer Date: Tue, 2 Oct 2012 12:10:13 -0700 Subject: [PATCH 4/4] Updating submodule(s). [nomail] Closes #889 --- cmake | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmake b/cmake index 2a72c5e08e..125f9a5fa8 160000 --- a/cmake +++ b/cmake @@ -1 +1 @@ -Subproject commit 2a72c5e08e018cf632033af3920432d5f684e130 +Subproject commit 125f9a5fa851381d0350efa41a4d14f27be263a2