From 5e8e12182a10165d8111fb79f6650cdd8f8d6614 Mon Sep 17 00:00:00 2001 From: Bernhard Amann Date: Tue, 5 Mar 2013 16:05:07 -0800 Subject: [PATCH 1/5] add base64-encode functionality and bif. This allows replacing an ugly openssl-call from one of the policy scripts. The openssl call is now replaced with a still-but-less-ugly call to base64_encode. I do not know if I split the Base64 classes in a "smart" way... :) --- scripts/base/protocols/ssl/main.bro | 5 -- .../protocols/ssl/extract-certs-pem.bro | 33 ++++++--- src/Base64.cc | 73 ++++++++++++++++++- src/Base64.h | 26 +++++-- src/bro.bif | 42 +++++++++++ 5 files changed, 158 insertions(+), 21 deletions(-) diff --git a/scripts/base/protocols/ssl/main.bro b/scripts/base/protocols/ssl/main.bro index 2dcbfee5ef..c6c4091a87 100644 --- a/scripts/base/protocols/ssl/main.bro +++ b/scripts/base/protocols/ssl/main.bro @@ -67,11 +67,6 @@ export { ## (especially with large file transfers). const disable_analyzer_after_detection = T &redef; - ## The openssl command line utility. If it's in the path the default - ## value will work, otherwise a full path string can be supplied for the - ## utility. - const openssl_util = "openssl" &redef; - ## The maximum amount of time a script can delay records from being logged. const max_log_delay = 15secs &redef; diff --git a/scripts/policy/protocols/ssl/extract-certs-pem.bro b/scripts/policy/protocols/ssl/extract-certs-pem.bro index 420c60a4fd..bdd7d08586 100644 --- a/scripts/policy/protocols/ssl/extract-certs-pem.bro +++ b/scripts/policy/protocols/ssl/extract-certs-pem.bro @@ -8,10 +8,6 @@ ##! own certificate files and no duplicate checking is done across ##! clusters so each node would log each certificate. ##! -##! - If there is a certificate input based vulnerability found in the -##! openssl command line utility, you could be in trouble because this -##! script uses that utility to convert from DER to PEM certificates. -##! @load base/protocols/ssl @load base/utils/directions-and-hosts @@ -35,15 +31,32 @@ event ssl_established(c: connection) &priority=5 { if ( ! c$ssl?$cert ) return; + if ( ! addr_matches_host(c$id$resp_h, extract_certs_pem) ) return; - + if ( c$ssl$cert_hash in extracted_certs ) # If we already extracted this cert, don't do it again. return; - - add extracted_certs[c$ssl$cert_hash]; - local side = Site::is_local_addr(c$id$resp_h) ? "local" : "remote"; - local cmd = fmt("%s x509 -inform DER -outform PEM >> certs-%s.pem", openssl_util, side); - piped_exec(cmd, c$ssl$cert); + + local filename = Site::is_local_addr(c$id$resp_h) ? "certs-local.pem" : "certs-remote.pem"; + local outfile = open_for_append(filename); + + print outfile, "-----BEGIN CERTIFICATE-----"; + + # encode to base64 and format to fit 50 lines. Otherwise openssl won't like it later. + local lines = split_all(encode_base64(c$ssl$cert), /.{50}/); + local i = 1; + for ( line in lines ) + { + if ( byte_len(lines[i]) > 0 ) + { + print outfile, lines[i]; + } + i+=1; + } + + print outfile, "-----END CERTIFICATE-----"; + print outfile, ""; + close(outfile); } diff --git a/src/Base64.cc b/src/Base64.cc index b0da8ea74c..030f013d6f 100644 --- a/src/Base64.cc +++ b/src/Base64.cc @@ -1,8 +1,59 @@ #include "config.h" #include "Base64.h" +#include int Base64Decoder::default_base64_table[256]; -const string Base64Decoder::default_alphabet = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/"; +const string Base64::default_alphabet = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/"; + +Base64Encoder::Base64Encoder(const string& arg_alphabet) + { + if ( arg_alphabet.size() > 0 ) + { + assert(arg_alphabet.size() == 64); + alphabet = arg_alphabet; + } + else + { + alphabet = default_alphabet; + } + } + +void Base64Encoder::Encode(int len, const unsigned char* data, int* pblen, char** pbuf) + { + int blen; + char *buf; + + if ( ! pbuf ) + reporter->InternalError("nil pointer to encoding result buffer"); + + if ( *pbuf && (*pblen % 4 != 0) ) + reporter->InternalError("Base64 encode buffer not a multiple of 4"); + + if ( *pbuf ) + { + buf = *pbuf; + blen = *pblen; + } + else + { + blen = int( 4 * ceil ( (double) len / 3 ) ); + *pbuf = buf = new char[blen]; + *pblen = blen; + } + + for ( int i = 0, j = 0; (i < len) && ( j < blen ); ) + { + uint32_t bit32 = ( ( i < len ? data[i++] : 0 ) << 16 ) + + ( ( i < len ? data[i++] : 0 & i++ ) << 8 ) + + ( i < len ? data[i++] : 0 & i++ ); + + buf[j++] = alphabet[(bit32 >> 18) & 0x3f]; + buf[j++] = alphabet[(bit32 >> 12) & 0x3f]; + buf[j++] = ( i == (len+2) ) ? '=' : alphabet[(bit32 >> 6) & 0x3f]; + buf[j++] = ( i >= (len+1) ) ? '=' : alphabet[bit32 & 0x3f]; + } + } + int* Base64Decoder::InitBase64Table(const string& alphabet) { @@ -44,6 +95,8 @@ int* Base64Decoder::InitBase64Table(const string& alphabet) return base64_table; } + + Base64Decoder::Base64Decoder(Analyzer* arg_analyzer, const string& alphabet) { base64_table = InitBase64Table(alphabet.size() ? alphabet : default_alphabet); @@ -195,3 +248,21 @@ err: delete [] rbuf; return 0; } + +BroString* encode_base64(const BroString* s, const BroString* a) + { + if ( a && a->Len() != 64 ) + { + reporter->Error("base64 alphabet is not 64 characters: %s", + a->CheckString()); + return 0; + } + + char* outbuf = 0; + int outlen = 0; + Base64Encoder enc; + enc.Encode(s->Len(), (const unsigned char*) s->Bytes(), &outlen, &outbuf); + + return new BroString(1, (u_char*)outbuf, outlen); + } + diff --git a/src/Base64.h b/src/Base64.h index 0e02c94cf0..5b94ec7c2c 100644 --- a/src/Base64.h +++ b/src/Base64.h @@ -11,7 +11,23 @@ // Maybe we should have a base class for generic decoders? -class Base64Decoder { +class Base64 { +public: +protected: + static const string default_alphabet; +}; + + +class Base64Encoder : public Base64 { +public: + Base64Encoder(const string& arg_alphabet = ""); + void Encode(int len, const unsigned char* data, int* blen, char** buf); +protected: + string alphabet; + +}; + +class Base64Decoder : public Base64 { public: // is used for error reporting, and it should be zero when // the decoder is called by the built-in function decode_base64(). @@ -51,19 +67,19 @@ protected: char error_msg[256]; protected: + static int* InitBase64Table(const string& alphabet); + static int default_base64_table[256]; char base64_group[4]; int base64_group_next; int base64_padding; int base64_after_padding; + int* base64_table; int errored; // if true, we encountered an error - skip further processing Analyzer* analyzer; - int* base64_table; - static int* InitBase64Table(const string& alphabet); - static int default_base64_table[256]; - static const string default_alphabet; }; BroString* decode_base64(const BroString* s, const BroString* a = 0); +BroString* encode_base64(const BroString* s, const BroString* a = 0); #endif /* base64_h */ diff --git a/src/bro.bif b/src/bro.bif index 8cea9d9123..3714839ab8 100644 --- a/src/bro.bif +++ b/src/bro.bif @@ -2868,6 +2868,48 @@ function bytestring_to_hexstr%(bytestring: string%): string return new StringVal(hexstr); %} +## Encodes a Base64-encoded string. +## +## s: The string to encode +## +## Returns: The encoded version of *s*. +## +## .. bro:see:: encode_base64_custom +function encode_base64%(s: string%): string + %{ + BroString* t = encode_base64(s->AsString()); + if ( t ) + return new StringVal(t); + else + { + reporter->Error("error in encoding string %s", s->CheckString()); + return new StringVal(""); + } + %} + +## Encodes a Base64-encoded string with a custom alphabet. +## +## s: The string to encode +## +## a: The custom alphabet. The empty string indicates the default alphabet. The +## length of *a* must be 64. For example, a custom alphabet could be +## ``"!#$%&/(),-.:;<>@[]^ `_{|}~abcdefghijklmnopqrstuvwxyz0123456789+?"``. +## +## Returns: The encoded version of *s*. +## +## .. bro:see:: encode_base64 +function encode_base64_custom%(s: string, a: string%): string + %{ + BroString* t = encode_base64(s->AsString(), a->AsString()); + if ( t ) + return new StringVal(t); + else + { + reporter->Error("error in encoding string %s", s->CheckString()); + return new StringVal(""); + } + %} + ## Decodes a Base64-encoded string. ## ## s: The Base64-encoded string. From cfada6167223ef20630ea7b6ca5cd771ee3dd33c Mon Sep 17 00:00:00 2001 From: Bernhard Amann Date: Wed, 6 Mar 2013 13:30:13 -0800 Subject: [PATCH 2/5] and modernize script. thanks Seth. --- scripts/policy/protocols/ssl/extract-certs-pem.bro | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/policy/protocols/ssl/extract-certs-pem.bro b/scripts/policy/protocols/ssl/extract-certs-pem.bro index bdd7d08586..f48d4bc369 100644 --- a/scripts/policy/protocols/ssl/extract-certs-pem.bro +++ b/scripts/policy/protocols/ssl/extract-certs-pem.bro @@ -49,7 +49,7 @@ event ssl_established(c: connection) &priority=5 local i = 1; for ( line in lines ) { - if ( byte_len(lines[i]) > 0 ) + if ( |lines[i]| > 0 ) { print outfile, lines[i]; } From 2b28c3a578166da59c42494beede75707acbfc9b Mon Sep 17 00:00:00 2001 From: Bernhard Amann Date: Tue, 12 Mar 2013 09:27:59 -0700 Subject: [PATCH 3/5] re-unify classes --- src/Base64.cc | 37 +++++++++++++++++++------------------ src/Base64.h | 25 ++++++------------------- 2 files changed, 25 insertions(+), 37 deletions(-) diff --git a/src/Base64.cc b/src/Base64.cc index 030f013d6f..1826ede93a 100644 --- a/src/Base64.cc +++ b/src/Base64.cc @@ -3,22 +3,9 @@ #include int Base64Decoder::default_base64_table[256]; -const string Base64::default_alphabet = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/"; +const string Base64Decoder::default_alphabet = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/"; -Base64Encoder::Base64Encoder(const string& arg_alphabet) - { - if ( arg_alphabet.size() > 0 ) - { - assert(arg_alphabet.size() == 64); - alphabet = arg_alphabet; - } - else - { - alphabet = default_alphabet; - } - } - -void Base64Encoder::Encode(int len, const unsigned char* data, int* pblen, char** pbuf) +void Base64Decoder::Encode(int len, const unsigned char* data, int* pblen, char** pbuf) { int blen; char *buf; @@ -97,9 +84,19 @@ int* Base64Decoder::InitBase64Table(const string& alphabet) -Base64Decoder::Base64Decoder(Analyzer* arg_analyzer, const string& alphabet) +Base64Decoder::Base64Decoder(Analyzer* arg_analyzer, const string& arg_alphabet) { - base64_table = InitBase64Table(alphabet.size() ? alphabet : default_alphabet); + if ( arg_alphabet.size() > 0 ) + { + assert(arg_alphabet.size() == 64); + alphabet = arg_alphabet; + } + else + { + alphabet = default_alphabet; + } + + base64_table = 0; base64_group_next = 0; base64_padding = base64_after_padding = 0; errored = 0; @@ -117,6 +114,10 @@ int Base64Decoder::Decode(int len, const char* data, int* pblen, char** pbuf) int blen; char* buf; + // initialization of table on first_time call of Decode + if ( base64_table == 0 ) + base64_table = InitBase64Table(alphabet); + if ( ! pbuf ) reporter->InternalError("nil pointer to decoding result buffer"); @@ -260,7 +261,7 @@ BroString* encode_base64(const BroString* s, const BroString* a) char* outbuf = 0; int outlen = 0; - Base64Encoder enc; + Base64Decoder enc(0); enc.Encode(s->Len(), (const unsigned char*) s->Bytes(), &outlen, &outbuf); return new BroString(1, (u_char*)outbuf, outlen); diff --git a/src/Base64.h b/src/Base64.h index 5b94ec7c2c..00bd208f08 100644 --- a/src/Base64.h +++ b/src/Base64.h @@ -10,27 +10,10 @@ #include "Analyzer.h" // Maybe we should have a base class for generic decoders? - -class Base64 { -public: -protected: - static const string default_alphabet; -}; - - -class Base64Encoder : public Base64 { -public: - Base64Encoder(const string& arg_alphabet = ""); - void Encode(int len, const unsigned char* data, int* blen, char** buf); -protected: - string alphabet; - -}; - -class Base64Decoder : public Base64 { +class Base64Decoder { public: // is used for error reporting, and it should be zero when - // the decoder is called by the built-in function decode_base64(). + // the decoder is called by the built-in function decode_base64() or encode_base64(). // Empty alphabet indicates the default base64 alphabet. Base64Decoder(Analyzer* analyzer, const string& alphabet = ""); ~Base64Decoder(); @@ -46,6 +29,7 @@ public: // is not enough output buffer space. int Decode(int len, const char* data, int* blen, char** buf); + void Encode(int len, const unsigned char* data, int* blen, char** buf); int Done(int* pblen, char** pbuf); int HasData() const { return base64_group_next != 0; } @@ -67,6 +51,9 @@ protected: char error_msg[256]; protected: + static const string default_alphabet; + string alphabet; + static int* InitBase64Table(const string& alphabet); static int default_base64_table[256]; char base64_group[4]; From a5161783ef1f043140fe75323de0de098218b9e8 Mon Sep 17 00:00:00 2001 From: Bernhard Amann Date: Tue, 12 Mar 2013 09:33:49 -0700 Subject: [PATCH 4/5] and add bae64 bif tests. --- src/Base64.cc | 2 +- testing/btest/Baseline/bifs.encode_base64/out | 6 ++++++ testing/btest/bifs/encode_base64.bro | 14 ++++++++++++++ 3 files changed, 21 insertions(+), 1 deletion(-) create mode 100644 testing/btest/Baseline/bifs.encode_base64/out create mode 100644 testing/btest/bifs/encode_base64.bro diff --git a/src/Base64.cc b/src/Base64.cc index 1826ede93a..628d872c07 100644 --- a/src/Base64.cc +++ b/src/Base64.cc @@ -261,7 +261,7 @@ BroString* encode_base64(const BroString* s, const BroString* a) char* outbuf = 0; int outlen = 0; - Base64Decoder enc(0); + Base64Decoder enc(0, a ? a->CheckString() : ""); enc.Encode(s->Len(), (const unsigned char*) s->Bytes(), &outlen, &outbuf); return new BroString(1, (u_char*)outbuf, outlen); diff --git a/testing/btest/Baseline/bifs.encode_base64/out b/testing/btest/Baseline/bifs.encode_base64/out new file mode 100644 index 0000000000..84c2c98264 --- /dev/null +++ b/testing/btest/Baseline/bifs.encode_base64/out @@ -0,0 +1,6 @@ +YnJv +YnJv +}n-v +cGFkZGluZw== +cGFkZGluZzE= +cGFkZGluZzEy diff --git a/testing/btest/bifs/encode_base64.bro b/testing/btest/bifs/encode_base64.bro new file mode 100644 index 0000000000..a351392bb5 --- /dev/null +++ b/testing/btest/bifs/encode_base64.bro @@ -0,0 +1,14 @@ +# @TEST-EXEC: bro -b %INPUT >out +# @TEST-EXEC: btest-diff out + +global default_alphabet: string = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/"; + +global my_alphabet: string = "!#$%&/(),-.:;<>@[]^ `_{|}~abcdefghijklmnopqrstuvwxyz0123456789+?"; + +print encode_base64("bro"); +print encode_base64_custom("bro", default_alphabet); +print encode_base64_custom("bro", my_alphabet); + +print encode_base64("padding"); +print encode_base64("padding1"); +print encode_base64("padding12"); From 457ce10e99549b93a01214ba776be45183aa0548 Mon Sep 17 00:00:00 2001 From: Bernhard Amann Date: Wed, 13 Mar 2013 00:34:15 -0700 Subject: [PATCH 5/5] and re-enable caching of extracted certs I kind of deleted the line by accident... --- scripts/policy/protocols/ssl/extract-certs-pem.bro | 1 + 1 file changed, 1 insertion(+) diff --git a/scripts/policy/protocols/ssl/extract-certs-pem.bro b/scripts/policy/protocols/ssl/extract-certs-pem.bro index f48d4bc369..c64e8f86dc 100644 --- a/scripts/policy/protocols/ssl/extract-certs-pem.bro +++ b/scripts/policy/protocols/ssl/extract-certs-pem.bro @@ -39,6 +39,7 @@ event ssl_established(c: connection) &priority=5 # If we already extracted this cert, don't do it again. return; + add extracted_certs[c$ssl$cert_hash]; local filename = Site::is_local_addr(c$id$resp_h) ? "certs-local.pem" : "certs-remote.pem"; local outfile = open_for_append(filename);