Mirror/zeek
1
0
Fork 0
mirror of https://github.com/zeek/zeek.git synced 2025-10-08 17:48:21 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 3

Merge remote-tracking branch 'origin/master' into topic/seth/files-tracking

Browse source 
Conflicts:
	src/Reassem.cc
	src/Reassem.h
	src/analyzer/protocol/tcp/TCP_Reassembler.cc
	testing/btest/Baseline/scripts.base.frameworks.file-analysis.bifs.set_timeout_interval/bro..stdout
	testing/btest/Baseline/scripts.base.frameworks.file-analysis.http.partial-content/b.out
	testing/btest/Baseline/scripts.base.frameworks.file-analysis.http.partial-content/c.out
	testing/btest/Baseline/scripts.base.frameworks.file-analysis.logging/files.log
This commit is contained in:
Seth Hall 2014-05-17 02:12:52 -04:00
commit fb0a658a7c
658 changed files with 21875 additions and 5792 deletions
Download patch file Download diff file Expand all files Collapse all files

45
src/file_analysis/File.cc
View file

@ -10,6 +10,7 @@
#include "Val.h"
#include "Type.h"
#include "Event.h"
#include "RuleMatcher.h"
#include "analyzer/Analyzer.h"
#include "analyzer/Manager.h"
@ -52,6 +53,7 @@ int File::timeout_interval_idx = -1;
int File::bof_buffer_size_idx = -1;
int File::bof_buffer_idx = -1;
int File::mime_type_idx = -1;
int File::mime_types_idx = -1;
void File::StaticInit()
{
@ -72,6 +74,7 @@ void File::StaticInit()
bof_buffer_size_idx = Idx("bof_buffer_size");
bof_buffer_idx = Idx("bof_buffer");
mime_type_idx = Idx("mime_type");
mime_types_idx = Idx("mime_types");
}
File::File(const string& file_id, Connection* conn, analyzer::Tag tag,
@ -104,7 +107,6 @@ File::~File()
DBG_LOG(DBG_FILE_ANALYSIS, "Destroying File object %s", id.c_str());
Unref(val);
// Queue may not be empty in the case where only content gaps were seen.
while ( ! fonc_queue.empty() )
{
delete_vals(fonc_queue.front().second);
@ -284,20 +286,18 @@ bool File::BufferBOF(const u_char* data, uint64 len)
bool File::DetectMIME(const u_char* data, uint64 len)
{
const char* mime = bro_magic_buffer(magic_mime_cookie, data, len);
RuleMatcher::MIME_Matches matches;
len = min(len, LookupFieldDefaultCount(bof_buffer_size_idx));
file_mgr->DetectMIME(data, len, &matches);
if ( mime )
{
const char* mime_end = strchr(mime, ';');
if ( matches.empty() )
return false;
if ( mime_end )
// strip off charset
val->Assign(mime_type_idx, new StringVal(mime_end - mime, mime));
else
val->Assign(mime_type_idx, new StringVal(mime));
}
val->Assign(mime_type_idx,
new StringVal(*(matches.begin()->second.begin())));
val->Assign(mime_types_idx, file_analysis::GenMIMEMatchesVal(matches));
return mime;
return true;
}
void File::EnableReassembly()
@ -520,20 +520,27 @@ void File::FileEvent(EventHandlerPtr h)
FileEvent(h, vl);
}
static void flush_file_event_queue(queue<pair<EventHandlerPtr, val_list*> >& q)
{
while ( ! q.empty() )
{
pair<EventHandlerPtr, val_list*> p = q.front();
mgr.QueueEvent(p.first, p.second);
q.pop();
}
}
void File::FileEvent(EventHandlerPtr h, val_list* vl)
{
if ( h == file_state_remove )
flush_file_event_queue(fonc_queue);
mgr.QueueEvent(h, vl);
if ( h == file_new )
{
did_file_new_event = true;
while ( ! fonc_queue.empty() )
{
pair<EventHandlerPtr, val_list*> p = fonc_queue.front();
mgr.QueueEvent(p.first, p.second);
fonc_queue.pop();
}
flush_file_event_queue(fonc_queue);
}
if ( h == file_new || h == file_timeout || h == file_extraction_limit )

6
src/file_analysis/File.h
View file

@ -229,11 +229,12 @@ protected:
void ReplayBOF();
/**
* Does mime type detection and assigns type (if available) to \c mime_type
* Does mime type detection via file magic signatures and assigns
* strongest matching mime type (if available) to \c mime_type
* field in #val.
* @param data pointer to a chunk of file data.
* @param len number of bytes in the data chunk.
* @return whether mime type was available.
* @return whether a mime type match was found.
*/
bool DetectMIME(const u_char* data, uint64 len);
@ -314,6 +315,7 @@ protected:
static int bof_buffer_size_idx;
static int bof_buffer_idx;
static int mime_type_idx;
static int mime_types_idx;
};
} // namespace file_analysis

6
src/file_analysis/FileReassembler.cc
View file

@ -7,7 +7,7 @@ namespace file_analysis {
class File;
FileReassembler::FileReassembler(File *f, int starting_offset)
FileReassembler::FileReassembler(File *f, uint64 starting_offset)
: Reassembler(starting_offset), the_file(f)
{
}
@ -35,12 +35,12 @@ void FileReassembler::BlockInserted(DataBlock* start_block)
}
}
void FileReassembler::Undelivered(int up_to_seq)
void FileReassembler::Undelivered(uint64 up_to_seq)
{
// Not doing anything here yet.
}
void FileReassembler::Overlap(const u_char* b1, const u_char* b2, int n)
void FileReassembler::Overlap(const u_char* b1, const u_char* b2, uint64 n)
{
// Not doing anything here yet.
}

6
src/file_analysis/FileReassembler.h
View file

@ -17,7 +17,7 @@ class File;
class FileReassembler : public Reassembler {
public:
FileReassembler(File* f, int starting_offset);
FileReassembler(File* f, uint64 starting_offset);
virtual ~FileReassembler();
void Done();
@ -32,9 +32,9 @@ protected:
DECLARE_SERIAL(FileReassembler);
void Undelivered(int up_to_seq);
void Undelivered(uint64 up_to_seq);
void BlockInserted(DataBlock* b);
void Overlap(const u_char* b1, const u_char* b2, int n);
void Overlap(const u_char* b1, const u_char* b2, uint64 n);
unsigned int had_gap:1;
unsigned int did_EOF:1;

188
src/file_analysis/Manager.cc
View file

@ -20,13 +20,30 @@ string Manager::salt;
Manager::Manager()
: plugin::ComponentManager<file_analysis::Tag,
file_analysis::Component>("Files")
file_analysis::Component>("Files"),
id_map(), ignored(), current_file_id(), magic_state()
{
}
Manager::~Manager()
{
Terminate();
// Have to assume that too much of Bro has been shutdown by this point
// to do anything more than reclaim memory.
File* f;
bool* b;
IterCookie* it = id_map.InitForIteration();
while ( (f = id_map.NextEntry(it)) )
delete f;
it = ignored.InitForIteration();
while( (b = ignored.NextEntry(it)) )
delete b;
delete magic_state;
}
void Manager::InitPreScript()
@ -42,15 +59,30 @@ void Manager::InitPostScript()
{
}
void Manager::InitMagic()
{
delete magic_state;
magic_state = rule_matcher->InitFileMagic();
}
void Manager::Terminate()
{
vector<string> keys;
for ( IDMap::iterator it = id_map.begin(); it != id_map.end(); ++it )
keys.push_back(it->first);
IterCookie* it = id_map.InitForIteration();
HashKey* key;
while ( id_map.NextEntry(key, it) )
{
keys.push_back(string(static_cast<const char*>(key->Key()),
key->Size()));
delete key;
}
for ( size_t i = 0; i < keys.size(); ++i )
Timeout(keys[i], true);
mgr.Drain();
}
string Manager::HashHandle(const string& handle) const
@ -75,36 +107,47 @@ void Manager::SetHandle(const string& handle)
current_file_id = HashHandle(handle);
}
void Manager::DataIn(const u_char* data, uint64 len, uint64 offset,
analyzer::Tag tag, Connection* conn, bool is_orig)
string Manager::DataIn(const u_char* data, uint64 len, uint64 offset,
analyzer::Tag tag, Connection* conn, bool is_orig,
const string& precomputed_id)
{
GetFileHandle(tag, conn, is_orig);
File* file = GetFile(current_file_id, conn, tag, is_orig);
string id = precomputed_id.empty() ? GetFileID(tag, conn, is_orig) : precomputed_id;
File* file = GetFile(id, conn, tag, is_orig);
if ( ! file )
return;
return "";
file->DataIn(data, len, offset);
if ( file->IsComplete() )
{
RemoveFile(file->GetID());
return "";
}
return id;
}
void Manager::DataIn(const u_char* data, uint64 len, analyzer::Tag tag,
Connection* conn, bool is_orig)
string Manager::DataIn(const u_char* data, uint64 len, analyzer::Tag tag,
Connection* conn, bool is_orig, const string& precomputed_id)
{
GetFileHandle(tag, conn, is_orig);
string id = precomputed_id.empty() ? GetFileID(tag, conn, is_orig) : precomputed_id;
// Sequential data input shouldn't be going over multiple conns, so don't
// do the check to update connection set.
File* file = GetFile(current_file_id, conn, tag, is_orig, false);
File* file = GetFile(id, conn, tag, is_orig, false);
if ( ! file )
return;
return "";
file->DataIn(data, len);
if ( file->IsComplete() )
{
RemoveFile(file->GetID());
return "";
}
return id;
}
void Manager::DataIn(const u_char* data, uint64 len, const string& file_id,
@ -133,8 +176,7 @@ void Manager::EndOfFile(analyzer::Tag tag, Connection* conn)
void Manager::EndOfFile(analyzer::Tag tag, Connection* conn, bool is_orig)
{
// Don't need to create a file if we're just going to remove it right away.
GetFileHandle(tag, conn, is_orig);
RemoveFile(current_file_id);
RemoveFile(GetFileID(tag, conn, is_orig));
}
void Manager::EndOfFile(const string& file_id)
@ -142,31 +184,37 @@ void Manager::EndOfFile(const string& file_id)
RemoveFile(file_id);
}
void Manager::Gap(uint64 offset, uint64 len, analyzer::Tag tag,
Connection* conn, bool is_orig)
string Manager::Gap(uint64 offset, uint64 len, analyzer::Tag tag,
Connection* conn, bool is_orig, const string& precomputed_id)
{
GetFileHandle(tag, conn, is_orig);
File* file = GetFile(current_file_id, conn, tag, is_orig);
string id = precomputed_id.empty() ? GetFileID(tag, conn, is_orig) : precomputed_id;
File* file = GetFile(id, conn, tag, is_orig);
if ( ! file )
return;
return "";
file->Gap(offset, len);
return id;
}
void Manager::SetSize(uint64 size, analyzer::Tag tag, Connection* conn,
bool is_orig)
string Manager::SetSize(uint64 size, analyzer::Tag tag, Connection* conn,
bool is_orig, const string& precomputed_id)
{
GetFileHandle(tag, conn, is_orig);
File* file = GetFile(current_file_id, conn, tag, is_orig);
string id = precomputed_id.empty() ? GetFileID(tag, conn, is_orig) : precomputed_id;
File* file = GetFile(id, conn, tag, is_orig);
if ( ! file )
return;
return "";
file->SetTotalBytes(size);
if ( file->IsComplete() )
{
RemoveFile(file->GetID());
return "";
}
return id;
}
bool Manager::SetTimeoutInterval(const string& file_id, double interval) const
@ -258,11 +306,12 @@ File* Manager::GetFile(const string& file_id, Connection* conn,
if ( IsIgnored(file_id) )
return 0;
File* rval = id_map[file_id];
File* rval = id_map.Lookup(file_id.c_str());
if ( ! rval )
{
rval = id_map[file_id] = new File(file_id, conn, tag, is_orig);
rval = new File(file_id, conn, tag, is_orig);
id_map.Insert(file_id.c_str(), rval);
rval->ScheduleInactivityTimer();
if ( IsIgnored(file_id) )
@ -281,12 +330,7 @@ File* Manager::GetFile(const string& file_id, Connection* conn,
File* Manager::LookupFile(const string& file_id) const
{
IDMap::const_iterator it = id_map.find(file_id);
if ( it == id_map.end() )
return 0;
return it->second;
return id_map.Lookup(file_id.c_str());
}
void Manager::Timeout(const string& file_id, bool is_terminating)
@ -317,48 +361,49 @@ void Manager::Timeout(const string& file_id, bool is_terminating)
bool Manager::IgnoreFile(const string& file_id)
{
if ( id_map.find(file_id) == id_map.end() )
if ( ! id_map.Lookup(file_id.c_str()) )
return false;
DBG_LOG(DBG_FILE_ANALYSIS, "Ignore FileID %s", file_id.c_str());
ignored.insert(file_id);
delete ignored.Insert(file_id.c_str(), new bool);
return true;
}
bool Manager::RemoveFile(const string& file_id)
{
IDMap::iterator it = id_map.find(file_id);
HashKey key(file_id.c_str());
// Can't remove from the dictionary/map right away as invoking EndOfFile
// may cause some events to be executed which actually depend on the file
// still being in the dictionary/map.
File* f = static_cast<File*>(id_map.Lookup(&key));
if ( it == id_map.end() )
if ( ! f )
return false;
DBG_LOG(DBG_FILE_ANALYSIS, "Remove FileID %s", file_id.c_str());
it->second->EndOfFile();
delete it->second;
id_map.erase(file_id);
ignored.erase(file_id);
f->EndOfFile();
delete f;
id_map.Remove(&key);
delete static_cast<bool*>(ignored.Remove(&key));
return true;
}
bool Manager::IsIgnored(const string& file_id)
{
return ignored.find(file_id) != ignored.end();
return ignored.Lookup(file_id.c_str()) != 0;
}
void Manager::GetFileHandle(analyzer::Tag tag, Connection* c, bool is_orig)
string Manager::GetFileID(analyzer::Tag tag, Connection* c, bool is_orig)
{
current_file_id.clear();
if ( IsDisabled(tag) )
return;
return "";
if ( ! get_file_handle )
return;
return "";
EnumVal* tagval = tag.AsEnumVal();
Ref(tagval);
@ -370,6 +415,7 @@ void Manager::GetFileHandle(analyzer::Tag tag, Connection* c, bool is_orig)
mgr.QueueEvent(get_file_handle, vl);
mgr.Drain(); // need file handle immediately so we don't have to buffer data
return current_file_id;
}
bool Manager::IsDisabled(analyzer::Tag tag)
@ -411,3 +457,47 @@ Analyzer* Manager::InstantiateAnalyzer(Tag tag, RecordVal* args, File* f) const
return c->Factory()(args, f);
}
RuleMatcher::MIME_Matches* Manager::DetectMIME(const u_char* data, uint64 len,
RuleMatcher::MIME_Matches* rval) const
{
if ( ! magic_state )
reporter->InternalError("file magic signature state not initialized");
rval = rule_matcher->Match(magic_state, data, len, rval);
rule_matcher->ClearFileMagicState(magic_state);
return rval;
}
string Manager::DetectMIME(const u_char* data, uint64 len) const
{
RuleMatcher::MIME_Matches matches;
DetectMIME(data, len, &matches);
if ( matches.empty() )
return "";
return *(matches.begin()->second.begin());
}
VectorVal* file_analysis::GenMIMEMatchesVal(const RuleMatcher::MIME_Matches& m)
{
VectorVal* rval = new VectorVal(mime_matches);
for ( RuleMatcher::MIME_Matches::const_iterator it = m.begin();
it != m.end(); ++it )
{
RecordVal* element = new RecordVal(mime_match);
for ( set<string>::const_iterator it2 = it->second.begin();
it2 != it->second.end(); ++it2 )
{
element->Assign(0, new Val(it->first, TYPE_INT));
element->Assign(1, new StringVal(*it2));
}
rval->Assign(rval->Size(), element);
}
return rval;
}

102
src/file_analysis/Manager.h
View file

@ -4,16 +4,16 @@
#define FILE_ANALYSIS_MANAGER_H
#include <string>
#include <map>
#include <set>
#include <queue>
#include "Dict.h"
#include "Net.h"
#include "Conn.h"
#include "Val.h"
#include "Analyzer.h"
#include "Timer.h"
#include "EventHandler.h"
#include "RuleMatcher.h"
#include "File.h"
#include "FileTimer.h"
@ -26,6 +26,9 @@
namespace file_analysis {
declare(PDict,bool);
declare(PDict,File);
/**
* Main entry point for interacting with file analysis.
*/
@ -54,6 +57,12 @@ public:
*/
void InitPostScript();
/**
* Initializes the state required to match against file magic signatures
* for MIME type identification.
*/
void InitMagic();
/**
* Times out any active file analysis to prepare for shutdown.
*/
@ -82,9 +91,17 @@ public:
* @param conn network connection over which the file data is transferred.
* @param is_orig true if the file is being sent from connection originator
* or false if is being sent in the opposite direction.
* @param precomputed_file_id may be set to a previous return value in order to
* bypass costly file handle lookups.
* @return a unique file ID string which, in certain contexts, may be
* cached and passed back in to a subsequent function call in order
* to avoid costly file handle lookups (which have to go through
* the \c get_file_handle script-layer event). An empty string
* indicates the associate file is not going to be analyzed further.
*/
void DataIn(const u_char* data, uint64 len, uint64 offset,
analyzer::Tag tag, Connection* conn, bool is_orig);
std::string DataIn(const u_char* data, uint64 len, uint64 offset,
analyzer::Tag tag, Connection* conn, bool is_orig,
const std::string& precomputed_file_id = "");
/**
* Pass in sequential file data.
@ -94,9 +111,17 @@ public:
* @param conn network connection over which the file data is transferred.
* @param is_orig true if the file is being sent from connection originator
* or false if is being sent in the opposite direction.
* @param precomputed_file_id may be set to a previous return value in order to
* bypass costly file handle lookups.
* @return a unique file ID string which, in certain contexts, may be
* cached and passed back in to a subsequent function call in order
* to avoid costly file handle lookups (which have to go through
* the \c get_file_handle script-layer event). An empty string
* indicates the associated file is not going to be analyzed further.
*/
void DataIn(const u_char* data, uint64 len, analyzer::Tag tag,
Connection* conn, bool is_orig);
std::string DataIn(const u_char* data, uint64 len, analyzer::Tag tag,
Connection* conn, bool is_orig,
const std::string& precomputed_file_id = "");
/**
* Pass in sequential file data from external source (e.g. input framework).
@ -140,9 +165,17 @@ public:
* @param conn network connection over which the file data is transferred.
* @param is_orig true if the file is being sent from connection originator
* or false if is being sent in the opposite direction.
* @param precomputed_file_id may be set to a previous return value in order to
* bypass costly file handle lookups.
* @return a unique file ID string which, in certain contexts, may be
* cached and passed back in to a subsequent function call in order
* to avoid costly file handle lookups (which have to go through
* the \c get_file_handle script-layer event). An empty string
* indicates the associate file is not going to be analyzed further.
*/
void Gap(uint64 offset, uint64 len, analyzer::Tag tag, Connection* conn,
bool is_orig);
std::string Gap(uint64 offset, uint64 len, analyzer::Tag tag,
Connection* conn, bool is_orig,
const std::string& precomputed_file_id = "");
/**
* Provide the expected number of bytes that comprise a file.
@ -151,9 +184,16 @@ public:
* @param conn network connection over which the file data is transferred.
* @param is_orig true if the file is being sent from connection originator
* or false if is being sent in the opposite direction.
* @param precomputed_file_id may be set to a previous return value in order to
* bypass costly file handle lookups.
* @return a unique file ID string which, in certain contexts, may be
* cached and passed back in to a subsequent function call in order
* to avoid costly file handle lookups (which have to go through
* the \c get_file_handle script-layer event). An empty string
* indicates the associate file is not going to be analyzed further.
*/
void SetSize(uint64 size, analyzer::Tag tag, Connection* conn,
bool is_orig);
std::string SetSize(uint64 size, analyzer::Tag tag, Connection* conn,
bool is_orig, const std::string& precomputed_file_id = "");
/**
* Starts ignoring a file, which will finally be removed from internal
@ -239,11 +279,34 @@ public:
*/
Analyzer* InstantiateAnalyzer(Tag tag, RecordVal* args, File* f) const;
/**
* Returns a set of all matching MIME magic signatures for a given
* chunk of data.
* @param data A chunk of bytes to match magic MIME signatures against.
* @param len The number of bytes in \a data.
* @param rval An optional pre-existing structure in which to insert
* new matches. If it's a null pointer, an object is
* allocated and returned from the method.
* @return Set of all matching file magic signatures, which may be
* an object allocated by the method if \a rval is a null pointer.
*/
RuleMatcher::MIME_Matches* DetectMIME(const u_char* data, uint64 len,
RuleMatcher::MIME_Matches* rval) const;
/**
* Returns the strongest MIME magic signature match for a given data chunk.
* @param data A chunk of bytes to match magic MIME signatures against.
* @param len The number of bytes in \a data.
* @returns The MIME type string of the strongest file magic signature
* match, or an empty string if nothing matched.
*/
std::string DetectMIME(const u_char* data, uint64 len) const;
protected:
friend class FileTimer;
typedef set<string> IDSet;
typedef map<string, File*> IDMap;
typedef PDict(bool) IDSet;
typedef PDict(File) IDMap;
/**
* Create a new file to be analyzed or retrieve an existing one.
@ -298,8 +361,10 @@ protected:
* @param conn network connection over which the file is transferred.
* @param is_orig true if the file is being sent from connection originator
* or false if is being sent in the opposite direction.
* @return #current_file_id, which is a hash of a unique file handle string
* set by a \c get_file_handle event handler.
*/
void GetFileHandle(analyzer::Tag tag, Connection* c, bool is_orig);
std::string GetFileID(analyzer::Tag tag, Connection* c, bool is_orig);
/**
* Check if analysis is available for files transferred over a given
@ -313,14 +378,21 @@ protected:
private:
IDMap id_map; /**< Map file ID to file_analysis::File records. */
IDSet ignored; /**< Ignored files. Will be finally removed on EOF. */
PDict(File) id_map; /**< Map file ID to file_analysis::File records. */
PDict(bool) ignored; /**< Ignored files. Will be finally removed on EOF. */
string current_file_id; /**< Hash of what get_file_handle event sets. */
RuleFileMagicState* magic_state; /**< File magic signature match state. */
static TableVal* disabled; /**< Table of disabled analyzers. */
static string salt; /**< A salt added to file handles before hashing. */
};
/**
* Returns a script-layer value corresponding to the \c mime_matches type.
* @param m The MIME match information with which to populate the value.
*/
VectorVal* GenMIMEMatchesVal(const RuleMatcher::MIME_Matches& m);
} // namespace file_analysis
extern file_analysis::Manager* file_mgr;

1
src/file_analysis/analyzer/CMakeLists.txt
View file

@ -2,3 +2,4 @@ add_subdirectory(data_event)
add_subdirectory(extract)
add_subdirectory(hash)
add_subdirectory(unified2)
add_subdirectory(x509)

10
src/file_analysis/analyzer/unified2/events.bif
View file

@ -1,7 +1,17 @@
## Abstract all of the various Unified2 event formats into
## a single event.
##
## f: The file.
##
## ev: TODO.
##
event unified2_event%(f: fa_file, ev: Unified2::IDSEvent%);
## The Unified2 packet format event.
##
## f: The file.
##
## pkt: TODO.
##
event unified2_packet%(f: fa_file, pkt: Unified2::Packet%);

10
src/file_analysis/analyzer/x509/CMakeLists.txt Normal file
View file

@ -0,0 +1,10 @@
include(BroPlugin)
include_directories(BEFORE ${CMAKE_CURRENT_SOURCE_DIR}
${CMAKE_CURRENT_BINARY_DIR})
bro_plugin_begin(Bro X509)
bro_plugin_cc(X509.cc Plugin.cc)
bro_plugin_bif(events.bif types.bif functions.bif)
bro_plugin_end()

11
src/file_analysis/analyzer/x509/Plugin.cc Normal file
View file

@ -0,0 +1,11 @@
#include "plugin/Plugin.h"
#include "X509.h"
BRO_PLUGIN_BEGIN(Bro, X509)
BRO_PLUGIN_DESCRIPTION("X509 certificate parser");
BRO_PLUGIN_FILE_ANALYZER("X509", X509);
BRO_PLUGIN_BIF_FILE(events);
BRO_PLUGIN_BIF_FILE(types);
BRO_PLUGIN_BIF_FILE(functions);
BRO_PLUGIN_END

633
src/file_analysis/analyzer/x509/X509.cc Normal file
View file

@ -0,0 +1,633 @@
// See the file "COPYING" in the main distribution directory for copyright.
#include <string>
#include "X509.h"
#include "Event.h"
#include "events.bif.h"
#include "types.bif.h"
#include "file_analysis/Manager.h"
#include <openssl/x509.h>
#include <openssl/x509v3.h>
#include <openssl/asn1.h>
#include <openssl/opensslconf.h>
using namespace file_analysis;
IMPLEMENT_SERIAL(X509Val, SER_X509_VAL);
file_analysis::X509::X509(RecordVal* args, file_analysis::File* file)
: file_analysis::Analyzer(file_mgr->GetComponentTag("X509"), args, file)
{
cert_data.clear();
}
bool file_analysis::X509::DeliverStream(const u_char* data, uint64 len)
{
// just add it to the data we have so far, since we cannot do anything else anyways...
cert_data.append(reinterpret_cast<const char*>(data), len);
return true;
}
bool file_analysis::X509::Undelivered(uint64 offset, uint64 len)
{
return false;
}
bool file_analysis::X509::EndOfFile()
{
// ok, now we can try to parse the certificate with openssl. Should
// be rather straightforward...
const unsigned char* cert_char = reinterpret_cast<const unsigned char*>(cert_data.data());
::X509* ssl_cert = d2i_X509(NULL, &cert_char, cert_data.size());
if ( ! ssl_cert )
{
reporter->Weird(fmt("Could not parse X509 certificate (fuid %s)", GetFile()->GetID().c_str()));
return false;
}
X509Val* cert_val = new X509Val(ssl_cert); // cert_val takes ownership of ssl_cert
RecordVal* cert_record = ParseCertificate(cert_val); // parse basic information into record
// and send the record on to scriptland
val_list* vl = new val_list();
vl->append(GetFile()->GetVal()->Ref());
vl->append(cert_val->Ref());
vl->append(cert_record->Ref()); // we Ref it here, because we want to keep a copy around for now...
mgr.QueueEvent(x509_certificate, vl);
// after parsing the certificate - parse the extensions...
int num_ext = X509_get_ext_count(ssl_cert);
for ( int k = 0; k < num_ext; ++k )
{
X509_EXTENSION* ex = X509_get_ext(ssl_cert, k);
if ( ! ex )
continue;
ParseExtension(ex);
}
// X509_free(ssl_cert); We do _not_ free the certificate here. It is refcounted
// inside the X509Val that is sent on in the cert record to scriptland.
//
// The certificate will be freed when the last X509Val is Unref'd.
Unref(cert_record); // Unref the RecordVal that we kept around from ParseCertificate
Unref(cert_val); // Same for cert_val
return false;
}
RecordVal* file_analysis::X509::ParseCertificate(X509Val* cert_val)
{
::X509* ssl_cert = cert_val->GetCertificate();
char buf[256]; // we need a buffer for some of the openssl functions
memset(buf, 0, sizeof(buf));
RecordVal* pX509Cert = new RecordVal(BifType::Record::X509::Certificate);
BIO *bio = BIO_new(BIO_s_mem());
pX509Cert->Assign(0, new Val((uint64) X509_get_version(ssl_cert) + 1, TYPE_COUNT));
i2a_ASN1_INTEGER(bio, X509_get_serialNumber(ssl_cert));
int len = BIO_read(bio, &(*buf), sizeof(buf));
pX509Cert->Assign(1, new StringVal(len, buf));
X509_NAME_print_ex(bio, X509_get_subject_name(ssl_cert), 0, XN_FLAG_RFC2253);
len = BIO_gets(bio, &(*buf), sizeof(buf));
pX509Cert->Assign(2, new StringVal(len, buf));
X509_NAME_print_ex(bio, X509_get_issuer_name(ssl_cert), 0, XN_FLAG_RFC2253);
len = BIO_gets(bio, &(*buf), sizeof(buf));
pX509Cert->Assign(3, new StringVal(len, buf));
BIO_free(bio);
pX509Cert->Assign(4, new Val(GetTimeFromAsn1(X509_get_notBefore(ssl_cert)), TYPE_TIME));
pX509Cert->Assign(5, new Val(GetTimeFromAsn1(X509_get_notAfter(ssl_cert)), TYPE_TIME));
// we only read 255 bytes because byte 256 is always 0.
// if the string is longer than 255, that will be our null-termination,
// otherwhise i2t does null-terminate.
if ( ! i2t_ASN1_OBJECT(buf, 255, ssl_cert->cert_info->key->algor->algorithm) )
buf[0] = 0;
pX509Cert->Assign(6, new StringVal(buf));
if ( ! i2t_ASN1_OBJECT(buf, 255, ssl_cert->sig_alg->algorithm) )
buf[0] = 0;
pX509Cert->Assign(7, new StringVal(buf));
// Things we can do when we have the key...
EVP_PKEY *pkey = X509_extract_key(ssl_cert);
if ( pkey != NULL )
{
if ( pkey->type == EVP_PKEY_DSA )
pX509Cert->Assign(8, new StringVal("dsa"));
else if ( pkey->type == EVP_PKEY_RSA )
{
pX509Cert->Assign(8, new StringVal("rsa"));
char *exponent = BN_bn2dec(pkey->pkey.rsa->e);
if ( exponent != NULL )
{
pX509Cert->Assign(10, new StringVal(exponent));
OPENSSL_free(exponent);
exponent = NULL;
}
}
#ifndef OPENSSL_NO_EC
else if ( pkey->type == EVP_PKEY_EC )
{
pX509Cert->Assign(8, new StringVal("dsa"));
pX509Cert->Assign(11, KeyCurve(pkey));
}
#endif
unsigned int length = KeyLength(pkey);
if ( length > 0 )
pX509Cert->Assign(9, new Val(length, TYPE_COUNT));
EVP_PKEY_free(pkey);
}
return pX509Cert;
}
StringVal* file_analysis::X509::GetExtensionFromBIO(BIO* bio)
{
BIO_flush(bio);
ERR_clear_error();
int length = BIO_pending(bio);
if ( ERR_peek_error() != 0 )
{
char tmp[120];
ERR_error_string_n(ERR_get_error(), tmp, sizeof(tmp));
reporter->Weird(fmt("X509::GetExtensionFromBIO: %s", tmp));
BIO_free_all(bio);
return 0;
}
if ( length == 0 )
{
BIO_free_all(bio);
return new StringVal("");
}
char* buffer = (char*) malloc(length);
if ( ! buffer )
{
// Just emit an error here and try to continue instead of aborting
// because it's unclear the length value is very reliable.
reporter->Error("X509::GetExtensionFromBIO malloc(%d) failed", length);
BIO_free_all(bio);
return 0;
}
BIO_read(bio, (void*) buffer, length);
StringVal* ext_val = new StringVal(length, buffer);
free(buffer);
BIO_free_all(bio);
return ext_val;
}
void file_analysis::X509::ParseExtension(X509_EXTENSION* ex)
{
char name[256];
char oid[256];
ASN1_OBJECT* ext_asn = X509_EXTENSION_get_object(ex);
const char* short_name = OBJ_nid2sn(OBJ_obj2nid(ext_asn));
OBJ_obj2txt(name, 255, ext_asn, 0);
OBJ_obj2txt(oid, 255, ext_asn, 1);
int critical = 0;
if ( X509_EXTENSION_get_critical(ex) != 0 )
critical = 1;
BIO *bio = BIO_new(BIO_s_mem());
if( ! X509V3_EXT_print(bio, ex, 0, 0))
M_ASN1_OCTET_STRING_print(bio,ex->value);
StringVal* ext_val = GetExtensionFromBIO(bio);
if ( ! ext_val )
ext_val = new StringVal(0, "");
RecordVal* pX509Ext = new RecordVal(BifType::Record::X509::Extension);
pX509Ext->Assign(0, new StringVal(name));
if ( short_name and strlen(short_name) > 0 )
pX509Ext->Assign(1, new StringVal(short_name));
pX509Ext->Assign(2, new StringVal(oid));
pX509Ext->Assign(3, new Val(critical, TYPE_BOOL));
pX509Ext->Assign(4, ext_val);
// send off generic extension event
//
// and then look if we have a specialized event for the extension we just
// parsed. And if we have it, we send the specialized event on top of the
// generic event that we just had. I know, that is... kind of not nice,
// but I am not sure if there is a better way to do it...
val_list* vl = new val_list();
vl->append(GetFile()->GetVal()->Ref());
vl->append(pX509Ext);
mgr.QueueEvent(x509_extension, vl);
// look if we have a specialized handler for this event...
if ( OBJ_obj2nid(ext_asn) == NID_basic_constraints )
ParseBasicConstraints(ex);
else if ( OBJ_obj2nid(ext_asn) == NID_subject_alt_name )
ParseSAN(ex);
}
void file_analysis::X509::ParseBasicConstraints(X509_EXTENSION* ex)
{
assert(OBJ_obj2nid(X509_EXTENSION_get_object(ex)) == NID_basic_constraints);
BASIC_CONSTRAINTS *constr = (BASIC_CONSTRAINTS *) X509V3_EXT_d2i(ex);
if ( constr )
{
RecordVal* pBasicConstraint = new RecordVal(BifType::Record::X509::BasicConstraints);
pBasicConstraint->Assign(0, new Val(constr->ca ? 1 : 0, TYPE_BOOL));
if ( constr->pathlen )
pBasicConstraint->Assign(1, new Val((int32_t) ASN1_INTEGER_get(constr->pathlen), TYPE_COUNT));
val_list* vl = new val_list();
vl->append(GetFile()->GetVal()->Ref());
vl->append(pBasicConstraint);
mgr.QueueEvent(x509_ext_basic_constraints, vl);
BASIC_CONSTRAINTS_free(constr);
}
else
reporter->Weird(fmt("Certificate with invalid BasicConstraint. fuid %s", GetFile()->GetID().c_str()));
}
void file_analysis::X509::ParseSAN(X509_EXTENSION* ext)
{
assert(OBJ_obj2nid(X509_EXTENSION_get_object(ext)) == NID_subject_alt_name);
GENERAL_NAMES *altname = (GENERAL_NAMES*)X509V3_EXT_d2i(ext);
if ( ! altname )
{
reporter->Weird(fmt("Could not parse subject alternative names. fuid %s", GetFile()->GetID().c_str()));
return;
}
VectorVal* names = 0;
VectorVal* emails = 0;
VectorVal* uris = 0;
VectorVal* ips = 0;
unsigned int otherfields = 0;
for ( int i = 0; i < sk_GENERAL_NAME_num(altname); i++ )
{
GENERAL_NAME *gen = sk_GENERAL_NAME_value(altname, i);
assert(gen);
if ( gen->type == GEN_DNS || gen->type == GEN_URI || gen->type == GEN_EMAIL )
{
if ( ASN1_STRING_type(gen->d.ia5) != V_ASN1_IA5STRING )
{
reporter->Weird(fmt("DNS-field does not contain an IA5String. fuid %s", GetFile()->GetID().c_str()));
continue;
}
const char* name = (const char*) ASN1_STRING_data(gen->d.ia5);
StringVal* bs = new StringVal(name);
switch ( gen->type )
{
case GEN_DNS:
if ( names == 0 )
names = new VectorVal(internal_type("string_vec")->AsVectorType());
names->Assign(names->Size(), bs);
break;
case GEN_URI:
if ( uris == 0 )
uris = new VectorVal(internal_type("string_vec")->AsVectorType());
uris->Assign(uris->Size(), bs);
break;
case GEN_EMAIL:
if ( emails == 0 )
emails = new VectorVal(internal_type("string_vec")->AsVectorType());
emails->Assign(emails->Size(), bs);
break;
}
}
else if ( gen->type == GEN_IPADD )
{
if ( ips == 0 )
ips = new VectorVal(internal_type("addr_vec")->AsVectorType());
uint32* addr = (uint32*) gen->d.ip->data;
if( gen->d.ip->length == 4 )
ips->Assign(ips->Size(), new AddrVal(*addr));
else if ( gen->d.ip->length == 16 )
ips->Assign(ips->Size(), new AddrVal(addr));
else
{
reporter->Weird(fmt("Weird IP address length %d in subject alternative name. fuid %s", gen->d.ip->length, GetFile()->GetID().c_str()));
continue;
}
}
else
{
// reporter->Error("Subject alternative name contained unsupported fields. fuid %s", GetFile()->GetID().c_str());
// This happens quite often - just mark it
otherfields = 1;
continue;
}
}
RecordVal* sanExt = new RecordVal(BifType::Record::X509::SubjectAlternativeName);
if ( names != 0 )
sanExt->Assign(0, names);
if ( uris != 0 )
sanExt->Assign(1, uris);
if ( emails != 0 )
sanExt->Assign(2, emails);
if ( ips != 0 )
sanExt->Assign(3, ips);
sanExt->Assign(4, new Val(otherfields, TYPE_BOOL));
val_list* vl = new val_list();
vl->append(GetFile()->GetVal()->Ref());
vl->append(sanExt);
mgr.QueueEvent(x509_ext_subject_alternative_name, vl);
GENERAL_NAMES_free(altname);
}
StringVal* file_analysis::X509::KeyCurve(EVP_PKEY *key)
{
assert(key != NULL);
#ifdef OPENSSL_NO_EC
// well, we do not have EC-Support...
return NULL;
#else
if ( key->type != EVP_PKEY_EC )
{
// no EC-key - no curve name
return NULL;
}
const EC_GROUP *group;
int nid;
if ( (group = EC_KEY_get0_group(key->pkey.ec)) == NULL)
// I guess we could not parse this
return NULL;
nid = EC_GROUP_get_curve_name(group);
if ( nid == 0 )
// and an invalid nid...
return NULL;
const char * curve_name = OBJ_nid2sn(nid);
if ( curve_name == NULL )
return NULL;
return new StringVal(curve_name);
#endif
}
unsigned int file_analysis::X509::KeyLength(EVP_PKEY *key)
{
assert(key != NULL);
switch(key->type) {
case EVP_PKEY_RSA:
return BN_num_bits(key->pkey.rsa->n);
case EVP_PKEY_DSA:
return BN_num_bits(key->pkey.dsa->p);
#ifndef OPENSSL_NO_EC
case EVP_PKEY_EC:
{
BIGNUM* ec_order = BN_new();
if ( ! ec_order )
// could not malloc bignum?
return 0;
const EC_GROUP *group = EC_KEY_get0_group(key->pkey.ec);
if ( ! group )
{
// unknown ex-group
BN_free(ec_order);
return 0;
}
if ( ! EC_GROUP_get_order(group, ec_order, NULL) )
{
// could not get ec-group-order
BN_free(ec_order);
return 0;
}
unsigned int length = BN_num_bits(ec_order);
BN_free(ec_order);
return length;
}
#endif
default:
return 0; // unknown public key type
}
reporter->InternalError("cannot be reached");
}
double file_analysis::X509::GetTimeFromAsn1(const ASN1_TIME* atime)
{
time_t lResult = 0;
char lBuffer[24];
char* pBuffer = lBuffer;
size_t lTimeLength = atime->length;
char * pString = (char *) atime->data;
if ( atime->type == V_ASN1_UTCTIME )
{
if ( lTimeLength < 11 || lTimeLength > 17 )
return 0;
memcpy(pBuffer, pString, 10);
pBuffer += 10;
pString += 10;
}
else
{
if ( lTimeLength < 13 )
return 0;
memcpy(pBuffer, pString, 12);
pBuffer += 12;
pString += 12;
}
if ((*pString == 'Z') || (*pString == '-') || (*pString == '+'))
{
*(pBuffer++) = '0';
*(pBuffer++) = '0';
}
else
{
*(pBuffer++) = *(pString++);
*(pBuffer++) = *(pString++);
// Skip any fractional seconds...
if (*pString == '.')
{
pString++;
while ((*pString >= '0') && (*pString <= '9'))
pString++;
}
}
*(pBuffer++) = 'Z';
*(pBuffer++) = '\0';
time_t lSecondsFromUTC;
if ( *pString == 'Z' )
lSecondsFromUTC = 0;
else
{
if ((*pString != '+') && (pString[5] != '-'))
return 0;
lSecondsFromUTC = ((pString[1]-'0') * 10 + (pString[2]-'0')) * 60;
lSecondsFromUTC += (pString[3]-'0') * 10 + (pString[4]-'0');
if (*pString == '-')
lSecondsFromUTC = -lSecondsFromUTC;
}
tm lTime;
lTime.tm_sec = ((lBuffer[10] - '0') * 10) + (lBuffer[11] - '0');
lTime.tm_min = ((lBuffer[8] - '0') * 10) + (lBuffer[9] - '0');
lTime.tm_hour = ((lBuffer[6] - '0') * 10) + (lBuffer[7] - '0');
lTime.tm_mday = ((lBuffer[4] - '0') * 10) + (lBuffer[5] - '0');
lTime.tm_mon = (((lBuffer[2] - '0') * 10) + (lBuffer[3] - '0')) - 1;
lTime.tm_year = ((lBuffer[0] - '0') * 10) + (lBuffer[1] - '0');
if ( lTime.tm_year < 50 )
lTime.tm_year += 100; // RFC 2459
lTime.tm_wday = 0;
lTime.tm_yday = 0;
lTime.tm_isdst = 0; // No DST adjustment requested
lResult = mktime(&lTime);
if ( lResult )
{
if ( 0 != lTime.tm_isdst )
lResult -= 3600; // mktime may adjust for DST (OS dependent)
lResult += lSecondsFromUTC;
}
else
lResult = 0;
return lResult;
}
X509Val::X509Val(::X509* arg_certificate) : OpaqueVal(x509_opaque_type)
{
certificate = arg_certificate;
}
X509Val::X509Val() : OpaqueVal(x509_opaque_type)
{
certificate = 0;
}
X509Val::~X509Val()
{
if ( certificate )
X509_free(certificate);
}
::X509* X509Val::GetCertificate() const
{
return certificate;
}
bool X509Val::DoSerialize(SerialInfo* info) const
{
DO_SERIALIZE(SER_X509_VAL, OpaqueVal);
unsigned char *buf = NULL;
int length = i2d_X509(certificate, &buf);
if ( length < 0 )
return false;
bool res = SERIALIZE_STR(reinterpret_cast<const char*>(buf), length);
OPENSSL_free(buf);
return res;
}
bool X509Val::DoUnserialize(UnserialInfo* info)
{
DO_UNSERIALIZE(OpaqueVal)
int length;
unsigned char *certbuf, *opensslbuf;
if ( ! UNSERIALIZE_STR(reinterpret_cast<char **>(&certbuf), &length) )
return false;
opensslbuf = certbuf; // OpenSSL likes to shift pointers around. really.
certificate = d2i_X509(NULL, const_cast<const unsigned char**>(&opensslbuf), length);
delete[] certbuf;
if ( !certificate )
return false;
return true;
}

112
src/file_analysis/analyzer/x509/X509.h Normal file
View file

@ -0,0 +1,112 @@
// See the file "COPYING" in the main distribution directory for copyright.
#ifndef FILE_ANALYSIS_X509_H
#define FILE_ANALYSIS_X509_H
#include <string>
#include "Val.h"
#include "../File.h"
#include "Analyzer.h"
#include <openssl/x509.h>
#include <openssl/asn1.h>
namespace file_analysis {
class X509Val;
class X509 : public file_analysis::Analyzer {
public:
virtual bool DeliverStream(const u_char* data, uint64 len);
virtual bool Undelivered(uint64 offset, uint64 len);
virtual bool EndOfFile();
/**
* Converts an X509 certificate into a \c X509::Certificate record
* value. This is a static function that can be called from external,
* it doesn't depend on the state of any particular file analyzer.
*
* @param cert_val The certificate to converts.
*
* @param Returns the new record value and passes ownership to
* caller.
*/
static RecordVal* ParseCertificate(X509Val* cert_val);
static file_analysis::Analyzer* Instantiate(RecordVal* args, File* file)
{ return new X509(args, file); }
/**
* Retrieve an X509 extension value from an OpenSSL BIO to which it was
* written.
*
* @param bio the OpenSSL BIO to read. It will be freed by the function,
* including when an error occurs.
*
* @return The X509 extension value.
*/
static StringVal* GetExtensionFromBIO(BIO* bio);
protected:
X509(RecordVal* args, File* file);
private:
void ParseExtension(X509_EXTENSION* ex);
void ParseBasicConstraints(X509_EXTENSION* ex);
void ParseSAN(X509_EXTENSION* ex);
std::string cert_data;
// Helpers for ParseCertificate.
static double GetTimeFromAsn1(const ASN1_TIME * atime);
static StringVal* KeyCurve(EVP_PKEY *key);
static unsigned int KeyLength(EVP_PKEY *key);
};
/**
* This class wraps an OpenSSL X509 data structure.
*
* We need these to be able to pass OpenSSL pointers around in Bro
* script-land. Otherwise, we cannot verify certificates from Bro
* scriptland
*/
class X509Val : public OpaqueVal {
public:
/**
* Construct an X509Val.
*
* @param certificate specifies the wrapped OpenSSL certificate
*
* @return A newly initialized X509Val.
*/
explicit X509Val(::X509* certificate);
/**
* Destructor.
*/
~X509Val();
/**
* Get the wrapped X509 certificate. Please take care, that the
* internal OpenSSL reference counting stays the same.
*
* @return The wrapped OpenSSL X509 certificate.
*/
::X509* GetCertificate() const;
protected:
/**
* Construct an empty X509Val. Only used for deserialization
*/
X509Val();
private:
::X509* certificate; // the wrapped certificate
DECLARE_SERIAL(X509Val);
};
}
#endif

57
src/file_analysis/analyzer/x509/events.bif Normal file
View file

@ -0,0 +1,57 @@
## Generated for encountered X509 certificates, e.g., in the clear SSL/TLS
## connection handshake.
##
## See `Wikipedia <http://en.wikipedia.org/wiki/X.509>`__ for more information
## about the X.509 format.
##
## f: The file.
##
## cert_ref: An opaque pointer to the underlying OpenSSL data structure of the
## certificate.
##
## cert: The parsed certificate information.
##
## .. bro:see:: x509_extension x509_ext_basic_constraints
## x509_ext_subject_alternative_name x509_parse x509_verify
## x509_get_certificate_string
event x509_certificate%(f: fa_file, cert_ref: opaque of x509, cert: X509::Certificate%);
## Generated for X509 extensions seen in a certificate.
##
## See `Wikipedia <http://en.wikipedia.org/wiki/X.509>`__ for more information
## about the X.509 format.
##
## f: The file.
##
## ext: The parsed extension.
##
## .. bro:see:: x509_certificate x509_ext_basic_constraints
## x509_ext_subject_alternative_name x509_parse x509_verify
## x509_get_certificate_string
event x509_extension%(f: fa_file, ext: X509::Extension%);
## Generated for the X509 basic constraints extension seen in a certificate.
## This extension can be used to identify the subject of a certificate as a CA.
##
## f: The file.
##
## ext: The parsed basic constraints extension.
##
## .. bro:see:: x509_certificate x509_extension
## x509_ext_subject_alternative_name x509_parse x509_verify
## x509_get_certificate_string
event x509_ext_basic_constraints%(f: fa_file, ext: X509::BasicConstraints%);
## Generated for the X509 subject alternative name extension seen in a certificate.
## This extension can be used to allow additional entities to be bound to the subject
## of the certificate. Usually it is used to specify one or multiple DNS names for
## which a certificate is valid.
##
## f: The file.
##
## ext: The parsed subject alternative name extension.
##
## .. bro:see:: x509_certificate x509_extension x509_ext_basic_constraints
## x509_parse x509_verify
## x509_get_certificate_string
event x509_ext_subject_alternative_name%(f: fa_file, ext: X509::SubjectAlternativeName%);

435
src/file_analysis/analyzer/x509/functions.bif Normal file
View file

@ -0,0 +1,435 @@
%%{
#include "file_analysis/analyzer/x509/X509.h"
#include "types.bif.h"
#include <openssl/x509.h>
#include <openssl/asn1.h>
#include <openssl/x509_vfy.h>
#include <openssl/ocsp.h>
// This is the indexed map of X509 certificate stores.
static map<Val*, X509_STORE*> x509_stores;
// ### NOTE: while d2i_X509 does not take a const u_char** pointer,
// here we assume d2i_X509 does not write to <data>, so it is safe to
// convert data to a non-const pointer. Could some X509 guru verify
// this?
X509* d2i_X509_(X509** px, const u_char** in, int len)
{
#ifdef OPENSSL_D2I_X509_USES_CONST_CHAR
return d2i_X509(px, in, len);
#else
return d2i_X509(px, (u_char**)in, len);
#endif
}
// construct an error record
RecordVal* x509_error_record(uint64_t num, const char* reason)
{
RecordVal* rrecord = new RecordVal(BifType::Record::X509::Result);
rrecord->Assign(0, new Val(num, TYPE_COUNT));
rrecord->Assign(1, new StringVal(reason));
return rrecord;
}
X509_STORE* x509_get_root_store(TableVal* root_certs)
{
// If this certificate store was built previously, just reuse the old one.
if ( x509_stores.count(root_certs) > 0 )
return x509_stores[root_certs];
X509_STORE* ctx = X509_STORE_new();
ListVal* idxs = root_certs->ConvertToPureList();
// Build the validation store
for ( int i = 0; i < idxs->Length(); ++i )
{
Val* key = idxs->Index(i);
StringVal *sv = root_certs->Lookup(key)->AsStringVal();
assert(sv);
const uint8* data = sv->Bytes();
X509* x = d2i_X509_(NULL, &data, sv->Len());
if ( ! x )
{
builtin_error(fmt("Root CA error: %s", ERR_error_string(ERR_get_error(),NULL)));
return 0;
}
X509_STORE_add_cert(ctx, x);
}
delete idxs;
// Save the newly constructed certificate store into the cacheing map.
x509_stores[root_certs] = ctx;
return ctx;
}
// get all cretificates starting at the second one (assuming the first one is the host certificate)
STACK_OF(X509)* x509_get_untrusted_stack(VectorVal* certs_vec)
{
STACK_OF(X509)* untrusted_certs = sk_X509_new_null();
if ( ! untrusted_certs )
{
builtin_error(fmt("Untrusted certificate stack initialization error: %s", ERR_error_string(ERR_get_error(),NULL)));
return 0;
}
for ( int i = 1; i < (int) certs_vec->Size(); ++i ) // start at 1 - 0 is host cert
{
Val *sv = certs_vec->Lookup(i);
// Fixme: check type
X509* x = ((file_analysis::X509Val*) sv)->GetCertificate();
if ( ! x )
{
sk_X509_free(untrusted_certs);
builtin_error(fmt("No certificate in opaque in stack"));
return 0;
}
sk_X509_push(untrusted_certs, x);
}
return untrusted_certs;
}
%%}
## Parses a certificate into an X509::Certificate structure.
##
## cert: The X509 certificicate opaque handle
##
## Returns: A X509::Certificate structure
##
## .. bro:see:: x509_certificate x509_extension x509_ext_basic_constraints
## x509_ext_subject_alternative_name x509_verify
## x509_get_certificate_string
function x509_parse%(cert: opaque of x509%): X509::Certificate
%{
assert(cert);
file_analysis::X509Val* h = (file_analysis::X509Val*) cert;
return file_analysis::X509::ParseCertificate(h);
%}
## Returns the string form of a certificate.
##
## cert: The X509 certificate opaque handle
##
## pem: A boolean that specifies if the certificate is returned
## in pem-form (true), or as the raw ASN1 encoded binary
## (false).
##
## Returns: X509 certificate as a string
##
## .. bro:see:: x509_certificate x509_extension x509_ext_basic_constraints
## x509_ext_subject_alternative_name x509_parse x509_verify
function x509_get_certificate_string%(cert: opaque of x509, pem: bool &default=F%): string
%{
assert(cert);
file_analysis::X509Val* h = (file_analysis::X509Val*) cert;
BIO *bio = BIO_new(BIO_s_mem());
if ( pem )
PEM_write_bio_X509(bio, h->GetCertificate());
else
i2d_X509_bio(bio, h->GetCertificate());
StringVal* ext_val = file_analysis::X509::GetExtensionFromBIO(bio);
if ( ! ext_val )
ext_val = new StringVal("");
return ext_val;
%}
## Verifies an OCSP reply.
##
## certs: Specifies the certificate chain to use. Server certificate first.
##
## ocsp_reply: the ocsp reply to validate
##
## root_certs: A list of root certificates to validate the certificate chain
##
## verify_time: Time for the validity check of the certificates.
##
## Returns: A record of type X509::Result containing the result code of the verify
## operation.
##
## .. bro:see:: x509_certificate x509_extension x509_ext_basic_constraints
## x509_ext_subject_alternative_name x509_parse
## x509_get_certificate_string x509_verify
function x509_ocsp_verify%(certs: x509_opaque_vector, ocsp_reply: string, root_certs: table_string_of_string, verify_time: time &default=network_time()%): X509::Result
%{
BIO* bio_err = BIO_new_fp(stderr, BIO_NOCLOSE);
X509_STORE* ctx = x509_get_root_store(root_certs->AsTableVal());
if ( ! ctx )
return x509_error_record(-1, "Problem initializing root store");
VectorVal *certs_vec = certs->AsVectorVal();
if ( certs_vec->Size() < 1 )
{
reporter->Error("No certificates given in vector");
return x509_error_record(-1, "no certificates");
}
// host certificate
unsigned int index = 0; // to prevent overloading to 0pointer
Val *sv = certs_vec->Lookup(index);
if ( ! sv )
{
builtin_error("undefined value in certificate vector");
return x509_error_record(-1, "undefined value in certificate vector");
}
file_analysis::X509Val* cert_handle = (file_analysis::X509Val*) sv;
X509* cert = cert_handle->GetCertificate();
if ( ! cert )
{
builtin_error(fmt("No certificate in opaque"));
return x509_error_record(-1, "No certificate in opaque");
}
const unsigned char* start = ocsp_reply->Bytes();
OCSP_RESPONSE *resp = d2i_OCSP_RESPONSE(NULL, &start, ocsp_reply->Len());
if ( ! resp )
return x509_error_record(-1, "Could not parse OCSP response");
int status = OCSP_response_status(resp);
if ( status != OCSP_RESPONSE_STATUS_SUCCESSFUL )
return x509_error_record(-2, OCSP_response_status_str(status));
OCSP_BASICRESP *basic = OCSP_response_get1_basic(resp);
if ( ! basic )
return x509_error_record(-1, "Could not parse OCSP response");
STACK_OF(X509)* untrusted_certs = x509_get_untrusted_stack(certs_vec);
if ( ! untrusted_certs )
return x509_error_record(-1, "Problem initializing list of untrusted certificates");
// the following code took me _forever_ to get right.
// The OCSP_basic_verify command takes a list of certificates. However (which is not immediately
// visible or understandable), those are only used to find the signer certificate. They are _not_
// used for chain building during the actual verification (this would be stupid). But - if we sneakily
// inject the certificates in the certificate list of the OCSP reply, they actually are used during
// the lookup.
// Yay.
X509* issuer_certificate = 0;
for ( int i = 0; i < sk_X509_num(untrusted_certs); i++)
{
sk_X509_push(basic->certs, sk_X509_value(untrusted_certs, i));
if ( X509_NAME_cmp(X509_get_issuer_name(cert), X509_get_subject_name(sk_X509_value(untrusted_certs, i))) )
issuer_certificate = sk_X509_value(untrusted_certs, i);
}
// Because we actually want to be able to give nice error messages that show why we were
// not able to verify the OCSP response - do our own verification logic first.
X509_STORE_CTX csc;
X509_STORE_CTX_init(&csc, ctx, sk_X509_value(basic->certs, 0), basic->certs);
X509_STORE_CTX_set_time(&csc, 0, (time_t) verify_time);
X509_STORE_CTX_set_purpose(&csc, X509_PURPOSE_OCSP_HELPER);
int result = X509_verify_cert(&csc);
if ( result != 1 )
{
const char *reason = X509_verify_cert_error_string(csc.error);
X509_STORE_CTX_cleanup(&csc);
sk_X509_free(untrusted_certs);
return x509_error_record(result, X509_verify_cert_error_string(csc.error));
}
int out = OCSP_basic_verify(basic, NULL, ctx, 0);
if ( result < 1 )
{
X509_STORE_CTX_cleanup(&csc);
sk_X509_free(untrusted_certs);
return x509_error_record(out, ERR_error_string(ERR_get_error(),NULL));
}
// ok, now we verified the OCSP response. This means that we have a valid chain tying it
// to a root that we trust and that the signature also hopefully is valid. This does not yet
// mean that the ocsp response actually matches the certificate the server send us or that
// the OCSP response even says that the certificate is valid.
// let's start this out by checking that the response is actually for the certificate we want
// to validate and not for something completely unrelated that the server is trying to trick us
// into accepting.
OCSP_CERTID *certid;
if ( issuer_certificate )
certid = OCSP_cert_to_id(NULL, cert, issuer_certificate);
else
{
// issuer not in list sent by server, check store
X509_OBJECT obj;
int lookup = X509_STORE_get_by_subject(&csc, X509_LU_X509, X509_get_subject_name(cert), &obj);
if ( lookup <= 0)
{
sk_X509_free(untrusted_certs);
X509_STORE_CTX_cleanup(&csc);
return x509_error_record(lookup, "Could not find issuer of host certificate");
}
certid = OCSP_cert_to_id(NULL, cert, obj.data.x509);
}
sk_X509_free(untrusted_certs);
X509_STORE_CTX_cleanup(&csc);
if ( ! certid )
return x509_error_record(-1, "Certificate ID construction failed");
// for now, assume we have one reply...
OCSP_SINGLERESP *single = sk_OCSP_SINGLERESP_value(basic->tbsResponseData->responses, 0);
if ( ! single )
return x509_error_record(-1, "Could not lookup OCSP response information");
if ( ! OCSP_id_cmp(certid, single->certId) )
return x509_error_record(-1, "OCSP reply is not for host certificate");
// next - check freshness of proof...
if ( ! ASN1_GENERALIZEDTIME_check(single->thisUpdate) || ! ASN1_GENERALIZEDTIME_check(single->nextUpdate) )
return x509_error_record(-1, "OCSP reply contains invalid dates");
// now - nearly done. Check freshness and status code.
// There is a function to check the freshness of the ocsp reply in the ocsp code of OpenSSL. But - it only
// supports comparing it against the current time, not against arbitrary times. Hence it is kind of unusable
// for us...
// Well, we will do it manually.
time_t vtime = (time_t) verify_time;
if ( X509_cmp_time(single->thisUpdate, &vtime) > 0 )
return x509_error_record(-1, "OCSP reply specifies time in future");
if ( X509_cmp_time(single->nextUpdate, &vtime) < 0 )
return x509_error_record(-1, "OCSP reply expired");
if ( single->certStatus->type != V_OCSP_CERTSTATUS_GOOD )
return x509_error_record(-1, OCSP_cert_status_str(single->certStatus->type));
return x509_error_record(1, OCSP_cert_status_str(single->certStatus->type));
%}
## Verifies a certificate.
##
## certs: Specifies a certificate chain that is being used to validate
## the given certificate against the root store given in *root_certs*.
## The host certificate has to be at index 0.
##
## root_certs: A list of root certificates to validate the certificate chain
##
## verify_time: Time for the validity check of the certificates.
##
## Returns: A record of type X509::Result containing the result code of the verify
## operation. In case of success also returns the full certificate chain.
##
## .. bro:see:: x509_certificate x509_extension x509_ext_basic_constraints
## x509_ext_subject_alternative_name x509_parse
## x509_get_certificate_string x509_ocsp_verify
function x509_verify%(certs: x509_opaque_vector, root_certs: table_string_of_string, verify_time: time &default=network_time()%): X509::Result
%{
X509_STORE* ctx = x509_get_root_store(root_certs->AsTableVal());
if ( ! ctx )
return x509_error_record(-1, "Problem initializing root store");
VectorVal *certs_vec = certs->AsVectorVal();
if ( ! certs_vec || certs_vec->Size() < 1 )
{
reporter->Error("No certificates given in vector");
return x509_error_record(-1, "no certificates");
}
// host certificate
unsigned int index = 0; // to prevent overloading to 0pointer
Val *sv = certs_vec->Lookup(index);
if ( !sv )
{
builtin_error("undefined value in certificate vector");
return x509_error_record(-1, "undefined value in certificate vector");
}
file_analysis::X509Val* cert_handle = (file_analysis::X509Val*) sv;
X509* cert = cert_handle->GetCertificate();
if ( ! cert )
{
builtin_error(fmt("No certificate in opaque"));
return x509_error_record(-1, "No certificate in opaque");
}
STACK_OF(X509)* untrusted_certs = x509_get_untrusted_stack(certs_vec);
if ( ! untrusted_certs )
return x509_error_record(-1, "Problem initializing list of untrusted certificates");
X509_STORE_CTX csc;
X509_STORE_CTX_init(&csc, ctx, cert, untrusted_certs);
X509_STORE_CTX_set_time(&csc, 0, (time_t) verify_time);
X509_STORE_CTX_set_flags(&csc, X509_V_FLAG_USE_CHECK_TIME);
int result = X509_verify_cert(&csc);
VectorVal* chainVector = 0;
if ( result == 1 ) // we have a valid chain. try to get it...
{
STACK_OF(X509)* chain = X509_STORE_CTX_get1_chain(&csc); // get1 = deep copy
if ( ! chain )
{
reporter->Error("Encountered valid chain that could not be resolved");
sk_X509_pop_free(chain, X509_free);
goto x509_verify_chainerror;
}
int num_certs = sk_X509_num(chain);
chainVector = new VectorVal(internal_type("x509_opaque_vector")->AsVectorType());
for ( int i = 0; i < num_certs; i++ )
{
X509* currcert = sk_X509_value(chain, i);
if ( currcert )
// X509Val takes ownership of currcert.
chainVector->Assign(i, new file_analysis::X509Val(currcert));
else
{
reporter->InternalWarning("OpenSSL returned null certificate");
for ( int j = i + 1; i < num_certs; ++j )
X509_free(sk_X509_value(chain, j));
break;
}
}
sk_X509_free(chain);
}
x509_verify_chainerror:
X509_STORE_CTX_cleanup(&csc);
sk_X509_free(untrusted_certs);
RecordVal* rrecord = new RecordVal(BifType::Record::X509::Result);
rrecord->Assign(0, new Val((uint64) csc.error, TYPE_COUNT));
rrecord->Assign(1, new StringVal(X509_verify_cert_error_string(csc.error)));
if ( chainVector )
rrecord->Assign(2, chainVector);
return rrecord;
%}

5
src/file_analysis/analyzer/x509/types.bif Normal file
View file

@ -0,0 +1,5 @@
type X509::Certificate: record;
type X509::Extension: record;
type X509::BasicConstraints: record;
type X509::SubjectAlternativeName: record;
type X509::Result: record;