Mirror/zeek
1
0
Fork 0
mirror of https://github.com/zeek/zeek.git synced 2025-10-09 18:18:19 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 3

Merge remote-tracking branch 'origin/topic/jsiwek/plist-and-event-cleanup'

Browse source 
* origin/topic/jsiwek/plist-and-event-cleanup:
  Add comments to QueueEvent() and ConnectionEvent()
  Add methods to queue events without handler existence check
  Cleanup/improve PList usage and Event API
This commit is contained in:
Johanna Amann 2019-05-01 08:28:14 -07:00
commit 29d9b5b554
109 changed files with 2080 additions and 1850 deletions
Download patch file Download diff file Expand all files Collapse all files

53
src/file_analysis/File.cc
View file

@ -154,11 +154,11 @@ void File::RaiseFileOverNewConnection(Connection* conn, bool is_orig)
{
if ( conn && FileEventAvailable(file_over_new_connection) )
{
val_list* vl = new val_list();
vl->append(val->Ref());
vl->append(conn->BuildConnVal());
vl->append(val_mgr->GetBool(is_orig));
FileEvent(file_over_new_connection, vl);
FileEvent(file_over_new_connection, {
val->Ref(),
conn->BuildConnVal(),
val_mgr->GetBool(is_orig),
});
}
}
@ -303,13 +303,11 @@ bool File::SetMime(const string& mime_type)
if ( ! FileEventAvailable(file_sniff) )
return false;
val_list* vl = new val_list();
vl->append(val->Ref());
RecordVal* meta = new RecordVal(fa_metadata_type);
vl->append(meta);
meta->Assign(meta_mime_type_idx, new StringVal(mime_type));
meta->Assign(meta_inferred_idx, val_mgr->GetBool(0));
FileEvent(file_sniff, vl);
FileEvent(file_sniff, {val->Ref(), meta});
return true;
}
@ -338,10 +336,7 @@ void File::InferMetadata()
len = min(len, LookupFieldDefaultCount(bof_buffer_size_idx));
file_mgr->DetectMIME(data, len, &matches);
val_list* vl = new val_list();
vl->append(val->Ref());
RecordVal* meta = new RecordVal(fa_metadata_type);
vl->append(meta);
if ( ! matches.empty() )
{
@ -351,7 +346,7 @@ void File::InferMetadata()
file_analysis::GenMIMEMatchesVal(matches));
}
FileEvent(file_sniff, vl);
FileEvent(file_sniff, {val->Ref(), meta});
return;
}
@ -463,11 +458,11 @@ void File::DeliverChunk(const u_char* data, uint64 len, uint64 offset)
if ( FileEventAvailable(file_reassembly_overflow) )
{
val_list* vl = new val_list();
vl->append(val->Ref());
vl->append(val_mgr->GetCount(current_offset));
vl->append(val_mgr->GetCount(gap_bytes));
FileEvent(file_reassembly_overflow, vl);
FileEvent(file_reassembly_overflow, {
val->Ref(),
val_mgr->GetCount(current_offset),
val_mgr->GetCount(gap_bytes),
});
}
}
@ -608,11 +603,11 @@ void File::Gap(uint64 offset, uint64 len)
if ( FileEventAvailable(file_gap) )
{
val_list* vl = new val_list();
vl->append(val->Ref());
vl->append(val_mgr->GetCount(offset));
vl->append(val_mgr->GetCount(len));
FileEvent(file_gap, vl);
FileEvent(file_gap, {
val->Ref(),
val_mgr->GetCount(offset),
val_mgr->GetCount(len),
});
}
analyzers.DrainModifications();
@ -631,14 +626,18 @@ void File::FileEvent(EventHandlerPtr h)
if ( ! FileEventAvailable(h) )
return;
val_list* vl = new val_list();
vl->append(val->Ref());
FileEvent(h, vl);
FileEvent(h, {val->Ref()});
}
void File::FileEvent(EventHandlerPtr h, val_list* vl)
{
mgr.QueueEvent(h, vl);
FileEvent(h, std::move(*vl));
delete vl;
}
void File::FileEvent(EventHandlerPtr h, val_list vl)
{
mgr.QueueEventFast(h, std::move(vl));
if ( h == file_new || h == file_over_new_connection ||
h == file_sniff ||

6
src/file_analysis/File.h
View file

@ -172,6 +172,12 @@ public:
*/
void FileEvent(EventHandlerPtr h, val_list* vl);
/**
* Raises an event related to the file's life-cycle.
* @param h pointer to an event handler.
* @param vl list of argument values to pass to event call.
*/
void FileEvent(EventHandlerPtr h, val_list vl);
/**
* Sets the MIME type for a file to a specific value.

11
src/file_analysis/Manager.cc
View file

@ -443,12 +443,11 @@ string Manager::GetFileID(analyzer::Tag tag, Connection* c, bool is_orig)
EnumVal* tagval = tag.AsEnumVal();
Ref(tagval);
val_list* vl = new val_list();
vl->append(tagval);
vl->append(c->BuildConnVal());
vl->append(val_mgr->GetBool(is_orig));
mgr.QueueEvent(get_file_handle, vl);
mgr.QueueEventFast(get_file_handle, {
tagval,
c->BuildConnVal(),
val_mgr->GetBool(is_orig),
});
mgr.Drain(); // need file handle immediately so we don't have to buffer data
return current_file_id;
}

20
src/file_analysis/analyzer/data_event/DataEvent.cc
View file

@ -41,12 +41,11 @@ bool DataEvent::DeliverChunk(const u_char* data, uint64 len, uint64 offset)
{
if ( ! chunk_event ) return true;
val_list* args = new val_list;
args->append(GetFile()->GetVal()->Ref());
args->append(new StringVal(new BroString(data, len, 0)));
args->append(val_mgr->GetCount(offset));
mgr.QueueEvent(chunk_event, args);
mgr.QueueEventFast(chunk_event, {
GetFile()->GetVal()->Ref(),
new StringVal(new BroString(data, len, 0)),
val_mgr->GetCount(offset),
});
return true;
}
@ -55,11 +54,10 @@ bool DataEvent::DeliverStream(const u_char* data, uint64 len)
{
if ( ! stream_event ) return true;
val_list* args = new val_list;
args->append(GetFile()->GetVal()->Ref());
args->append(new StringVal(new BroString(data, len, 0)));
mgr.QueueEvent(stream_event, args);
mgr.QueueEventFast(stream_event, {
GetFile()->GetVal()->Ref(),
new StringVal(new BroString(data, len, 0)),
});
return true;
}

10
src/file_analysis/analyzer/entropy/Entropy.cc
View file

@ -53,8 +53,8 @@ void Entropy::Finalize()
if ( ! fed )
return;
val_list* vl = new val_list();
vl->append(GetFile()->GetVal()->Ref());
if ( ! file_entropy )
return;
double montepi, scc, ent, mean, chisq;
montepi = scc = ent = mean = chisq = 0.0;
@ -67,6 +67,8 @@ void Entropy::Finalize()
ent_result->Assign(3, new Val(montepi, TYPE_DOUBLE));
ent_result->Assign(4, new Val(scc, TYPE_DOUBLE));
vl->append(ent_result);
mgr.QueueEvent(file_entropy, vl);
mgr.QueueEventFast(file_entropy, {
GetFile()->GetVal()->Ref(),
ent_result,
});
}

12
src/file_analysis/analyzer/extract/Extract.cc
View file

@ -90,12 +90,12 @@ bool Extract::DeliverStream(const u_char* data, uint64 len)
if ( limit_exceeded && file_extraction_limit )
{
File* f = GetFile();
val_list* vl = new val_list();
vl->append(f->GetVal()->Ref());
vl->append(Args()->Ref());
vl->append(val_mgr->GetCount(limit));
vl->append(val_mgr->GetCount(len));
f->FileEvent(file_extraction_limit, vl);
f->FileEvent(file_extraction_limit, {
f->GetVal()->Ref(),
Args()->Ref(),
val_mgr->GetCount(limit),
val_mgr->GetCount(len),
});
// Limit may have been modified by a BIF, re-check it.
limit_exceeded = check_limit_exceeded(limit, depth, len, &towrite);

12
src/file_analysis/analyzer/hash/Hash.cc
View file

@ -48,10 +48,12 @@ void Hash::Finalize()
if ( ! hash->IsValid() || ! fed )
return;
val_list* vl = new val_list();
vl->append(GetFile()->GetVal()->Ref());
vl->append(new StringVal(kind));
vl->append(hash->Get());
if ( ! file_hash )
return;
mgr.QueueEvent(file_hash, vl);
mgr.QueueEventFast(file_hash, {
GetFile()->GetVal()->Ref(),
new StringVal(kind),
hash->Get(),
});
}

27
src/file_analysis/analyzer/unified2/unified2-analyzer.pac
View file

@ -81,10 +81,11 @@ refine flow Flow += {
ids_event->Assign(11, to_port(${ev.dst_p}, ${ev.protocol}));
ids_event->Assign(17, val_mgr->GetCount(${ev.packet_action}));
val_list* vl = new val_list();
vl->append(connection()->bro_analyzer()->GetFile()->GetVal()->Ref());
vl->append(ids_event);
mgr.QueueEvent(::unified2_event, vl, SOURCE_LOCAL);
mgr.QueueEventFast(::unified2_event, {
connection()->bro_analyzer()->GetFile()->GetVal()->Ref(),
ids_event,
},
SOURCE_LOCAL);
}
return true;
%}
@ -112,10 +113,11 @@ refine flow Flow += {
ids_event->Assign(15, val_mgr->GetCount(${ev.mpls_label}));
ids_event->Assign(16, val_mgr->GetCount(${ev.vlan_id}));
val_list* vl = new val_list();
vl->append(connection()->bro_analyzer()->GetFile()->GetVal()->Ref());
vl->append(ids_event);
mgr.QueueEvent(::unified2_event, vl, SOURCE_LOCAL);
mgr.QueueEventFast(::unified2_event, {
connection()->bro_analyzer()->GetFile()->GetVal()->Ref(),
ids_event,
},
SOURCE_LOCAL);
}
return true;
@ -133,10 +135,11 @@ refine flow Flow += {
packet->Assign(4, val_mgr->GetCount(${pkt.link_type}));
packet->Assign(5, bytestring_to_val(${pkt.packet_data}));
val_list* vl = new val_list();
vl->append(connection()->bro_analyzer()->GetFile()->GetVal()->Ref());
vl->append(packet);
mgr.QueueEvent(::unified2_packet, vl, SOURCE_LOCAL);
mgr.QueueEventFast(::unified2_packet, {
connection()->bro_analyzer()->GetFile()->GetVal()->Ref(),
packet,
},
SOURCE_LOCAL);
}
return true;

94
src/file_analysis/analyzer/x509/OCSP.cc
View file

@ -417,10 +417,6 @@ void file_analysis::OCSP::ParseRequest(OCSP_REQUEST* req)
char buf[OCSP_STRING_BUF_SIZE]; // we need a buffer for some of the openssl functions
memset(buf, 0, sizeof(buf));
// build up our response as we go along...
val_list* vl = new val_list();
vl->append(GetFile()->GetVal()->Ref());
uint64 version = 0;
#if ( OPENSSL_VERSION_NUMBER < 0x10100000L ) || defined(LIBRESSL_VERSION_NUMBER)
@ -431,23 +427,25 @@ void file_analysis::OCSP::ParseRequest(OCSP_REQUEST* req)
// TODO: try to parse out general name ?
#endif
vl->append(val_mgr->GetCount(version));
if ( ocsp_request )
mgr.QueueEventFast(ocsp_request, {
GetFile()->GetVal()->Ref(),
val_mgr->GetCount(version),
});
BIO *bio = BIO_new(BIO_s_mem());
mgr.QueueEvent(ocsp_request, vl);
int req_count = OCSP_request_onereq_count(req);
for ( int i=0; i<req_count; i++ )
{
val_list* rvl = new val_list();
rvl->append(GetFile()->GetVal()->Ref());
val_list rvl(5);
rvl.append(GetFile()->GetVal()->Ref());
OCSP_ONEREQ *one_req = OCSP_request_onereq_get0(req, i);
OCSP_CERTID *cert_id = OCSP_onereq_get0_id(one_req);
ocsp_add_cert_id(cert_id, rvl, bio);
mgr.QueueEvent(ocsp_request_certificate, rvl);
ocsp_add_cert_id(cert_id, &rvl, bio);
mgr.QueueEvent(ocsp_request_certificate, std::move(rvl));
}
BIO_free(bio);
@ -470,14 +468,14 @@ void file_analysis::OCSP::ParseResponse(OCSP_RESPVal *resp_val)
char buf[OCSP_STRING_BUF_SIZE];
memset(buf, 0, sizeof(buf));
val_list* vl = new val_list();
vl->append(GetFile()->GetVal()->Ref());
const char *status_str = OCSP_response_status_str(OCSP_response_status(resp));
StringVal* status_val = new StringVal(strlen(status_str), status_str);
vl->append(status_val->Ref());
mgr.QueueEvent(ocsp_response_status, vl);
vl = nullptr;
if ( ocsp_response_status )
mgr.QueueEventFast(ocsp_response_status, {
GetFile()->GetVal()->Ref(),
status_val->Ref(),
});
//if (!resp_bytes)
// {
@ -490,39 +488,46 @@ void file_analysis::OCSP::ParseResponse(OCSP_RESPVal *resp_val)
//int len = BIO_read(bio, buf, sizeof(buf));
//BIO_reset(bio);
val_list vl(8);
// get the basic response
basic_resp = OCSP_response_get1_basic(resp);
if ( !basic_resp )
{
Unref(status_val);
goto clean_up;
}
#if ( OPENSSL_VERSION_NUMBER < 0x10100000L ) || defined(LIBRESSL_VERSION_NUMBER)
resp_data = basic_resp->tbsResponseData;
if ( !resp_data )
{
Unref(status_val);
goto clean_up;
}
#endif
vl = new val_list();
vl->append(GetFile()->GetVal()->Ref());
vl->append(resp_val->Ref());
vl->append(status_val);
vl.append(GetFile()->GetVal()->Ref());
vl.append(resp_val->Ref());
vl.append(status_val);
#if ( OPENSSL_VERSION_NUMBER < 0x10100000L ) || defined(LIBRESSL_VERSION_NUMBER)
vl->append(val_mgr->GetCount((uint64)ASN1_INTEGER_get(resp_data->version)));
vl.append(val_mgr->GetCount((uint64)ASN1_INTEGER_get(resp_data->version)));
#else
vl->append(parse_basic_resp_data_version(basic_resp));
vl.append(parse_basic_resp_data_version(basic_resp));
#endif
// responderID
if ( OCSP_RESPID_bio(basic_resp, bio) )
{
len = BIO_read(bio, buf, sizeof(buf));
vl->append(new StringVal(len, buf));
vl.append(new StringVal(len, buf));
BIO_reset(bio);
}
else
{
reporter->Weird("OpenSSL failed to get OCSP responder id");
vl->append(val_mgr->GetEmptyString());
vl.append(val_mgr->GetEmptyString());
}
// producedAt
@ -532,7 +537,7 @@ void file_analysis::OCSP::ParseResponse(OCSP_RESPVal *resp_val)
produced_at = OCSP_resp_get0_produced_at(basic_resp);
#endif
vl->append(new Val(GetTimeFromAsn1(produced_at, GetFile(), reporter), TYPE_TIME));
vl.append(new Val(GetTimeFromAsn1(produced_at, GetFile(), reporter), TYPE_TIME));
// responses
@ -545,8 +550,8 @@ void file_analysis::OCSP::ParseResponse(OCSP_RESPVal *resp_val)
if ( !single_resp )
continue;
val_list* rvl = new val_list();
rvl->append(GetFile()->GetVal()->Ref());
val_list rvl(10);
rvl.append(GetFile()->GetVal()->Ref());
// cert id
const OCSP_CERTID* cert_id = nullptr;
@ -557,7 +562,7 @@ void file_analysis::OCSP::ParseResponse(OCSP_RESPVal *resp_val)
cert_id = OCSP_SINGLERESP_get0_id(single_resp);
#endif
ocsp_add_cert_id(cert_id, rvl, bio);
ocsp_add_cert_id(cert_id, &rvl, bio);
BIO_reset(bio);
// certStatus
@ -574,38 +579,38 @@ void file_analysis::OCSP::ParseResponse(OCSP_RESPVal *resp_val)
reporter->Weird("OpenSSL failed to find status of OCSP response");
const char* cert_status_str = OCSP_cert_status_str(status);
rvl->append(new StringVal(strlen(cert_status_str), cert_status_str));
rvl.append(new StringVal(strlen(cert_status_str), cert_status_str));
// revocation time and reason if revoked
if ( status == V_OCSP_CERTSTATUS_REVOKED )
{
rvl->append(new Val(GetTimeFromAsn1(revoke_time, GetFile(), reporter), TYPE_TIME));
rvl.append(new Val(GetTimeFromAsn1(revoke_time, GetFile(), reporter), TYPE_TIME));
if ( reason != OCSP_REVOKED_STATUS_NOSTATUS )
{
const char* revoke_reason = OCSP_crl_reason_str(reason);
rvl->append(new StringVal(strlen(revoke_reason), revoke_reason));
rvl.append(new StringVal(strlen(revoke_reason), revoke_reason));
}
else
rvl->append(new StringVal(0, ""));
rvl.append(new StringVal(0, ""));
}
else
{
rvl->append(new Val(0.0, TYPE_TIME));
rvl->append(new StringVal(0, ""));
rvl.append(new Val(0.0, TYPE_TIME));
rvl.append(new StringVal(0, ""));
}
if ( this_update )
rvl->append(new Val(GetTimeFromAsn1(this_update, GetFile(), reporter), TYPE_TIME));
rvl.append(new Val(GetTimeFromAsn1(this_update, GetFile(), reporter), TYPE_TIME));
else
rvl->append(new Val(0.0, TYPE_TIME));
rvl.append(new Val(0.0, TYPE_TIME));
if ( next_update )
rvl->append(new Val(GetTimeFromAsn1(next_update, GetFile(), reporter), TYPE_TIME));
rvl.append(new Val(GetTimeFromAsn1(next_update, GetFile(), reporter), TYPE_TIME));
else
rvl->append(new Val(0.0, TYPE_TIME));
rvl.append(new Val(0.0, TYPE_TIME));
mgr.QueueEvent(ocsp_response_certificate, rvl);
mgr.QueueEvent(ocsp_response_certificate, std::move(rvl));
num_ext = OCSP_SINGLERESP_get_ext_count(single_resp);
for ( int k = 0; k < num_ext; ++k )
@ -621,10 +626,10 @@ void file_analysis::OCSP::ParseResponse(OCSP_RESPVal *resp_val)
#if ( OPENSSL_VERSION_NUMBER < 0x10100000L ) || defined(LIBRESSL_VERSION_NUMBER)
i2a_ASN1_OBJECT(bio, basic_resp->signatureAlgorithm->algorithm);
len = BIO_read(bio, buf, sizeof(buf));
vl->append(new StringVal(len, buf));
vl.append(new StringVal(len, buf));
BIO_reset(bio);
#else
vl->append(parse_basic_resp_sig_alg(basic_resp, bio, buf, sizeof(buf)));
vl.append(parse_basic_resp_sig_alg(basic_resp, bio, buf, sizeof(buf)));
#endif
//i2a_ASN1_OBJECT(bio, basic_resp->signature);
@ -633,7 +638,7 @@ void file_analysis::OCSP::ParseResponse(OCSP_RESPVal *resp_val)
//BIO_reset(bio);
certs_vector = new VectorVal(internal_type("x509_opaque_vector")->AsVectorType());
vl->append(certs_vector);
vl.append(certs_vector);
#if ( OPENSSL_VERSION_NUMBER < 0x10100000L ) || defined(LIBRESSL_VERSION_NUMBER)
certs = basic_resp->certs;
@ -654,7 +659,8 @@ void file_analysis::OCSP::ParseResponse(OCSP_RESPVal *resp_val)
reporter->Weird("OpenSSL returned null certificate");
}
}
mgr.QueueEvent(ocsp_response_bytes, vl);
mgr.QueueEvent(ocsp_response_bytes, std::move(vl));
// ok, now that we are done with the actual certificate - let's parse extensions :)
num_ext = OCSP_BASICRESP_get_ext_count(basic_resp);

37
src/file_analysis/analyzer/x509/X509.cc
View file

@ -57,11 +57,11 @@ bool file_analysis::X509::EndOfFile()
RecordVal* cert_record = ParseCertificate(cert_val, GetFile());
// and send the record on to scriptland
val_list* vl = new val_list();
vl->append(GetFile()->GetVal()->Ref());
vl->append(cert_val->Ref());
vl->append(cert_record->Ref()); // we Ref it here, because we want to keep a copy around for now...
mgr.QueueEvent(x509_certificate, vl);
mgr.QueueEvent(x509_certificate, {
GetFile()->GetVal()->Ref(),
cert_val->Ref(),
cert_record->Ref(), // we Ref it here, because we want to keep a copy around for now...
});
// after parsing the certificate - parse the extensions...
@ -221,17 +221,20 @@ void file_analysis::X509::ParseBasicConstraints(X509_EXTENSION* ex)
if ( constr )
{
RecordVal* pBasicConstraint = new RecordVal(BifType::Record::X509::BasicConstraints);
pBasicConstraint->Assign(0, val_mgr->GetBool(constr->ca ? 1 : 0));
if ( x509_ext_basic_constraints )
{
RecordVal* pBasicConstraint = new RecordVal(BifType::Record::X509::BasicConstraints);
pBasicConstraint->Assign(0, val_mgr->GetBool(constr->ca ? 1 : 0));
if ( constr->pathlen )
pBasicConstraint->Assign(1, val_mgr->GetCount((int32_t) ASN1_INTEGER_get(constr->pathlen)));
if ( constr->pathlen )
pBasicConstraint->Assign(1, val_mgr->GetCount((int32_t) ASN1_INTEGER_get(constr->pathlen)));
val_list* vl = new val_list();
vl->append(GetFile()->GetVal()->Ref());
vl->append(pBasicConstraint);
mgr.QueueEventFast(x509_ext_basic_constraints, {
GetFile()->GetVal()->Ref(),
pBasicConstraint,
});
}
mgr.QueueEvent(x509_ext_basic_constraints, vl);
BASIC_CONSTRAINTS_free(constr);
}
@ -367,10 +370,10 @@ void file_analysis::X509::ParseSAN(X509_EXTENSION* ext)
sanExt->Assign(4, val_mgr->GetBool(otherfields));
val_list* vl = new val_list();
vl->append(GetFile()->GetVal()->Ref());
vl->append(sanExt);
mgr.QueueEvent(x509_ext_subject_alternative_name, vl);
mgr.QueueEvent(x509_ext_subject_alternative_name, {
GetFile()->GetVal()->Ref(),
sanExt,
});
GENERAL_NAMES_free(altname);
}

17
src/file_analysis/analyzer/x509/X509Common.cc
View file

@ -277,13 +277,18 @@ void file_analysis::X509Common::ParseExtension(X509_EXTENSION* ex, EventHandlerP
// parsed. And if we have it, we send the specialized event on top of the
// generic event that we just had. I know, that is... kind of not nice,
// but I am not sure if there is a better way to do it...
val_list* vl = new val_list();
vl->append(GetFile()->GetVal()->Ref());
vl->append(pX509Ext);
if ( h == ocsp_extension )
vl->append(val_mgr->GetBool(global ? 1 : 0));
mgr.QueueEvent(h, vl);
if ( h == ocsp_extension )
mgr.QueueEvent(h, {
GetFile()->GetVal()->Ref(),
pX509Ext,
val_mgr->GetBool(global ? 1 : 0),
});
else
mgr.QueueEvent(h, {
GetFile()->GetVal()->Ref(),
pX509Ext,
});
// let individual analyzers parse more.
ParseExtensionsSpecific(ex, global, ext_asn, oid);

3
src/file_analysis/analyzer/x509/x509-extension.pac
View file

@ -35,6 +35,9 @@ refine connection MockConnection += {
function proc_signedcertificatetimestamp(rec: HandshakeRecord, version: uint8, logid: const_bytestring, timestamp: uint64, digitally_signed_algorithms: SignatureAndHashAlgorithm, digitally_signed_signature: const_bytestring) : bool
%{
if ( ! x509_ocsp_ext_signed_certificate_timestamp )
return true;
BifEvent::generate_x509_ocsp_ext_signed_certificate_timestamp((analyzer::Analyzer *) bro_analyzer(),
bro_analyzer()->GetFile()->GetVal()->Ref(),
version,