Mirror/zeek
1
0
Fork 0
mirror of https://github.com/zeek/zeek.git synced 2025-10-04 15:48:19 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 2

Cleanup/improve PList usage and Event API

Browse source 
Majority of PLists are now created as automatic/stack objects,
rather than on heap and initialized either with the known-capacity
reserved upfront or directly from an initializer_list (so there's no
wasted slack in the memory that gets allocated for lists containing
a fixed/known number of elements).

Added versions of the ConnectionEvent/QueueEvent methods that take
a val_list by value.

Added a move ctor/assign-operator to Plists to allow passing them
around without having to copy the underlying array of pointers.
This commit is contained in:
Jon Siwek 2019-04-11 19:02:13 -07:00
parent 78dcbcc71a
commit 8bc65f09ec
92 changed files with 1585 additions and 1679 deletions
Download patch file Download diff file Expand all files Collapse all files

53
src/file_analysis/File.cc
View file

@ -154,11 +154,11 @@ void File::RaiseFileOverNewConnection(Connection* conn, bool is_orig)
{
if ( conn && FileEventAvailable(file_over_new_connection) )
{
val_list* vl = new val_list();
vl->append(val->Ref());
vl->append(conn->BuildConnVal());
vl->append(val_mgr->GetBool(is_orig));
FileEvent(file_over_new_connection, vl);
FileEvent(file_over_new_connection, {
val->Ref(),
conn->BuildConnVal(),
val_mgr->GetBool(is_orig),
});
}
}
@ -303,13 +303,11 @@ bool File::SetMime(const string& mime_type)
if ( ! FileEventAvailable(file_sniff) )
return false;
val_list* vl = new val_list();
vl->append(val->Ref());
RecordVal* meta = new RecordVal(fa_metadata_type);
vl->append(meta);
meta->Assign(meta_mime_type_idx, new StringVal(mime_type));
meta->Assign(meta_inferred_idx, val_mgr->GetBool(0));
FileEvent(file_sniff, vl);
FileEvent(file_sniff, {val->Ref(), meta});
return true;
}
@ -338,10 +336,7 @@ void File::InferMetadata()
len = min(len, LookupFieldDefaultCount(bof_buffer_size_idx));
file_mgr->DetectMIME(data, len, &matches);
val_list* vl = new val_list();
vl->append(val->Ref());
RecordVal* meta = new RecordVal(fa_metadata_type);
vl->append(meta);
if ( ! matches.empty() )
{
@ -351,7 +346,7 @@ void File::InferMetadata()
file_analysis::GenMIMEMatchesVal(matches));
}
FileEvent(file_sniff, vl);
FileEvent(file_sniff, {val->Ref(), meta});
return;
}
@ -463,11 +458,11 @@ void File::DeliverChunk(const u_char* data, uint64 len, uint64 offset)
if ( FileEventAvailable(file_reassembly_overflow) )
{
val_list* vl = new val_list();
vl->append(val->Ref());
vl->append(val_mgr->GetCount(current_offset));
vl->append(val_mgr->GetCount(gap_bytes));
FileEvent(file_reassembly_overflow, vl);
FileEvent(file_reassembly_overflow, {
val->Ref(),
val_mgr->GetCount(current_offset),
val_mgr->GetCount(gap_bytes),
});
}
}
@ -608,11 +603,11 @@ void File::Gap(uint64 offset, uint64 len)
if ( FileEventAvailable(file_gap) )
{
val_list* vl = new val_list();
vl->append(val->Ref());
vl->append(val_mgr->GetCount(offset));
vl->append(val_mgr->GetCount(len));
FileEvent(file_gap, vl);
FileEvent(file_gap, {
val->Ref(),
val_mgr->GetCount(offset),
val_mgr->GetCount(len),
});
}
analyzers.DrainModifications();
@ -631,14 +626,18 @@ void File::FileEvent(EventHandlerPtr h)
if ( ! FileEventAvailable(h) )
return;
val_list* vl = new val_list();
vl->append(val->Ref());
FileEvent(h, vl);
FileEvent(h, {val->Ref()});
}
void File::FileEvent(EventHandlerPtr h, val_list* vl)
{
mgr.QueueEvent(h, vl);
FileEvent(h, std::move(*vl));
delete vl;
}
void File::FileEvent(EventHandlerPtr h, val_list vl)
{
mgr.QueueEvent(h, std::move(vl));
if ( h == file_new || h == file_over_new_connection ||
h == file_sniff ||

6
src/file_analysis/File.h
View file

@ -172,6 +172,12 @@ public:
*/
void FileEvent(EventHandlerPtr h, val_list* vl);
/**
* Raises an event related to the file's life-cycle.
* @param h pointer to an event handler.
* @param vl list of argument values to pass to event call.
*/
void FileEvent(EventHandlerPtr h, val_list vl);
/**
* Sets the MIME type for a file to a specific value.

11
src/file_analysis/Manager.cc
View file

@ -443,12 +443,11 @@ string Manager::GetFileID(analyzer::Tag tag, Connection* c, bool is_orig)
EnumVal* tagval = tag.AsEnumVal();
Ref(tagval);
val_list* vl = new val_list();
vl->append(tagval);
vl->append(c->BuildConnVal());
vl->append(val_mgr->GetBool(is_orig));
mgr.QueueEvent(get_file_handle, vl);
mgr.QueueEvent(get_file_handle, {
tagval,
c->BuildConnVal(),
val_mgr->GetBool(is_orig),
});
mgr.Drain(); // need file handle immediately so we don't have to buffer data
return current_file_id;
}

20
src/file_analysis/analyzer/data_event/DataEvent.cc
View file

@ -41,12 +41,11 @@ bool DataEvent::DeliverChunk(const u_char* data, uint64 len, uint64 offset)
{
if ( ! chunk_event ) return true;
val_list* args = new val_list;
args->append(GetFile()->GetVal()->Ref());
args->append(new StringVal(new BroString(data, len, 0)));
args->append(val_mgr->GetCount(offset));
mgr.QueueEvent(chunk_event, args);
mgr.QueueEvent(chunk_event, {
GetFile()->GetVal()->Ref(),
new StringVal(new BroString(data, len, 0)),
val_mgr->GetCount(offset),
});
return true;
}
@ -55,11 +54,10 @@ bool DataEvent::DeliverStream(const u_char* data, uint64 len)
{
if ( ! stream_event ) return true;
val_list* args = new val_list;
args->append(GetFile()->GetVal()->Ref());
args->append(new StringVal(new BroString(data, len, 0)));
mgr.QueueEvent(stream_event, args);
mgr.QueueEvent(stream_event, {
GetFile()->GetVal()->Ref(),
new StringVal(new BroString(data, len, 0)),
});
return true;
}

9
src/file_analysis/analyzer/entropy/Entropy.cc
View file

@ -53,9 +53,6 @@ void Entropy::Finalize()
if ( ! fed )
return;
val_list* vl = new val_list();
vl->append(GetFile()->GetVal()->Ref());
double montepi, scc, ent, mean, chisq;
montepi = scc = ent = mean = chisq = 0.0;
entropy->Get(&ent, &chisq, &mean, &montepi, &scc);
@ -67,6 +64,8 @@ void Entropy::Finalize()
ent_result->Assign(3, new Val(montepi, TYPE_DOUBLE));
ent_result->Assign(4, new Val(scc, TYPE_DOUBLE));
vl->append(ent_result);
mgr.QueueEvent(file_entropy, vl);
mgr.QueueEvent(file_entropy, {
GetFile()->GetVal()->Ref(),
ent_result,
});
}

12
src/file_analysis/analyzer/extract/Extract.cc
View file

@ -90,12 +90,12 @@ bool Extract::DeliverStream(const u_char* data, uint64 len)
if ( limit_exceeded && file_extraction_limit )
{
File* f = GetFile();
val_list* vl = new val_list();
vl->append(f->GetVal()->Ref());
vl->append(Args()->Ref());
vl->append(val_mgr->GetCount(limit));
vl->append(val_mgr->GetCount(len));
f->FileEvent(file_extraction_limit, vl);
f->FileEvent(file_extraction_limit, {
f->GetVal()->Ref(),
Args()->Ref(),
val_mgr->GetCount(limit),
val_mgr->GetCount(len),
});
// Limit may have been modified by a BIF, re-check it.
limit_exceeded = check_limit_exceeded(limit, depth, len, &towrite);

11
src/file_analysis/analyzer/hash/Hash.cc
View file

@ -48,10 +48,9 @@ void Hash::Finalize()
if ( ! hash->IsValid() || ! fed )
return;
val_list* vl = new val_list();
vl->append(GetFile()->GetVal()->Ref());
vl->append(new StringVal(kind));
vl->append(hash->Get());
mgr.QueueEvent(file_hash, vl);
mgr.QueueEvent(file_hash, {
GetFile()->GetVal()->Ref(),
new StringVal(kind),
hash->Get(),
});
}

27
src/file_analysis/analyzer/unified2/unified2-analyzer.pac
View file

@ -81,10 +81,11 @@ refine flow Flow += {
ids_event->Assign(11, to_port(${ev.dst_p}, ${ev.protocol}));
ids_event->Assign(17, val_mgr->GetCount(${ev.packet_action}));
val_list* vl = new val_list();
vl->append(connection()->bro_analyzer()->GetFile()->GetVal()->Ref());
vl->append(ids_event);
mgr.QueueEvent(::unified2_event, vl, SOURCE_LOCAL);
mgr.QueueEvent(::unified2_event, {
connection()->bro_analyzer()->GetFile()->GetVal()->Ref(),
ids_event,
},
SOURCE_LOCAL);
}
return true;
%}
@ -112,10 +113,11 @@ refine flow Flow += {
ids_event->Assign(15, val_mgr->GetCount(${ev.mpls_label}));
ids_event->Assign(16, val_mgr->GetCount(${ev.vlan_id}));
val_list* vl = new val_list();
vl->append(connection()->bro_analyzer()->GetFile()->GetVal()->Ref());
vl->append(ids_event);
mgr.QueueEvent(::unified2_event, vl, SOURCE_LOCAL);
mgr.QueueEvent(::unified2_event, {
connection()->bro_analyzer()->GetFile()->GetVal()->Ref(),
ids_event,
},
SOURCE_LOCAL);
}
return true;
@ -133,10 +135,11 @@ refine flow Flow += {
packet->Assign(4, val_mgr->GetCount(${pkt.link_type}));
packet->Assign(5, bytestring_to_val(${pkt.packet_data}));
val_list* vl = new val_list();
vl->append(connection()->bro_analyzer()->GetFile()->GetVal()->Ref());
vl->append(packet);
mgr.QueueEvent(::unified2_packet, vl, SOURCE_LOCAL);
mgr.QueueEvent(::unified2_packet, {
connection()->bro_analyzer()->GetFile()->GetVal()->Ref(),
packet,
},
SOURCE_LOCAL);
}
return true;

86
src/file_analysis/analyzer/x509/OCSP.cc
View file

@ -417,10 +417,6 @@ void file_analysis::OCSP::ParseRequest(OCSP_REQUEST* req)
char buf[OCSP_STRING_BUF_SIZE]; // we need a buffer for some of the openssl functions
memset(buf, 0, sizeof(buf));
// build up our response as we go along...
val_list* vl = new val_list();
vl->append(GetFile()->GetVal()->Ref());
uint64 version = 0;
#if ( OPENSSL_VERSION_NUMBER < 0x10100000L ) || defined(LIBRESSL_VERSION_NUMBER)
@ -431,23 +427,24 @@ void file_analysis::OCSP::ParseRequest(OCSP_REQUEST* req)
// TODO: try to parse out general name ?
#endif
vl->append(val_mgr->GetCount(version));
mgr.QueueEvent(ocsp_request, {
GetFile()->GetVal()->Ref(),
val_mgr->GetCount(version),
});
BIO *bio = BIO_new(BIO_s_mem());
mgr.QueueEvent(ocsp_request, vl);
int req_count = OCSP_request_onereq_count(req);
for ( int i=0; i<req_count; i++ )
{
val_list* rvl = new val_list();
rvl->append(GetFile()->GetVal()->Ref());
val_list rvl(5);
rvl.append(GetFile()->GetVal()->Ref());
OCSP_ONEREQ *one_req = OCSP_request_onereq_get0(req, i);
OCSP_CERTID *cert_id = OCSP_onereq_get0_id(one_req);
ocsp_add_cert_id(cert_id, rvl, bio);
mgr.QueueEvent(ocsp_request_certificate, rvl);
ocsp_add_cert_id(cert_id, &rvl, bio);
mgr.QueueEvent(ocsp_request_certificate, std::move(rvl));
}
BIO_free(bio);
@ -470,14 +467,13 @@ void file_analysis::OCSP::ParseResponse(OCSP_RESPVal *resp_val)
char buf[OCSP_STRING_BUF_SIZE];
memset(buf, 0, sizeof(buf));
val_list* vl = new val_list();
vl->append(GetFile()->GetVal()->Ref());
const char *status_str = OCSP_response_status_str(OCSP_response_status(resp));
StringVal* status_val = new StringVal(strlen(status_str), status_str);
vl->append(status_val->Ref());
mgr.QueueEvent(ocsp_response_status, vl);
vl = nullptr;
mgr.QueueEvent(ocsp_response_status, {
GetFile()->GetVal()->Ref(),
status_val->Ref(),
});
//if (!resp_bytes)
// {
@ -490,6 +486,8 @@ void file_analysis::OCSP::ParseResponse(OCSP_RESPVal *resp_val)
//int len = BIO_read(bio, buf, sizeof(buf));
//BIO_reset(bio);
val_list vl(8);
// get the basic response
basic_resp = OCSP_response_get1_basic(resp);
if ( !basic_resp )
@ -501,28 +499,27 @@ void file_analysis::OCSP::ParseResponse(OCSP_RESPVal *resp_val)
goto clean_up;
#endif
vl = new val_list();
vl->append(GetFile()->GetVal()->Ref());
vl->append(resp_val->Ref());
vl->append(status_val);
vl.append(GetFile()->GetVal()->Ref());
vl.append(resp_val->Ref());
vl.append(status_val);
#if ( OPENSSL_VERSION_NUMBER < 0x10100000L ) || defined(LIBRESSL_VERSION_NUMBER)
vl->append(val_mgr->GetCount((uint64)ASN1_INTEGER_get(resp_data->version)));
vl.append(val_mgr->GetCount((uint64)ASN1_INTEGER_get(resp_data->version)));
#else
vl->append(parse_basic_resp_data_version(basic_resp));
vl.append(parse_basic_resp_data_version(basic_resp));
#endif
// responderID
if ( OCSP_RESPID_bio(basic_resp, bio) )
{
len = BIO_read(bio, buf, sizeof(buf));
vl->append(new StringVal(len, buf));
vl.append(new StringVal(len, buf));
BIO_reset(bio);
}
else
{
reporter->Weird("OpenSSL failed to get OCSP responder id");
vl->append(val_mgr->GetEmptyString());
vl.append(val_mgr->GetEmptyString());
}
// producedAt
@ -532,7 +529,7 @@ void file_analysis::OCSP::ParseResponse(OCSP_RESPVal *resp_val)
produced_at = OCSP_resp_get0_produced_at(basic_resp);
#endif
vl->append(new Val(GetTimeFromAsn1(produced_at, GetFile(), reporter), TYPE_TIME));
vl.append(new Val(GetTimeFromAsn1(produced_at, GetFile(), reporter), TYPE_TIME));
// responses
@ -545,8 +542,8 @@ void file_analysis::OCSP::ParseResponse(OCSP_RESPVal *resp_val)
if ( !single_resp )
continue;
val_list* rvl = new val_list();
rvl->append(GetFile()->GetVal()->Ref());
val_list rvl(10);
rvl.append(GetFile()->GetVal()->Ref());
// cert id
const OCSP_CERTID* cert_id = nullptr;
@ -557,7 +554,7 @@ void file_analysis::OCSP::ParseResponse(OCSP_RESPVal *resp_val)
cert_id = OCSP_SINGLERESP_get0_id(single_resp);
#endif
ocsp_add_cert_id(cert_id, rvl, bio);
ocsp_add_cert_id(cert_id, &rvl, bio);
BIO_reset(bio);
// certStatus
@ -574,38 +571,38 @@ void file_analysis::OCSP::ParseResponse(OCSP_RESPVal *resp_val)
reporter->Weird("OpenSSL failed to find status of OCSP response");
const char* cert_status_str = OCSP_cert_status_str(status);
rvl->append(new StringVal(strlen(cert_status_str), cert_status_str));
rvl.append(new StringVal(strlen(cert_status_str), cert_status_str));
// revocation time and reason if revoked
if ( status == V_OCSP_CERTSTATUS_REVOKED )
{
rvl->append(new Val(GetTimeFromAsn1(revoke_time, GetFile(), reporter), TYPE_TIME));
rvl.append(new Val(GetTimeFromAsn1(revoke_time, GetFile(), reporter), TYPE_TIME));
if ( reason != OCSP_REVOKED_STATUS_NOSTATUS )
{
const char* revoke_reason = OCSP_crl_reason_str(reason);
rvl->append(new StringVal(strlen(revoke_reason), revoke_reason));
rvl.append(new StringVal(strlen(revoke_reason), revoke_reason));
}
else
rvl->append(new StringVal(0, ""));
rvl.append(new StringVal(0, ""));
}
else
{
rvl->append(new Val(0.0, TYPE_TIME));
rvl->append(new StringVal(0, ""));
rvl.append(new Val(0.0, TYPE_TIME));
rvl.append(new StringVal(0, ""));
}
if ( this_update )
rvl->append(new Val(GetTimeFromAsn1(this_update, GetFile(), reporter), TYPE_TIME));
rvl.append(new Val(GetTimeFromAsn1(this_update, GetFile(), reporter), TYPE_TIME));
else
rvl->append(new Val(0.0, TYPE_TIME));
rvl.append(new Val(0.0, TYPE_TIME));
if ( next_update )
rvl->append(new Val(GetTimeFromAsn1(next_update, GetFile(), reporter), TYPE_TIME));
rvl.append(new Val(GetTimeFromAsn1(next_update, GetFile(), reporter), TYPE_TIME));
else
rvl->append(new Val(0.0, TYPE_TIME));
rvl.append(new Val(0.0, TYPE_TIME));
mgr.QueueEvent(ocsp_response_certificate, rvl);
mgr.QueueEvent(ocsp_response_certificate, std::move(rvl));
num_ext = OCSP_SINGLERESP_get_ext_count(single_resp);
for ( int k = 0; k < num_ext; ++k )
@ -621,10 +618,10 @@ void file_analysis::OCSP::ParseResponse(OCSP_RESPVal *resp_val)
#if ( OPENSSL_VERSION_NUMBER < 0x10100000L ) || defined(LIBRESSL_VERSION_NUMBER)
i2a_ASN1_OBJECT(bio, basic_resp->signatureAlgorithm->algorithm);
len = BIO_read(bio, buf, sizeof(buf));
vl->append(new StringVal(len, buf));
vl.append(new StringVal(len, buf));
BIO_reset(bio);
#else
vl->append(parse_basic_resp_sig_alg(basic_resp, bio, buf, sizeof(buf)));
vl.append(parse_basic_resp_sig_alg(basic_resp, bio, buf, sizeof(buf)));
#endif
//i2a_ASN1_OBJECT(bio, basic_resp->signature);
@ -633,7 +630,7 @@ void file_analysis::OCSP::ParseResponse(OCSP_RESPVal *resp_val)
//BIO_reset(bio);
certs_vector = new VectorVal(internal_type("x509_opaque_vector")->AsVectorType());
vl->append(certs_vector);
vl.append(certs_vector);
#if ( OPENSSL_VERSION_NUMBER < 0x10100000L ) || defined(LIBRESSL_VERSION_NUMBER)
certs = basic_resp->certs;
@ -654,7 +651,8 @@ void file_analysis::OCSP::ParseResponse(OCSP_RESPVal *resp_val)
reporter->Weird("OpenSSL returned null certificate");
}
}
mgr.QueueEvent(ocsp_response_bytes, vl);
mgr.QueueEvent(ocsp_response_bytes, std::move(vl));
// ok, now that we are done with the actual certificate - let's parse extensions :)
num_ext = OCSP_BASICRESP_get_ext_count(basic_resp);

27
src/file_analysis/analyzer/x509/X509.cc
View file

@ -57,11 +57,11 @@ bool file_analysis::X509::EndOfFile()
RecordVal* cert_record = ParseCertificate(cert_val, GetFile());
// and send the record on to scriptland
val_list* vl = new val_list();
vl->append(GetFile()->GetVal()->Ref());
vl->append(cert_val->Ref());
vl->append(cert_record->Ref()); // we Ref it here, because we want to keep a copy around for now...
mgr.QueueEvent(x509_certificate, vl);
mgr.QueueEvent(x509_certificate, {
GetFile()->GetVal()->Ref(),
cert_val->Ref(),
cert_record->Ref(), // we Ref it here, because we want to keep a copy around for now...
});
// after parsing the certificate - parse the extensions...
@ -227,11 +227,10 @@ void file_analysis::X509::ParseBasicConstraints(X509_EXTENSION* ex)
if ( constr->pathlen )
pBasicConstraint->Assign(1, val_mgr->GetCount((int32_t) ASN1_INTEGER_get(constr->pathlen)));
val_list* vl = new val_list();
vl->append(GetFile()->GetVal()->Ref());
vl->append(pBasicConstraint);
mgr.QueueEvent(x509_ext_basic_constraints, vl);
mgr.QueueEvent(x509_ext_basic_constraints, {
GetFile()->GetVal()->Ref(),
pBasicConstraint,
});
BASIC_CONSTRAINTS_free(constr);
}
@ -367,10 +366,10 @@ void file_analysis::X509::ParseSAN(X509_EXTENSION* ext)
sanExt->Assign(4, val_mgr->GetBool(otherfields));
val_list* vl = new val_list();
vl->append(GetFile()->GetVal()->Ref());
vl->append(sanExt);
mgr.QueueEvent(x509_ext_subject_alternative_name, vl);
mgr.QueueEvent(x509_ext_subject_alternative_name, {
GetFile()->GetVal()->Ref(),
sanExt,
});
GENERAL_NAMES_free(altname);
}

17
src/file_analysis/analyzer/x509/X509Common.cc
View file

@ -277,13 +277,18 @@ void file_analysis::X509Common::ParseExtension(X509_EXTENSION* ex, EventHandlerP
// parsed. And if we have it, we send the specialized event on top of the
// generic event that we just had. I know, that is... kind of not nice,
// but I am not sure if there is a better way to do it...
val_list* vl = new val_list();
vl->append(GetFile()->GetVal()->Ref());
vl->append(pX509Ext);
if ( h == ocsp_extension )
vl->append(val_mgr->GetBool(global ? 1 : 0));
mgr.QueueEvent(h, vl);
if ( h == ocsp_extension )
mgr.QueueEvent(h, {
GetFile()->GetVal()->Ref(),
pX509Ext,
val_mgr->GetBool(global ? 1 : 0),
});
else
mgr.QueueEvent(h, {
GetFile()->GetVal()->Ref(),
pX509Ext,
});
// let individual analyzers parse more.
ParseExtensionsSpecific(ex, global, ext_asn, oid);