Mirror/zeek
1
0
Fork 0
mirror of https://github.com/zeek/zeek.git synced 2025-10-02 06:38:20 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 1

analyzer/protocol: Reformat with spicy-format

Browse source
This commit is contained in:
Arne Welzel 2025-07-28 09:39:40 +02:00
parent aa2afa3e9b
commit d70bcd07b9
6 changed files with 1433 additions and 1519 deletions
Download patch file Download diff file Expand all files Collapse all files

3
.pre-commit-config.yaml
View file

@ -55,5 +55,4 @@ repos:
rev: v0.26.0 rev: v0.26.0
hooks: hooks:
- id: spicy-format - id: spicy-format
# TODO: Reformat existing large analyzers just before 8.0. exclude: '^testing/.*'
exclude: '(^testing/.*)|(protocol/ldap/.*)|(protocol/quic/.*)|(protocol/websocket/.*)'

336
src/analyzer/protocol/ldap/asn1.spicy
View file

@ -23,237 +23,222 @@ import spicy;
# https://www.obj-sys.com/asn1tutorial/node10.html # https://www.obj-sys.com/asn1tutorial/node10.html
public type ASN1Type = enum { public type ASN1Type = enum {
Boolean = 1, Boolean = 1,
Integer = 2, Integer = 2,
BitString = 3, BitString = 3,
OctetString = 4, OctetString = 4,
NullVal = 5, NullVal = 5,
ObjectIdentifier = 6, ObjectIdentifier = 6,
ObjectDescriptor = 7, ObjectDescriptor = 7,
InstanceOf = 8, InstanceOf = 8,
Real = 9, Real = 9,
Enumerated = 10, Enumerated = 10,
EmbeddedPDV = 11, EmbeddedPDV = 11,
UTF8String = 12, UTF8String = 12,
RelativeOID = 13, RelativeOID = 13,
Sequence = 16, Sequence = 16,
Set = 17, Set = 17,
NumericString = 18, NumericString = 18,
PrintableString = 19, PrintableString = 19,
TeletextString = 20, TeletextString = 20,
VideotextString = 21, VideotextString = 21,
IA5String = 22, IA5String = 22,
UTCTime = 23, UTCTime = 23,
GeneralizedTime = 24, GeneralizedTime = 24,
GraphicString = 25, GraphicString = 25,
VisibleString = 26, VisibleString = 26,
GeneralString = 27, GeneralString = 27,
UniversalString = 28, UniversalString = 28,
CharacterString = 29, CharacterString = 29,
BMPString = 30 BMPString = 30,
}; };
#- ASN.1 data classes -------------------------------------------------------- #- ASN.1 data classes --------------------------------------------------------
public type ASN1Class = enum { public type ASN1Class = enum {
Universal = 0, Universal = 0,
Application = 1, Application = 1,
ContextSpecific = 2, ContextSpecific = 2,
Private = 3 Private = 3,
}; };
#- ASN.1 tag definition (including length) ------------------------------------ #- ASN.1 tag definition (including length) ------------------------------------
type LengthType = unit { type LengthType = unit {
var len: uint64; var len: uint64;
var tag_len: uint8; var tag_len: uint8;
data : bitfield(8) { data: bitfield(8) {
num: 0..6; num: 0..6;
islong: 7; islong: 7;
}; };
switch (self.data.islong) {
switch ( self.data.islong ) { 0 -> : void {
0 -> : void { self.len = self.data.num;
self.len = self.data.num; self.tag_len = 1;
self.tag_len = 1; }
} 1 -> : bytes &size=self.data.num &convert=$$.to_uint(spicy::ByteOrder::Network) {
1 -> : bytes &size=self.data.num self.len = $$;
&convert=$$.to_uint(spicy::ByteOrder::Network) { self.tag_len = self.data.num + 1;
self.len = $$; }
self.tag_len = self.data.num + 1; };
}
};
}; };
type ASN1Tag = unit { type ASN1Tag = unit {
: bitfield(8) { : bitfield(8) {
type_: 0..4 &convert=ASN1Type($$); type_: 0..4 &convert=ASN1Type($$);
constructed: 5 &convert=cast<bool>($$); constructed: 5 &convert=cast<bool>($$);
class: 6..7 &convert=ASN1Class($$); class: 6..7 &convert=ASN1Class($$);
}; };
}; };
#- ASN.1 bit string ----------------------------------------------------------- #- ASN.1 bit string -----------------------------------------------------------
# https://www.obj-sys.com/asn1tutorial/node10.html # https://www.obj-sys.com/asn1tutorial/node10.html
type ASN1BitString = unit(len: uint64, constructed: bool) { type ASN1BitString = unit(len: uint64, constructed: bool) {
: uint8; # unused bits : uint8; # unused bits
value_bits: bytes &size=(len - 1); value_bits: bytes &size=(len - 1);
# TODO - constructed form # TODO - constructed form
# https://github.com/zeek/spicy/issues/921 # https://github.com/zeek/spicy/issues/921
# `bytes` needs << and >> support before we can implement complex bitstrings # `bytes` needs << and >> support before we can implement complex bitstrings
# #
}; };
#- ASN.1 octet string --------------------------------------------------------- #- ASN.1 octet string ---------------------------------------------------------
# https://www.obj-sys.com/asn1tutorial/node10.html # https://www.obj-sys.com/asn1tutorial/node10.html
type ASN1OctetString = unit(len: uint64, constructed: bool) { type ASN1OctetString = unit(len: uint64, constructed: bool) {
value: bytes &size = len; value: bytes &size=len;
# TODO - constructed form # TODO - constructed form
}; };
#- ASN.1 various string types ------------------------------------------------- #- ASN.1 various string types -------------------------------------------------
# https://www.obj-sys.com/asn1tutorial/node124.html # https://www.obj-sys.com/asn1tutorial/node124.html
type ASN1String = unit(tag: ASN1Tag, len: uint64) { type ASN1String = unit(tag: ASN1Tag, len: uint64) {
var encoding: spicy::Charset; var encoding: spicy::Charset;
on %init { on %init {
switch ( tag.type_ ) { switch (tag.type_) {
# see "Restricted Character String Types" in # see "Restricted Character String Types" in
# "Generic String Encoding Rules (GSER) for ASN.1 Types" # "Generic String Encoding Rules (GSER) for ASN.1 Types"
# (https://datatracker.ietf.org/doc/html/rfc3641#section-3.2) # (https://datatracker.ietf.org/doc/html/rfc3641#section-3.2)
case ASN1Type::PrintableString,
case ASN1Type::PrintableString, ASN1Type::GeneralizedTime,
ASN1Type::GeneralizedTime, ASN1Type::UTCTime: {
ASN1Type::UTCTime: { self.encoding = spicy::Charset::ASCII;
self.encoding = spicy::Charset::ASCII; }
} case ASN1Type::UTF8String,
ASN1Type::GeneralString,
case ASN1Type::UTF8String, ASN1Type::CharacterString,
ASN1Type::GeneralString, ASN1Type::GraphicString,
ASN1Type::CharacterString, ASN1Type::IA5String,
ASN1Type::GraphicString, ASN1Type::NumericString,
ASN1Type::IA5String, ASN1Type::TeletextString,
ASN1Type::NumericString, ASN1Type::VideotextString,
ASN1Type::TeletextString, ASN1Type::VisibleString,
ASN1Type::VideotextString, # TODO: RFC3641 mentions special UTF-8 mapping rules for
ASN1Type::VisibleString, # BMPString and UniversalString. This *may* not be correct.
# TODO: RFC3641 mentions special UTF-8 mapping rules for ASN1Type::BMPString,
# BMPString and UniversalString. This *may* not be correct. ASN1Type::UniversalString: {
ASN1Type::BMPString, self.encoding = spicy::Charset::UTF8;
ASN1Type::UniversalString: { }
self.encoding = spicy::Charset::UTF8; }
}
} }
}
value: ASN1OctetString(len, tag.constructed) &convert=$$.value.decode(self.encoding); value: ASN1OctetString(len, tag.constructed) &convert=$$.value.decode(self.encoding);
} &convert=self.value; } &convert=self.value;
#- ASN.1 OID ------------------------------------------------------------------ #- ASN.1 OID ------------------------------------------------------------------
# https://www.obj-sys.com/asn1tutorial/node124.html # https://www.obj-sys.com/asn1tutorial/node124.html
type ASN1ObjectIdentifierNibble = unit { type ASN1ObjectIdentifierNibble = unit {
data : bitfield(8) { data: bitfield(8) {
num: 0..6; num: 0..6;
more: 7; more: 7;
}; };
} &convert=self.data; } &convert=self.data;
type ASN1ObjectIdentifier = unit(len: uint64) { type ASN1ObjectIdentifier = unit(len: uint64) {
var oidbytes: bytes; var oidbytes: bytes;
var temp: uint64; var temp: uint64;
var oidstring: string; var oidstring: string;
: uint8 if ( len >= 1 ) { : uint8 if(len >= 1) {
self.temp = $$ / 40; self.temp = $$ / 40;
self.oidbytes += ("%d" % (self.temp)).encode(); self.oidbytes += ("%d" % (self.temp)).encode();
self.temp = $$ % 40; self.temp = $$ % 40;
self.oidbytes += (".%d" % (self.temp)).encode(); self.oidbytes += (".%d" % (self.temp)).encode();
self.temp = 0; self.temp = 0;
}
sublist: ASN1ObjectIdentifierNibble[len - 1] foreach {
self.temp = ( self.temp<<7 ) | $$.num;
if ( $$.more != 1 ) {
self.oidbytes += (".%d" % (self.temp)).encode();
self.temp = 0;
} }
}
on %done { sublist: ASN1ObjectIdentifierNibble[len - 1] foreach {
self.oidstring = self.oidbytes.decode(); self.temp = (self.temp << 7) | $$.num;
} if ($$.more != 1) {
self.oidbytes += (".%d" % (self.temp)).encode();
self.temp = 0;
}
}
on %done {
self.oidstring = self.oidbytes.decode();
}
}; };
#- ASN.1 message header (tag + length information) ---------------------------- #- ASN.1 message header (tag + length information) ----------------------------
public type ASN1Header = unit { public type ASN1Header = unit {
tag: ASN1Tag; tag: ASN1Tag;
len: LengthType; len: LengthType;
}; };
#- ASN.1 message body --------------------------------------------------------- #- ASN.1 message body ---------------------------------------------------------
public type ASN1Body = unit(head: ASN1Header, recursive: bool) { public type ASN1Body = unit(head: ASN1Header, recursive: bool) {
switch ( head.tag.type_ ) { switch (head.tag.type_) {
ASN1Type::Boolean -> bool_value: uint8 &convert=cast<bool>($$) &requires=head.len.len == 1;
ASN1Type::Integer,
ASN1Type::Enumerated -> num_value: bytes &size=head.len.len &convert=$$.to_int(spicy::ByteOrder::Big);
ASN1Type::NullVal -> null_value: bytes &size=0 &requires=head.len.len == 0;
ASN1Type::BitString -> bitstr_value: ASN1BitString(head.len.len, head.tag.constructed);
ASN1Type::OctetString -> str_value: ASN1OctetString(head.len.len, head.tag.constructed) &convert=$$.value.decode(spicy::Charset::ASCII);
ASN1Type::ObjectIdentifier -> str_value: ASN1ObjectIdentifier(head.len.len) &convert=$$.oidstring;
ASN1Type::BMPString,
ASN1Type::CharacterString,
ASN1Type::GeneralizedTime,
ASN1Type::GeneralString,
ASN1Type::GraphicString,
ASN1Type::IA5String,
ASN1Type::NumericString,
ASN1Type::PrintableString,
ASN1Type::TeletextString,
ASN1Type::UTCTime,
ASN1Type::UTF8String,
ASN1Type::VideotextString,
ASN1Type::VisibleString,
ASN1Type::UniversalString -> str_value: ASN1String(head.tag, head.len.len);
ASN1Type::Sequence,
ASN1Type::Set -> seq: ASN1SubMessages(head.len.len) if(recursive);
ASN1Type::Boolean -> bool_value: uint8 &convert=cast<bool>($$) &requires=head.len.len==1; # TODO: ASN1Type values not handled yet
ASN1Type::ObjectDescriptor,
ASN1Type::InstanceOf,
ASN1Type::Real,
ASN1Type::EmbeddedPDV,
ASN1Type::RelativeOID -> unimplemented_value: bytes &size=head.len.len;
ASN1Type::Integer, # unknown (to me) ASN.1 enumeration, skip over silently
ASN1Type::Enumerated -> num_value: bytes &size=head.len.len * -> unimplemented_value: bytes &size=head.len.len;
&convert=$$.to_int(spicy::ByteOrder::Big); };
ASN1Type::NullVal -> null_value: bytes &size=0 &requires=head.len.len==0;
ASN1Type::BitString -> bitstr_value: ASN1BitString(head.len.len, head.tag.constructed);
ASN1Type::OctetString -> str_value: ASN1OctetString(head.len.len, head.tag.constructed)
&convert=$$.value.decode(spicy::Charset::ASCII);
ASN1Type::ObjectIdentifier -> str_value: ASN1ObjectIdentifier(head.len.len)
&convert=$$.oidstring;
ASN1Type::BMPString,
ASN1Type::CharacterString,
ASN1Type::GeneralizedTime,
ASN1Type::GeneralString,
ASN1Type::GraphicString,
ASN1Type::IA5String,
ASN1Type::NumericString,
ASN1Type::PrintableString,
ASN1Type::TeletextString,
ASN1Type::UTCTime,
ASN1Type::UTF8String,
ASN1Type::VideotextString,
ASN1Type::VisibleString,
ASN1Type::UniversalString -> str_value: ASN1String(head.tag, head.len.len);
ASN1Type::Sequence, ASN1Type::Set -> seq: ASN1SubMessages(head.len.len) if (recursive);
# TODO: ASN1Type values not handled yet
ASN1Type::ObjectDescriptor,
ASN1Type::InstanceOf,
ASN1Type::Real,
ASN1Type::EmbeddedPDV,
ASN1Type::RelativeOID -> unimplemented_value: bytes &size=head.len.len;
# unknown (to me) ASN.1 enumeration, skip over silently
* -> unimplemented_value: bytes &size=head.len.len;
};
}; };
#- ASN.1 array of ASN.1 sequence/set sub-messages (up to msgLen bytes) -------- #- ASN.1 array of ASN.1 sequence/set sub-messages (up to msgLen bytes) --------
public type ASN1SubMessages = unit(msgLen: uint64) { public type ASN1SubMessages = unit(msgLen: uint64) {
submessages: ASN1Message(True)[] &eod; submessages: ASN1Message(True)[] &eod;
} &size=msgLen; } &size=msgLen;
#- ASN.1 message with header and body ----------------------------------------- #- ASN.1 message with header and body -----------------------------------------
@ -262,18 +247,15 @@ public type ASN1SubMessages = unit(msgLen: uint64) {
# - else, application_data:bytes stores data array # - else, application_data:bytes stores data array
public type ASN1Message = unit(recursive: bool) { public type ASN1Message = unit(recursive: bool) {
var application_id: int32; var application_id: int32;
head: ASN1Header; head: ASN1Header;
switch ( self.head.tag.class ) { switch (self.head.tag.class) {
ASN1Class::Universal -> body: ASN1Body(self.head, recursive);
ASN1Class::Universal -> body: ASN1Body(self.head, recursive); ASN1Class::Application,
ASN1Class::ContextSpecific,
ASN1Class::Application, ASN1Class::Private -> application_data: bytes &size=self.head.len.len {
ASN1Class::ContextSpecific, self.application_id = cast<int32>(self.head.tag.type_);
ASN1Class::Private -> application_data: bytes &size=self.head.len.len { }
self.application_id = cast<int32>(self.head.tag.type_); };
}
};
}; };

1584
src/analyzer/protocol/ldap/ldap.spicy
View file

File diff suppressed because it is too large Load diff

4
src/analyzer/protocol/ldap/ldap_zeek.spicy
View file

@ -6,9 +6,9 @@ import LDAP;
import zeek; import zeek;
on LDAP::TlsForward::%init { on LDAP::TlsForward::%init {
zeek::protocol_begin("SSL"); zeek::protocol_begin("SSL");
} }
on LDAP::TlsForward::chunk { on LDAP::TlsForward::chunk {
zeek::protocol_data_in(zeek::is_orig(), self.chunk); zeek::protocol_data_in(zeek::is_orig(), self.chunk);
} }

860
src/analyzer/protocol/quic/QUIC.spicy
View file

File diff suppressed because it is too large Load diff

165
src/analyzer/protocol/websocket/websocket.spicy
View file

@ -6,114 +6,109 @@ import spicy;
import zeek; import zeek;
const OPCODE_CONTINUATION = 0x00; const OPCODE_CONTINUATION = 0x00;
const OPCODE_TEXT = 0x01; const OPCODE_TEXT = 0x01;
const OPCODE_BINARY = 0x02; const OPCODE_BINARY = 0x02;
const OPCODE_CLOSE = 0x08; const OPCODE_CLOSE = 0x08;
const OPCODE_PING = 0x09; const OPCODE_PING = 0x09;
const OPCODE_PONG = 0x0a; const OPCODE_PONG = 0x0a;
public function fast_unmask( public function fast_unmask(masking_key_idx: uint64, masking_key: vector<uint8>, chunk: bytes): bytes &cxxname="hlt_websocket::WebSocket::fast_unmask";
masking_key_idx: uint64,
masking_key: vector<uint8>,
chunk: bytes
): bytes &cxxname="hlt_websocket::WebSocket::fast_unmask";
type Frame = unit(m: Message) { type Frame = unit(m: Message) {
var payload_len: uint64; var payload_len: uint64;
var masking_key_idx: uint64; var masking_key_idx: uint64;
var close_data: bytes; var close_data: bytes;
var effective_opcode: uint8; var effective_opcode: uint8;
: bitfield(16) { : bitfield(16) {
fin: 0 &convert=cast<bool>($$); fin: 0 &convert=cast<bool>($$);
rsv: 1..3; rsv: 1..3;
opcode: 4..7 &convert=cast<uint8>($$); opcode: 4..7 &convert=cast<uint8>($$);
mask: 8 &convert=cast<bool>($$); mask: 8 &convert=cast<bool>($$);
payload_len1: 9..15; payload_len1: 9..15;
} &bit-order=spicy::BitOrder::MSB0; } &bit-order=spicy::BitOrder::MSB0;
# Verify that this is either a continuation frame, or the Message does not # Verify that this is either a continuation frame, or the Message does not
# yet have a non-continuation opcode, but this frame does. # yet have a non-continuation opcode, but this frame does.
: void &requires=(m.opcode != OPCODE_CONTINUATION && self.opcode == OPCODE_CONTINUATION || m.opcode == OPCODE_CONTINUATION && self.opcode != OPCODE_CONTINUATION); : void &requires=(m.opcode != OPCODE_CONTINUATION && self.opcode == OPCODE_CONTINUATION || m.opcode == OPCODE_CONTINUATION && self.opcode != OPCODE_CONTINUATION);
# Type/opcode to decide what to do with individual chunks. # Type/opcode to decide what to do with individual chunks.
: void { : void {
self.effective_opcode = m.opcode != OPCODE_CONTINUATION ? m.opcode : self.opcode; self.effective_opcode = m.opcode != OPCODE_CONTINUATION ? m.opcode : self.opcode;
}
payload_len2: uint16 if (self.payload_len1 == 126);
payload_len8: uint64 if (self.payload_len1 == 127);
: void {
self.payload_len = self.payload_len1;
if ( self?.payload_len2 )
self.payload_len = self.payload_len2;
else if ( self?.payload_len8 )
self.payload_len = self.payload_len8;
}
# This being an uint8[] allows masking_key[x] indexing, while a bytes
# object would require *masking_key.at(i) which took roughly 20% more
# runtime when I tested it.
masking_key: uint8[] &size=4 if (self.mask);
chunk: bytes &size=self.payload_len &chunked {
# Don't use &convert with &chunked: https://github.com/zeek/spicy/issues/1661
if ( self.mask ) {
self.chunk = fast_unmask(self.masking_key_idx, self.masking_key, $$);
self.masking_key_idx += |$$|;
} else {
self.chunk = $$;
} }
# Forward TEXT and BINARY data to dowstream analyzers. payload_len2: uint16 if(self.payload_len1 == 126);
if ( self.effective_opcode == OPCODE_TEXT || self.effective_opcode == OPCODE_BINARY ) payload_len8: uint64 if(self.payload_len1 == 127);
zeek::protocol_data_in(zeek::is_orig(), $$);
# Accumulate the unmasked data in close_data if this a close frame : void {
# so it can be parsed by the outer Message. It's a bit of a hack. self.payload_len = self.payload_len1;
if ( self.effective_opcode == OPCODE_CLOSE ) if (self?.payload_len2)
self.close_data += $$; self.payload_len = self.payload_len2;
} else if (self?.payload_len8)
self.payload_len = self.payload_len8;
}
# This being an uint8[] allows masking_key[x] indexing, while a bytes
# object would require *masking_key.at(i) which took roughly 20% more
# runtime when I tested it.
masking_key: uint8[] &size=4 if(self.mask);
chunk: bytes &size=self.payload_len &chunked {
# Don't use &convert with &chunked: https://github.com/zeek/spicy/issues/1661
if (self.mask) {
self.chunk = fast_unmask(self.masking_key_idx, self.masking_key, $$);
self.masking_key_idx += |$$|;
} else {
self.chunk = $$;
}
# Forward TEXT and BINARY data to dowstream analyzers.
if (self.effective_opcode == OPCODE_TEXT || self.effective_opcode == OPCODE_BINARY)
zeek::protocol_data_in(zeek::is_orig(), $$);
# Accumulate the unmasked data in close_data if this a close frame
# so it can be parsed by the outer Message. It's a bit of a hack.
if (self.effective_opcode == OPCODE_CLOSE)
self.close_data += $$;
}
}; };
type CloseFrame = unit { type CloseFrame = unit {
var status: uint16; var status: uint16;
var reason: bytes; var reason: bytes;
: bytes &eod { : bytes &eod {
if ( |$$| > 0 ) { if (|$$| > 0) {
self.status = cast<uint16>($$.sub(0, 2).to_uint(spicy::ByteOrder::Network)); self.status = cast<uint16>($$.sub(0, 2).to_uint(spicy::ByteOrder::Network));
self.reason = $$.sub(2, 0); self.reason = $$.sub(2, 0);
}
} }
}
}; };
public type Message = unit { public type Message = unit {
# transient trickery # transient trickery
var done: bool = False; var done: bool = False;
var opcode: uint8; var opcode: uint8;
# Keep the first one persistent to have access # Keep the first one persistent to have access
# to the payload if it's a close frame. # to the payload if it's a close frame.
first_frame: Frame(self) { first_frame: Frame(self) {
self.opcode = $$.opcode; self.opcode = $$.opcode;
self.done = $$.fin; self.done = $$.fin;
} }
: Frame(self)[] &until=(self.done) if (!self.done) foreach { : Frame(self)[] &until=(self.done) if(!self.done) foreach {
self.done = $$.fin; self.done = $$.fin;
} }
: CloseFrame &parse-from=self.first_frame.close_data if (self.opcode == OPCODE_CLOSE); : CloseFrame &parse-from=self.first_frame.close_data if(self.opcode == OPCODE_CLOSE);
on %done { on %done {
spicy::accept_input(); spicy::accept_input();
} }
}; };
public type Messages = unit { public type Messages = unit {
: Message[]; : Message[];
}; };