diff --git a/CHANGES b/CHANGES index 15bb9ff761..a8086af261 100644 --- a/CHANGES +++ b/CHANGES @@ -1,4 +1,61 @@ +2.4-41 | 2015-07-21 08:35:17 -0700 + + * Fixing compiler warning. (Robin Sommer) + + * Updates to IANA TLS registry. (Johanna Amann) + +2.4-38 | 2015-07-20 15:30:35 -0700 + + * Refactor code to use a common Packet type throught. (Jeff + Barber/Robin Sommer) + + * Extend parsing layer 2 and keeping track of layer 3 protoco. (Jeff Barber) + + * Add a raw_packet() event that generated for all packets and + include layer 2 information. (Jeff Barber) + +2.4-27 | 2015-07-15 13:31:49 -0700 + + * Fix race condition in intel test. (Johanna Amann) + +2.4-24 | 2015-07-14 08:04:11 -0700 + + * Correct Perl package name on FreeBSD in documentation.(Justin Azoff) + + * Adding an environment variable to BTest configuration for external + scripts. (Robin Sommer) + +2.4-20 | 2015-07-03 10:40:21 -0700 + + * Adding a weird for when truncated packets lead TCP reassembly to + ignore content. (Robin Sommer) + +2.4-19 | 2015-07-03 09:04:54 -0700 + + * A set of tests exercising IP defragmentation and TCP reassembly. + (Robin Sommer) + +2.4-17 | 2015-06-28 13:02:41 -0700 + + * BIT-1314: Add detection for Quantum Insert attacks. The TCP + reassembler can now keep a history of old TCP segments using the + tcp_max_old_segments option. An overlapping segment with different + data will then generate an rexmit_inconsistency event. The default + for tcp_max_old_segments is zero, which disabled any additional + buffering. (Yun Zheng Hu/Robin Sommer) + +2.4-14 | 2015-06-28 12:30:12 -0700 + + * BIT-1400: Allow '<' and '>' in MIME multipart boundaries. The spec + doesn't actually seem to permit these, but they seem to occur in + the wild. (Jon Siwek) + +2.4-12 | 2015-06-28 12:21:11 -0700 + + * BIT-1399: Trying to decompress deflated HTTP content even when + zlib headers are missing. (Seth Hall) + 2.4-10 | 2015-06-25 07:11:17 -0700 * Correct a name used in a header identifier (Justin Azoff) diff --git a/VERSION b/VERSION index f9a814dbf7..ab3d92f8ce 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -2.4-10 +2.4-41 diff --git a/aux/bro-aux b/aux/bro-aux index 6d6679506d..07af9748f4 160000 --- a/aux/bro-aux +++ b/aux/bro-aux @@ -1 +1 @@ -Subproject commit 6d6679506d8762ddbba16f0b34f7ad253e3aac45 +Subproject commit 07af9748f40dc47d3a2b3290db494a90dcbddbdc diff --git a/aux/broker b/aux/broker index f303cdbc60..d25efc7d5f 160000 --- a/aux/broker +++ b/aux/broker @@ -1 +1 @@ -Subproject commit f303cdbc60ad6eef35ebcd1473ee85b3123f5ef1 +Subproject commit d25efc7d5f495c30294b11180c1857477078f2d6 diff --git a/aux/btest b/aux/btest index 0e2da116a5..a89cd0fda0 160000 --- a/aux/btest +++ b/aux/btest @@ -1 +1 @@ -Subproject commit 0e2da116a5e29baacaecc6daac7bc4bc9ff387c5 +Subproject commit a89cd0fda0f17f69b96c935959cae89145b92927 diff --git a/aux/plugins b/aux/plugins index 99d7519991..98ad8a5b97 160000 --- a/aux/plugins +++ b/aux/plugins @@ -1 +1 @@ -Subproject commit 99d7519991b41a970809a99433ea9c7df42e9d93 +Subproject commit 98ad8a5b97f601a3ec9a773d87582438212b8290 diff --git a/doc/install/install.rst b/doc/install/install.rst index ab3f6cafc8..03b77fba8c 100644 --- a/doc/install/install.rst +++ b/doc/install/install.rst @@ -67,7 +67,7 @@ To install the required dependencies, you can use: .. console:: - sudo pkg install bash cmake swig bison python perl py27-sqlite3 + sudo pkg install bash cmake swig bison python perl5 py27-sqlite3 Note that in older versions of FreeBSD, you might have to use the "pkg_add -r" command instead of "pkg install". diff --git a/scripts/base/init-bare.bro b/scripts/base/init-bare.bro index a1dfe2ee06..6ec756c28e 100644 --- a/scripts/base/init-bare.bro +++ b/scripts/base/init-bare.bro @@ -740,6 +740,7 @@ type pcap_packet: record { caplen: count; ##< The number of bytes captured (<= *len*). len: count; ##< The length of the packet in bytes, including link-level header. data: string; ##< The payload of the packet, including link-level header. + link_type: link_encap; ##< Layer 2 link encapsulation type. }; ## GeoIP location information. @@ -954,6 +955,11 @@ const tcp_max_above_hole_without_any_acks = 16384 &redef; ## .. bro:see:: tcp_max_initial_window tcp_max_above_hole_without_any_acks const tcp_excessive_data_without_further_acks = 10 * 1024 * 1024 &redef; +## Number of TCP segments to buffer beyond what's been acknowledged already +## to detect retransmission inconsistencies. Zero disables any additonal +## buffering. +const tcp_max_old_segments = 0 &redef; + ## For services without a handler, these sets define originator-side ports ## that still trigger reassembly. ## @@ -1495,6 +1501,33 @@ type pkt_hdr: record { icmp: icmp_hdr &optional; ##< The ICMP header if an ICMP packet. }; +## Values extracted from the layer 2 header. +## +## .. bro:see:: pkt_hdr +type l2_hdr: record { + encap: link_encap; ##< L2 link encapsulation. + len: count; ##< Total frame length on wire. + cap_len: count; ##< Captured length. + src: string &optional; ##< L2 source (if Ethernet). + dst: string &optional; ##< L2 destination (if Ethernet). + vlan: count &optional; ##< Outermost VLAN tag if any (and Ethernet). + eth_type: count &optional; ##< Innermost Ethertype (if Ethernet). + proto: layer3_proto; ##< L3 protocol. +}; + +## A raw packet header, consisting of L2 header and everything in +## :bro:id:`pkt_hdr`. . +## +## .. bro:see:: raw_packet pkt_hdr +type raw_pkt_hdr: record { + l2: l2_hdr; ##< The layer 2 header. + ip: ip4_hdr &optional; ##< The IPv4 header if an IPv4 packet. + ip6: ip6_hdr &optional; ##< The IPv6 header if an IPv6 packet. + tcp: tcp_hdr &optional; ##< The TCP header if a TCP packet. + udp: udp_hdr &optional; ##< The UDP header if a UDP packet. + icmp: icmp_hdr &optional; ##< The ICMP header if an ICMP packet. +}; + ## A Teredo origin indication header. See :rfc:`4380` for more information ## about the Teredo protocol. ## diff --git a/scripts/base/protocols/ssl/consts.bro b/scripts/base/protocols/ssl/consts.bro index 05559ee5d0..7a95d63cc6 100644 --- a/scripts/base/protocols/ssl/consts.bro +++ b/scripts/base/protocols/ssl/consts.bro @@ -120,9 +120,9 @@ export { [18] = "signed_certificate_timestamp", [19] = "client_certificate_type", [20] = "server_certificate_type", - [21] = "padding", # temporary till 2015-03-12 + [21] = "padding", # temporary till 2016-03-12 [22] = "encrypt_then_mac", - [23] = "extended_master_secret", # temporary till 2015-09-26 + [23] = "extended_master_secret", [35] = "SessionTicket TLS", [40] = "extended_random", [13172] = "next_protocol_negotiation", @@ -169,7 +169,8 @@ export { [256] = "ffdhe2048", [257] = "ffdhe3072", [258] = "ffdhe4096", - [259] = "ffdhe8192", + [259] = "ffdhe6144", + [260] = "ffdhe8192", [0xFF01] = "arbitrary_explicit_prime_curves", [0xFF02] = "arbitrary_explicit_char2_curves" } &default=function(i: count):string { return fmt("unknown-%d", i); }; diff --git a/src/Conn.cc b/src/Conn.cc index 96e71ca52d..5c1a58a7b1 100644 --- a/src/Conn.cc +++ b/src/Conn.cc @@ -241,12 +241,8 @@ void Connection::NextPacket(double t, int is_orig, const u_char*& data, int& record_packet, int& record_content, // arguments for reproducing packets - const struct pcap_pkthdr* hdr, - const u_char* const pkt, - int hdr_size) + const Packet *pkt) { - current_hdr = hdr; - current_hdr_size = hdr_size; current_timestamp = t; current_pkt = pkt; @@ -264,8 +260,6 @@ void Connection::NextPacket(double t, int is_orig, else last_time = t; - current_hdr = 0; - current_hdr_size = 0; current_timestamp = 0; current_pkt = 0; } diff --git a/src/Conn.h b/src/Conn.h index 20e60d2617..62a1aa9613 100644 --- a/src/Conn.h +++ b/src/Conn.h @@ -86,9 +86,7 @@ public: const u_char*& data, int& record_packet, int& record_content, // arguments for reproducing packets - const struct pcap_pkthdr* hdr, - const u_char* const pkt, - int hdr_size); + const Packet *pkt); HashKey* Key() const { return key; } void ClearKey() { key = 0; } diff --git a/src/IP.cc b/src/IP.cc index 783c08da39..3a19f02d23 100644 --- a/src/IP.cc +++ b/src/IP.cc @@ -327,24 +327,31 @@ RecordVal* IP_Hdr::BuildIPHdrVal() const RecordVal* IP_Hdr::BuildPktHdrVal() const { static RecordType* pkt_hdr_type = 0; + + if ( ! pkt_hdr_type ) + pkt_hdr_type = internal_type("pkt_hdr")->AsRecordType(); + + RecordVal* pkt_hdr = new RecordVal(pkt_hdr_type); + return BuildPktHdrVal(pkt_hdr, 0); + } + +RecordVal* IP_Hdr::BuildPktHdrVal(RecordVal* pkt_hdr, int sindex) const + { static RecordType* tcp_hdr_type = 0; static RecordType* udp_hdr_type = 0; static RecordType* icmp_hdr_type = 0; - if ( ! pkt_hdr_type ) + if ( ! tcp_hdr_type ) { - pkt_hdr_type = internal_type("pkt_hdr")->AsRecordType(); tcp_hdr_type = internal_type("tcp_hdr")->AsRecordType(); udp_hdr_type = internal_type("udp_hdr")->AsRecordType(); icmp_hdr_type = internal_type("icmp_hdr")->AsRecordType(); } - RecordVal* pkt_hdr = new RecordVal(pkt_hdr_type); - if ( ip4 ) - pkt_hdr->Assign(0, BuildIPHdrVal()); + pkt_hdr->Assign(sindex + 0, BuildIPHdrVal()); else - pkt_hdr->Assign(1, BuildIPHdrVal()); + pkt_hdr->Assign(sindex + 1, BuildIPHdrVal()); // L4 header. const u_char* data = Payload(); @@ -368,7 +375,7 @@ RecordVal* IP_Hdr::BuildPktHdrVal() const tcp_hdr->Assign(6, new Val(tp->th_flags, TYPE_COUNT)); tcp_hdr->Assign(7, new Val(ntohs(tp->th_win), TYPE_COUNT)); - pkt_hdr->Assign(2, tcp_hdr); + pkt_hdr->Assign(sindex + 2, tcp_hdr); break; } @@ -381,7 +388,7 @@ RecordVal* IP_Hdr::BuildPktHdrVal() const udp_hdr->Assign(1, new PortVal(ntohs(up->uh_dport), TRANSPORT_UDP)); udp_hdr->Assign(2, new Val(ntohs(up->uh_ulen), TYPE_COUNT)); - pkt_hdr->Assign(3, udp_hdr); + pkt_hdr->Assign(sindex + 3, udp_hdr); break; } @@ -392,7 +399,7 @@ RecordVal* IP_Hdr::BuildPktHdrVal() const icmp_hdr->Assign(0, new Val(icmpp->icmp_type, TYPE_COUNT)); - pkt_hdr->Assign(4, icmp_hdr); + pkt_hdr->Assign(sindex + 4, icmp_hdr); break; } diff --git a/src/IP.h b/src/IP.h index b91c9130e4..bfd3ce8a41 100644 --- a/src/IP.h +++ b/src/IP.h @@ -574,8 +574,13 @@ public: */ RecordVal* BuildPktHdrVal() const; -private: + /** + * Same as above, but simply add our values into the record at the + * specified starting index. + */ + RecordVal* BuildPktHdrVal(RecordVal* pkt_hdr, int sindex) const; +private: const struct ip* ip4; const struct ip6_hdr* ip6; bool del; diff --git a/src/Net.cc b/src/Net.cc index af542cb1a6..2a368c47ef 100644 --- a/src/Net.cc +++ b/src/Net.cc @@ -62,10 +62,8 @@ double bro_start_network_time; // timestamp of first packet double last_watchdog_proc_time = 0.0; // value of above during last watchdog bool terminating = false; // whether we're done reading and finishing up -const struct pcap_pkthdr* current_hdr = 0; -const u_char* current_pkt = 0; +const Packet *current_pkt = 0; int current_dispatched = 0; -int current_hdr_size = 0; double current_timestamp = 0.0; iosource::PktSrc* current_pktsrc = 0; iosource::IOSource* current_iosrc = 0; @@ -109,7 +107,7 @@ RETSIGTYPE watchdog(int /* signo */) int frac_pst = int((processing_start_time - int_pst) * 1e6); - if ( current_hdr ) + if ( current_pkt ) { if ( ! pkt_dumper ) { @@ -126,12 +124,8 @@ RETSIGTYPE watchdog(int /* signo */) } if ( pkt_dumper ) - { - iosource::PktDumper::Packet p; - p.hdr = current_hdr; - p.data = current_pkt; - pkt_dumper->Dump(&p); - } + pkt_dumper->Dump(current_pkt); + } net_get_final_stats(); @@ -240,9 +234,7 @@ void expire_timers(iosource::PktSrc* src_ps) max_timer_expires - current_dispatched); } -void net_packet_dispatch(double t, const struct pcap_pkthdr* hdr, - const u_char* pkt, int hdr_size, - iosource::PktSrc* src_ps) +void net_packet_dispatch(double t, const Packet* pkt, iosource::PktSrc* src_ps) { if ( ! bro_start_network_time ) bro_start_network_time = t; @@ -278,7 +270,7 @@ void net_packet_dispatch(double t, const struct pcap_pkthdr* hdr, } } - sessions->DispatchPacket(t, hdr, pkt, hdr_size, src_ps); + sessions->NextPacket(t, pkt); mgr.Drain(); if ( sp ) diff --git a/src/Net.h b/src/Net.h index 2e466f8c7f..d19bd9083c 100644 --- a/src/Net.h +++ b/src/Net.h @@ -19,8 +19,7 @@ extern void net_get_final_stats(); extern void net_finish(int drain_events); extern void net_delete(); // Reclaim all memory, etc. extern void net_update_time(double new_network_time); -extern void net_packet_dispatch(double t, const struct pcap_pkthdr* hdr, - const u_char* pkt, int hdr_size, +extern void net_packet_dispatch(double t, const Packet* pkt, iosource::PktSrc* src_ps); extern void expire_timers(iosource::PktSrc* src_ps = 0); extern void termination_signal(); @@ -74,10 +73,8 @@ extern bool using_communication; // Snaplen passed to libpcap. extern int snaplen; -extern const struct pcap_pkthdr* current_hdr; -extern const u_char* current_pkt; +extern const Packet* current_pkt; extern int current_dispatched; -extern int current_hdr_size; extern double current_timestamp; extern iosource::PktSrc* current_pktsrc; extern iosource::IOSource* current_iosrc; diff --git a/src/NetVar.cc b/src/NetVar.cc index 123e947701..5585cf8211 100644 --- a/src/NetVar.cc +++ b/src/NetVar.cc @@ -49,6 +49,7 @@ double tcp_partial_close_delay; int tcp_max_initial_window; int tcp_max_above_hole_without_any_acks; int tcp_excessive_data_without_further_acks; +int tcp_max_old_segments; RecordType* socks_address; @@ -225,8 +226,6 @@ int suppress_local_output; double timer_mgr_inactivity_timeout; -int time_machine_profiling; - StringVal* trace_output_file; int record_all_packets; @@ -354,6 +353,7 @@ void init_net_var() opt_internal_int("tcp_max_above_hole_without_any_acks"); tcp_excessive_data_without_further_acks = opt_internal_int("tcp_excessive_data_without_further_acks"); + tcp_max_old_segments = opt_internal_int("tcp_max_old_segments"); socks_address = internal_type("SOCKS::Address")->AsRecordType(); @@ -520,7 +520,6 @@ void init_net_var() timer_mgr_inactivity_timeout = opt_internal_double("timer_mgr_inactivity_timeout"); - time_machine_profiling = opt_internal_int("time_machine_profiling"); script_id = internal_type("script_id")->AsRecordType(); id_table = internal_type("id_table")->AsTableType(); diff --git a/src/NetVar.h b/src/NetVar.h index bf2d9a5712..97018121f9 100644 --- a/src/NetVar.h +++ b/src/NetVar.h @@ -52,6 +52,7 @@ extern double tcp_reset_delay; extern int tcp_max_initial_window; extern int tcp_max_above_hole_without_any_acks; extern int tcp_excessive_data_without_further_acks; +extern int tcp_max_old_segments; extern RecordType* socks_address; @@ -229,8 +230,6 @@ extern int suppress_local_output; extern double timer_mgr_inactivity_timeout; -extern int time_machine_profiling; - extern StringVal* trace_output_file; extern int record_all_packets; diff --git a/src/Reassem.cc b/src/Reassem.cc index 8bf965427b..bfac7f7a07 100644 --- a/src/Reassem.cc +++ b/src/Reassem.cc @@ -34,12 +34,52 @@ uint64 Reassembler::total_size = 0; Reassembler::Reassembler(uint64 init_seq) { blocks = last_block = 0; + old_blocks = last_old_block = 0; + total_old_blocks = max_old_blocks = 0; trim_seq = last_reassem_seq = init_seq; } Reassembler::~Reassembler() { ClearBlocks(); + ClearOldBlocks(); + } + +void Reassembler::CheckOverlap(DataBlock *head, DataBlock *tail, + uint64 seq, uint64 len, const u_char* data) + { + if ( ! head || ! tail ) + return; + + uint64 upper = (seq + len); + + for ( DataBlock* b = head; b; b = b->next ) + { + uint64 nseq = seq; + uint64 nupper = upper; + const u_char* ndata = data; + + if ( nupper <= b->seq ) + continue; + + if ( nseq >= b->upper ) + continue; + + if ( nseq < b->seq ) + { + ndata += (b->seq - seq); + nseq = b->seq; + } + + if ( nupper > b->upper ) + nupper = b->upper; + + uint64 overlap_offset = (nseq - b->seq); + uint64 overlap_len = (nupper - nseq); + + if ( overlap_len ) + Overlap(&b->block[overlap_offset], ndata, overlap_len); + } } void Reassembler::NewBlock(double t, uint64 seq, uint64 len, const u_char* data) @@ -49,10 +89,14 @@ void Reassembler::NewBlock(double t, uint64 seq, uint64 len, const u_char* data) uint64 upper_seq = seq + len; + CheckOverlap(old_blocks, last_old_block, seq, len, data); + if ( upper_seq <= trim_seq ) // Old data, don't do any work for it. return; + CheckOverlap(blocks, last_block, seq, len, data); + if ( seq < trim_seq ) { // Partially old data, just keep the good stuff. uint64 amount_old = trim_seq - seq; @@ -119,7 +163,36 @@ uint64 Reassembler::TrimToSeq(uint64 seq) num_missing += seq - blocks->upper; } - delete blocks; + if ( max_old_blocks ) + { + // Move block over to old_blocks queue. + blocks->next = 0; + + if ( last_old_block ) + { + blocks->prev = last_old_block; + last_old_block->next = blocks; + } + else + { + blocks->prev = 0; + old_blocks = blocks; + } + + last_old_block = blocks; + total_old_blocks++; + + while ( old_blocks && total_old_blocks > max_old_blocks ) + { + DataBlock* next = old_blocks->next; + delete old_blocks; + old_blocks = next; + total_old_blocks--; + } + } + + else + delete blocks; blocks = b; } @@ -156,6 +229,18 @@ void Reassembler::ClearBlocks() last_block = 0; } +void Reassembler::ClearOldBlocks() + { + while ( old_blocks ) + { + DataBlock* b = old_blocks->next; + delete old_blocks; + old_blocks = b; + } + + last_old_block = 0; + } + uint64 Reassembler::TotalSize() const { uint64 size = 0; @@ -218,7 +303,7 @@ DataBlock* Reassembler::AddAndCheck(DataBlock* b, uint64 seq, uint64 upper, return new_b; } - // The blocks overlap, complain. + // The blocks overlap. if ( seq < b->seq ) { // The new block has a prefix that comes before b. @@ -239,8 +324,6 @@ DataBlock* Reassembler::AddAndCheck(DataBlock* b, uint64 seq, uint64 upper, uint64 b_len = b->upper - overlap_start; uint64 overlap_len = min(new_b_len, b_len); - Overlap(&b->block[overlap_offset], data, overlap_len); - if ( overlap_len < new_b_len ) { // Recurse to resolve remainder of the new data. diff --git a/src/Reassem.h b/src/Reassem.h index 39617f7816..e55c809990 100644 --- a/src/Reassem.h +++ b/src/Reassem.h @@ -36,6 +36,7 @@ public: // Delete all held blocks. void ClearBlocks(); + void ClearOldBlocks(); int HasBlocks() const { return blocks != 0; } uint64 LastReassemSeq() const { return last_reassem_seq; } @@ -50,6 +51,8 @@ public: // Sum over all data buffered in some reassembler. static uint64 TotalMemoryAllocation() { return total_size; } + void SetMaxOldBlocks(uint32 count) { max_old_blocks = count; } + protected: Reassembler() { } @@ -65,10 +68,19 @@ protected: DataBlock* AddAndCheck(DataBlock* b, uint64 seq, uint64 upper, const u_char* data); + void CheckOverlap(DataBlock *head, DataBlock *tail, + uint64 seq, uint64 len, const u_char* data); + DataBlock* blocks; DataBlock* last_block; + + DataBlock* old_blocks; + DataBlock* last_old_block; + uint64 last_reassem_seq; uint64 trim_seq; // how far we've trimmed + uint32 max_old_blocks; + uint32 total_old_blocks; static uint64 total_size; }; diff --git a/src/RemoteSerializer.cc b/src/RemoteSerializer.cc index d359c1ea5d..44ec678a0f 100644 --- a/src/RemoteSerializer.cc +++ b/src/RemoteSerializer.cc @@ -1453,7 +1453,7 @@ void RemoteSerializer::Process() if ( packets.length() ) { BufferedPacket* bp = packets[0]; - Packet* p = bp->p; + const Packet* p = bp->p; // FIXME: The following chunk of code is copied from // net_packet_dispatch(). We should change that function @@ -1465,14 +1465,12 @@ void RemoteSerializer::Process() current_dispatched = tmgr->Advance(network_time, max_timer_expires); - current_hdr = p->hdr; - current_pkt = p->pkt; + current_pkt = p; current_pktsrc = 0; current_iosrc = this; - sessions->NextPacket(p->time, p->hdr, p->pkt, p->hdr_size); + sessions->NextPacket(p->time, p); mgr.Drain(); - current_hdr = 0; // done with these current_pkt = 0; current_iosrc = 0; diff --git a/src/Serializer.cc b/src/Serializer.cc index 7306b0ded0..49e57c0216 100644 --- a/src/Serializer.cc +++ b/src/Serializer.cc @@ -1122,110 +1122,3 @@ void EventPlayer::Process() ne_time = 0; } - -void Packet::Describe(ODesc* d) const - { - const IP_Hdr ip = IP(); - d->Add(ip.SrcAddr()); - d->Add("->"); - d->Add(ip.DstAddr()); - } - -bool Packet::Serialize(SerialInfo* info) const - { - return SERIALIZE(uint32(hdr->ts.tv_sec)) && - SERIALIZE(uint32(hdr->ts.tv_usec)) && - SERIALIZE(uint32(hdr->len)) && - SERIALIZE(link_type) && - info->s->Write(tag.c_str(), 0, "tag") && - info->s->Write((const char*) pkt, hdr->caplen, "data"); - } - -static BroFile* profiling_output = 0; - -#ifdef DEBUG -static iosource::PktDumper* dump = 0; -#endif - -Packet* Packet::Unserialize(UnserialInfo* info) - { - Packet* p = new Packet("", true); - pcap_pkthdr* hdr = new pcap_pkthdr; - - uint32 tv_sec, tv_usec, len; - - if ( ! (UNSERIALIZE(&tv_sec) && - UNSERIALIZE(&tv_usec) && - UNSERIALIZE(&len) && - UNSERIALIZE(&p->link_type)) ) - { - delete p; - delete hdr; - return 0; - } - - hdr->ts.tv_sec = tv_sec; - hdr->ts.tv_usec = tv_usec; - hdr->len = len; - - char* tag; - if ( ! info->s->Read((char**) &tag, 0, "tag") ) - { - delete p; - delete hdr; - return 0; - } - - char* pkt; - int caplen; - if ( ! info->s->Read((char**) &pkt, &caplen, "data") ) - { - delete p; - delete hdr; - delete [] tag; - return 0; - } - - hdr->caplen = uint32(caplen); - p->hdr = hdr; - p->pkt = (u_char*) pkt; - p->tag = tag; - p->hdr_size = iosource::PktSrc::GetLinkHeaderSize(p->link_type); - - delete [] tag; - - // For the global timer manager, we take the global network_time as the - // packet's timestamp for feeding it into our packet loop. - if ( p->tag == "" ) - p->time = timer_mgr->Time(); - else - p->time = p->hdr->ts.tv_sec + double(p->hdr->ts.tv_usec) / 1e6; - - if ( time_machine_profiling ) - { - if ( ! profiling_output ) - profiling_output = - new BroFile("tm-prof.packets.log", "w"); - - profiling_output->Write(fmt("%.6f %s %d\n", current_time(), - (p->tag != "" ? p->tag.c_str() : "-"), hdr->len)); - } - -#ifdef DEBUG - if ( debug_logger.IsEnabled(DBG_TM) ) - { - if ( ! dump ) - dump = iosource_mgr->OpenPktDumper("tm.pcap", true); - - if ( dump ) - { - iosource::PktDumper::Packet dp; - dp.hdr = p->hdr; - dp.data = p->pkt; - dump->Dump(&dp); - } - } -#endif - - return p; - } diff --git a/src/Serializer.h b/src/Serializer.h index 558dce2086..43a9943cc5 100644 --- a/src/Serializer.h +++ b/src/Serializer.h @@ -378,64 +378,6 @@ protected: }; - -// A link-layer packet. -// -// Eventually we should use something like this consistently throughout Bro, -// replacing the current packet arguments in functions like *::NextPacket(). -// Before doing this, though, we should consider provisioning for packet -// formats other than just libpcap by designing a more abstract interface. -// -// Note that for serialization we don't use much of the support provided by -// the serialization framework. Serialize/Unserialize do all the work by -// themselves. In particular, Packets aren't derived from SerialObj. They are -// completely seperate and self-contained entities, and we don't need any of -// the sophisticated features like object caching. - -class Packet { -public: - // Argument is whether we should delete associatd memory upon - // destruction. - Packet(TimerMgr::Tag arg_tag, bool arg_free = false) - { - time = 0.0; - hdr = 0; - pkt = 0; - hdr_size = 0; - free = arg_free; - tag = arg_tag; - link_type = 0; - } - - ~Packet() - { - if ( free ) - { - delete hdr; - delete [] pkt; - } - } - - const IP_Hdr IP() const - { return IP_Hdr((struct ip *) (pkt + hdr_size), true); } - - void Describe(ODesc* d) const; - - bool Serialize(SerialInfo* info) const; - static Packet* Unserialize(UnserialInfo* info); - - const struct pcap_pkthdr* hdr; - const u_char* pkt; - TimerMgr::Tag tag; - uint32 link_type; - - double time; - int hdr_size; - -private: - bool free; -}; - extern FileSerializer* event_serializer; extern FileSerializer* state_serializer; diff --git a/src/Sessions.cc b/src/Sessions.cc index 086216e93d..7b7974bfce 100644 --- a/src/Sessions.cc +++ b/src/Sessions.cc @@ -165,98 +165,75 @@ void NetSessions::Done() { } -void NetSessions::DispatchPacket(double t, const struct pcap_pkthdr* hdr, - const u_char* pkt, int hdr_size, - iosource::PktSrc* src_ps) +void NetSessions::NextPacket(double t, const Packet* pkt) { - const struct ip* ip_hdr = 0; - const u_char* ip_data = 0; - int proto = 0; + SegmentProfiler(segment_logger, "dispatching-packet"); - if ( hdr->caplen >= hdr_size + sizeof(struct ip) ) + if ( raw_packet ) { - ip_hdr = reinterpret_cast(pkt + hdr_size); - if ( hdr->caplen >= unsigned(hdr_size + (ip_hdr->ip_hl << 2)) ) - ip_data = pkt + hdr_size + (ip_hdr->ip_hl << 2); + val_list* vl = new val_list(); + vl->append(pkt->BuildPktHdrVal()); + mgr.QueueEvent(raw_packet, vl); } - if ( encap_hdr_size > 0 && ip_data ) - // Blanket encapsulation - hdr_size += encap_hdr_size; - - NextPacket(t, hdr, pkt, hdr_size); - } - -void NetSessions::NextPacket(double t, const struct pcap_pkthdr* hdr, - const u_char* const pkt, int hdr_size) - { - SegmentProfiler(segment_logger, "processing-packet"); if ( pkt_profiler ) - pkt_profiler->ProfilePkt(t, hdr->caplen); + pkt_profiler->ProfilePkt(t, pkt->cap_len); ++num_packets_processed; dump_this_packet = 0; if ( record_all_packets ) - DumpPacket(hdr, pkt); + DumpPacket(pkt); - // ### The following isn't really correct. What we *should* - // do is understanding the different link layers in order to - // find the network-layer protocol ID. That's a big - // portability pain, though, unless we just assume everything's - // Ethernet .... not great, given the potential need to deal - // with PPP or FDDI (for some older traces). So instead - // we look to see if what we have is consistent with an - // IPv4 packet. If not, it's either ARP or IPv6 or weird. - - if ( hdr_size > static_cast(hdr->caplen) ) + if ( pkt->hdr_size > pkt->cap_len ) { - Weird("truncated_link_frame", hdr, pkt); + Weird("truncated_link_frame", pkt); return; } - uint32 caplen = hdr->caplen - hdr_size; - if ( caplen < sizeof(struct ip) ) - { - Weird("truncated_IP", hdr, pkt); - return; - } + uint32 caplen = pkt->cap_len - pkt->hdr_size; - const struct ip* ip = (const struct ip*) (pkt + hdr_size); - - if ( ip->ip_v == 4 ) + if ( pkt->l3_proto == L3_IPV4 ) { - IP_Hdr ip_hdr(ip, false); - DoNextPacket(t, hdr, &ip_hdr, pkt, hdr_size, 0); - } - - else if ( ip->ip_v == 6 ) - { - if ( caplen < sizeof(struct ip6_hdr) ) + if ( caplen < sizeof(struct ip) ) { - Weird("truncated_IP", hdr, pkt); + Weird("truncated_IP", pkt); return; } - IP_Hdr ip_hdr((const struct ip6_hdr*) (pkt + hdr_size), false, caplen); - DoNextPacket(t, hdr, &ip_hdr, pkt, hdr_size, 0); + const struct ip* ip = (const struct ip*) (pkt->data + pkt->hdr_size); + IP_Hdr ip_hdr(ip, false); + DoNextPacket(t, pkt, &ip_hdr, 0); } - else if ( analyzer::arp::ARP_Analyzer::IsARP(pkt, hdr_size) ) + else if ( pkt->l3_proto == L3_IPV6 ) + { + if ( caplen < sizeof(struct ip6_hdr) ) + { + Weird("truncated_IP", pkt); + return; + } + + IP_Hdr ip_hdr((const struct ip6_hdr*) (pkt->data + pkt->hdr_size), false, caplen); + DoNextPacket(t, pkt, &ip_hdr, 0); + } + + else if ( pkt->l3_proto == L3_ARP ) { if ( arp_analyzer ) - arp_analyzer->NextPacket(t, hdr, pkt, hdr_size); + arp_analyzer->NextPacket(t, pkt); } else { - Weird("unknown_packet_type", hdr, pkt); + Weird("unknown_packet_type", pkt); return; } + if ( dump_this_packet && ! record_all_packets ) - DumpPacket(hdr, pkt); + DumpPacket(pkt); } int NetSessions::CheckConnectionTag(Connection* conn) @@ -337,26 +314,25 @@ static unsigned int gre_header_len(uint16 flags) return len; } -void NetSessions::DoNextPacket(double t, const struct pcap_pkthdr* hdr, - const IP_Hdr* ip_hdr, const u_char* const pkt, - int hdr_size, const EncapsulationStack* encapsulation) +void NetSessions::DoNextPacket(double t, const Packet* pkt, const IP_Hdr* ip_hdr, + const EncapsulationStack* encapsulation) { - uint32 caplen = hdr->caplen - hdr_size; + uint32 caplen = pkt->cap_len - pkt->hdr_size; const struct ip* ip4 = ip_hdr->IP4_Hdr(); uint32 len = ip_hdr->TotalLen(); if ( len == 0 ) { // TCP segmentation offloading can zero out the ip_len field. - Weird("ip_hdr_len_zero", hdr, pkt, encapsulation); + Weird("ip_hdr_len_zero", pkt, encapsulation); // Cope with the zero'd out ip_len field by using the caplen. - len = hdr->caplen - hdr_size; + len = pkt->cap_len - pkt->hdr_size; } - if ( hdr->len < len + hdr_size ) + if ( pkt->len < len + pkt->hdr_size ) { - Weird("truncated_IP", hdr, pkt, encapsulation); + Weird("truncated_IP", pkt, encapsulation); return; } @@ -368,7 +344,7 @@ void NetSessions::DoNextPacket(double t, const struct pcap_pkthdr* hdr, if ( ! ignore_checksums && ip4 && ones_complement_checksum((void*) ip4, ip_hdr_len, 0) != 0xffff ) { - Weird("bad_IP_checksum", hdr, pkt, encapsulation); + Weird("bad_IP_checksum", pkt, encapsulation); return; } @@ -393,7 +369,7 @@ void NetSessions::DoNextPacket(double t, const struct pcap_pkthdr* hdr, } else { - f = NextFragment(t, ip_hdr, pkt + hdr_size); + f = NextFragment(t, ip_hdr, pkt->data + pkt->hdr_size); const IP_Hdr* ih = f->ReassembledPkt(); if ( ! ih ) // It didn't reassemble into anything yet. @@ -437,7 +413,7 @@ void NetSessions::DoNextPacket(double t, const struct pcap_pkthdr* hdr, if ( ! ignore_checksums && mobility_header_checksum(ip_hdr) != 0xffff ) { - Weird("bad_MH_checksum", hdr, pkt, encapsulation); + Weird("bad_MH_checksum", pkt, encapsulation); return; } @@ -449,7 +425,7 @@ void NetSessions::DoNextPacket(double t, const struct pcap_pkthdr* hdr, } if ( ip_hdr->NextProto() != IPPROTO_NONE ) - Weird("mobility_piggyback", hdr, pkt, encapsulation); + Weird("mobility_piggyback", pkt, encapsulation); return; } @@ -457,7 +433,7 @@ void NetSessions::DoNextPacket(double t, const struct pcap_pkthdr* hdr, int proto = ip_hdr->NextProto(); - if ( CheckHeaderTrunc(proto, len, caplen, hdr, pkt, encapsulation) ) + if ( CheckHeaderTrunc(proto, len, caplen, pkt, encapsulation) ) return; const u_char* data = ip_hdr->Payload(); @@ -594,7 +570,7 @@ void NetSessions::DoNextPacket(double t, const struct pcap_pkthdr* hdr, if ( ppp_proto != 0x0021 && ppp_proto != 0x0057 ) { - Weird("non_ip_packet_in_egre", ip_hdr, encapsulation); + Weird("non_ip_packet_in_encap", ip_hdr, encapsulation); return; } @@ -664,7 +640,7 @@ void NetSessions::DoNextPacket(double t, const struct pcap_pkthdr* hdr, else it->second.second = network_time; - DoNextInnerPacket(t, hdr, inner, encapsulation, + DoNextInnerPacket(t, pkt, inner, encapsulation, ip_tunnels[tunnel_idx].first); return; @@ -677,13 +653,13 @@ void NetSessions::DoNextPacket(double t, const struct pcap_pkthdr* hdr, // not sure the reason for the No Next header in the packet. if ( ! ( encapsulation && encapsulation->LastType() == BifEnum::Tunnel::TEREDO ) ) - Weird("ipv6_no_next", hdr, pkt); + Weird("ipv6_no_next", pkt); return; } default: - Weird(fmt("unknown_protocol_%d", proto), hdr, pkt, encapsulation); + Weird(fmt("unknown_protocol_%d", proto), pkt, encapsulation); return; } @@ -756,8 +732,7 @@ void NetSessions::DoNextPacket(double t, const struct pcap_pkthdr* hdr, pkt_hdr_val ? pkt_hdr_val->Ref() : ip_hdr->BuildPktHdrVal()); conn->NextPacket(t, is_orig, ip_hdr, len, caplen, data, - record_packet, record_content, - hdr, pkt, hdr_size); + record_packet, record_content, pkt); if ( f ) { @@ -772,40 +747,53 @@ void NetSessions::DoNextPacket(double t, const struct pcap_pkthdr* hdr, else { - int hdr_len = data - pkt; - DumpPacket(hdr, pkt, hdr_len); // just save the header + int hdr_len = data - pkt->data; + DumpPacket(pkt, hdr_len); // just save the header } } } -void NetSessions::DoNextInnerPacket(double t, const struct pcap_pkthdr* hdr, +void NetSessions::DoNextInnerPacket(double t, const Packet* pkt, const IP_Hdr* inner, const EncapsulationStack* prev, const EncapsulatingConn& ec) { - struct pcap_pkthdr fake_hdr; - fake_hdr.caplen = fake_hdr.len = inner->TotalLen(); + uint32 caplen, len; + caplen = len = inner->TotalLen(); - if ( hdr ) - fake_hdr.ts = hdr->ts; + struct timeval ts; + int link_type; + Layer3Proto l3_proto; + + if ( pkt ) + ts = pkt->ts; else { - fake_hdr.ts.tv_sec = (time_t) network_time; - fake_hdr.ts.tv_usec = (suseconds_t) - ((network_time - (double)fake_hdr.ts.tv_sec) * 1000000); + ts.tv_sec = (time_t) network_time; + ts.tv_usec = (suseconds_t) + ((network_time - (double)ts.tv_sec) * 1000000); } - const u_char* pkt = 0; + const u_char* data = 0; if ( inner->IP4_Hdr() ) - pkt = (const u_char*) inner->IP4_Hdr(); + { + data = (const u_char*) inner->IP4_Hdr(); + l3_proto = L3_IPV4; + } else - pkt = (const u_char*) inner->IP6_Hdr(); + { + data = (const u_char*) inner->IP6_Hdr(); + l3_proto = L3_IPV6; + } EncapsulationStack* outer = prev ? new EncapsulationStack(*prev) : new EncapsulationStack(); outer->Add(ec); - DoNextPacket(t, &fake_hdr, inner, pkt, 0, outer); + // Construct fake packet for DoNextPacket + Packet p; + p.Init(DLT_RAW, &ts, caplen, len, data, false, ""); + DoNextPacket(t, &p, inner, outer); delete inner; delete outer; @@ -843,8 +831,7 @@ int NetSessions::ParseIPPacket(int caplen, const u_char* const pkt, int proto, } bool NetSessions::CheckHeaderTrunc(int proto, uint32 len, uint32 caplen, - const struct pcap_pkthdr* h, - const u_char* p, const EncapsulationStack* encap) + const Packet* p, const EncapsulationStack* encap) { uint32 min_hdr_len = 0; switch ( proto ) { @@ -876,13 +863,13 @@ bool NetSessions::CheckHeaderTrunc(int proto, uint32 len, uint32 caplen, if ( len < min_hdr_len ) { - Weird("truncated_header", h, p, encap); + Weird("truncated_header", p, encap); return true; } if ( caplen < min_hdr_len ) { - Weird("internally_truncated_header", h, p, encap); + Weird("internally_truncated_header", p, encap); return true; } @@ -1387,45 +1374,26 @@ void NetSessions::ExpireTimerMgrs() } } -void NetSessions::DumpPacket(const struct pcap_pkthdr* hdr, - const u_char* pkt, int len) +void NetSessions::DumpPacket(const Packet *pkt, int len) { if ( ! pkt_dumper ) return; - if ( len == 0 ) + if ( len != 0 ) { - iosource::PktDumper::Packet p; - p.hdr = hdr; - p.data = pkt; - pkt_dumper->Dump(&p); + if ( (uint32)len > pkt->cap_len ) + reporter->Warning("bad modified caplen"); + else + const_cast(pkt)->cap_len = len; } - else - { - struct pcap_pkthdr h = *hdr; - h.caplen = len; - if ( h.caplen > hdr->caplen ) - reporter->InternalError("bad modified caplen"); - - iosource::PktDumper::Packet p; - p.hdr = &h; - p.data = pkt; - pkt_dumper->Dump(&p); - } + pkt_dumper->Dump(pkt); } -void NetSessions::Internal(const char* msg, const struct pcap_pkthdr* hdr, - const u_char* pkt) +void NetSessions::Weird(const char* name, const Packet* pkt, + const EncapsulationStack* encap) { - DumpPacket(hdr, pkt); - reporter->InternalError("%s", msg); - } - -void NetSessions::Weird(const char* name, const struct pcap_pkthdr* hdr, - const u_char* pkt, const EncapsulationStack* encap) - { - if ( hdr ) + if ( pkt ) dump_this_packet = 1; if ( encap && encap->LastType() != BifEnum::Tunnel::NONE ) diff --git a/src/Sessions.h b/src/Sessions.h index c46c092263..1780bbdb24 100644 --- a/src/Sessions.h +++ b/src/Sessions.h @@ -15,8 +15,6 @@ #include -struct pcap_pkthdr; - class EncapsulationStack; class Connection; class OSFingerprint; @@ -68,12 +66,8 @@ public: NetSessions(); ~NetSessions(); - // Main entry point for packet processing. Dispatches the packet - // either through NextPacket(), optionally employing the packet - // sorter first. - void DispatchPacket(double t, const struct pcap_pkthdr* hdr, - const u_char* const pkt, int hdr_size, - iosource::PktSrc* src_ps); + // Main entry point for packet processing. + void NextPacket(double t, const Packet* pkt); void Done(); // call to drain events before destructing @@ -106,8 +100,8 @@ public: void GetStats(SessionStats& s) const; - void Weird(const char* name, const struct pcap_pkthdr* hdr, - const u_char* pkt, const EncapsulationStack* encap = 0); + void Weird(const char* name, const Packet* pkt, + const EncapsulationStack* encap = 0); void Weird(const char* name, const IP_Hdr* ip, const EncapsulationStack* encap = 0); @@ -133,9 +127,8 @@ public: icmp_conns.Length(); } - void DoNextPacket(double t, const struct pcap_pkthdr* hdr, - const IP_Hdr* ip_hdr, const u_char* const pkt, - int hdr_size, const EncapsulationStack* encapsulation); + void DoNextPacket(double t, const Packet *pkt, const IP_Hdr* ip_hdr, + const EncapsulationStack* encapsulation); /** * Wrapper that recurses on DoNextPacket for encapsulated IP packets. @@ -151,7 +144,7 @@ public: * the most-recently found depth of encapsulation. * @param ec The most-recently found depth of encapsulation. */ - void DoNextInnerPacket(double t, const struct pcap_pkthdr* hdr, + void DoNextInnerPacket(double t, const Packet *pkt, const IP_Hdr* inner, const EncapsulationStack* prev, const EncapsulatingConn& ec); @@ -218,24 +211,16 @@ protected: TransportProto transport_proto, uint8 tcp_flags, bool& flip_roles); - void NextPacket(double t, const struct pcap_pkthdr* hdr, - const u_char* const pkt, int hdr_size); - // Record the given packet (if a dumper is active). If len=0 // then the whole packet is recorded, otherwise just the first // len bytes. - void DumpPacket(const struct pcap_pkthdr* hdr, const u_char* pkt, - int len=0); - - void Internal(const char* msg, const struct pcap_pkthdr* hdr, - const u_char* pkt); + void DumpPacket(const Packet *pkt, int len=0); // For a given protocol, checks whether the header's length as derived // from lower-level headers or the length actually captured is less // than that protocol's minimum header size. bool CheckHeaderTrunc(int proto, uint32 len, uint32 caplen, - const struct pcap_pkthdr* hdr, const u_char* pkt, - const EncapsulationStack* encap); + const Packet *pkt, const EncapsulationStack* encap); CompositeHash* ch; PDict(Connection) tcp_conns; diff --git a/src/analyzer/protocol/arp/ARP.cc b/src/analyzer/protocol/arp/ARP.cc index b3ef5383ce..5cbb25451b 100644 --- a/src/analyzer/protocol/arp/ARP.cc +++ b/src/analyzer/protocol/arp/ARP.cc @@ -19,21 +19,6 @@ ARP_Analyzer::~ARP_Analyzer() { } -bool ARP_Analyzer::IsARP(const u_char* pkt, int hdr_size) - { - unsigned short network_protocol = - *(unsigned short*) (pkt + hdr_size - 2); - - switch ( ntohs(network_protocol) ) { - case ETHERTYPE_ARP: - case ETHERTYPE_REVARP: - return true; - default: - return false; - } - } - - // Argh! FreeBSD and Linux have almost completely different net/if_arp.h . // ... and on Solaris we are missing half of the ARPOP codes, so define // them here as necessary: @@ -93,16 +78,16 @@ bool ARP_Analyzer::IsARP(const u_char* pkt, int hdr_size) #endif -void ARP_Analyzer::NextPacket(double t, const struct pcap_pkthdr* hdr, - const u_char* const pkt, int hdr_size) +void ARP_Analyzer::NextPacket(double t, const Packet* pkt) { + const u_char *data = pkt->data; // Check whether the packet is OK ("inspired" in tcpdump's print-arp.c). const struct arp_pkthdr* ah = - (const struct arp_pkthdr*) (pkt + hdr_size); + (const struct arp_pkthdr*) (data + pkt->hdr_size); // Check the size. - int min_length = (ar_tpa(ah) - (char*) (pkt + hdr_size)) + ah->ar_pln; - int real_length = hdr->caplen - hdr_size; + int min_length = (ar_tpa(ah) - (char*) (data + pkt->hdr_size)) + ah->ar_pln; + int real_length = pkt->cap_len - pkt->hdr_size; if ( min_length > real_length ) { Corrupted("truncated_ARP"); @@ -158,7 +143,7 @@ void ARP_Analyzer::NextPacket(double t, const struct pcap_pkthdr* hdr, // Check MAC src address = ARP sender MAC address. - if ( memcmp((const char*) (pkt+6), ar_sha(ah), ah->ar_hln) ) + if ( memcmp((const char*) (data+6), ar_sha(ah), ah->ar_hln) ) { BadARP(ah, "weird-arp-sha"); return; @@ -167,12 +152,12 @@ void ARP_Analyzer::NextPacket(double t, const struct pcap_pkthdr* hdr, // Check the code is supported. switch ( ntohs(ah->ar_op) ) { case ARPOP_REQUEST: - RREvent(arp_request, pkt+6, pkt, + RREvent(arp_request, data+6, data, ar_spa(ah), ar_sha(ah), ar_tpa(ah), ar_tha(ah)); break; case ARPOP_REPLY: - RREvent(arp_reply, pkt+6, pkt, + RREvent(arp_reply, data+6, data, ar_spa(ah), ar_sha(ah), ar_tpa(ah), ar_tha(ah)); break; diff --git a/src/analyzer/protocol/arp/ARP.h b/src/analyzer/protocol/arp/ARP.h index ad771b8db9..1778f5e200 100644 --- a/src/analyzer/protocol/arp/ARP.h +++ b/src/analyzer/protocol/arp/ARP.h @@ -25,6 +25,8 @@ #include "NetVar.h" +class Packet; + extern "C" { #include } @@ -36,17 +38,13 @@ public: ARP_Analyzer(); virtual ~ARP_Analyzer(); - void NextPacket(double t, const struct pcap_pkthdr* hdr, - const u_char* const pkt, int hdr_size); + void NextPacket(double t, const Packet* pkt); void Describe(ODesc* d) const; void RREvent(EventHandlerPtr e, const u_char* src, const u_char* dst, const char* spa, const char* sha, const char* tpa, const char* tha); - // Whether a packet is of interest for ARP analysis. - static bool IsARP(const u_char* pkt, int hdr_size); - protected: AddrVal* ConstructAddrVal(const void* addr); StringVal* EthAddrToStr(const u_char* addr); diff --git a/src/analyzer/protocol/http/HTTP.cc b/src/analyzer/protocol/http/HTTP.cc index f60d1372ac..ff72c6f350 100644 --- a/src/analyzer/protocol/http/HTTP.cc +++ b/src/analyzer/protocol/http/HTTP.cc @@ -1025,8 +1025,11 @@ void HTTP_Analyzer::DeliverStream(int len, const u_char* data, bool is_orig) } else { - ProtocolViolation("not a http reply line"); - reply_state = EXPECT_REPLY_NOTHING; + if ( line != end_of_line ) + { + ProtocolViolation("not a http reply line"); + reply_state = EXPECT_REPLY_NOTHING; + } } break; diff --git a/src/analyzer/protocol/mime/MIME.cc b/src/analyzer/protocol/mime/MIME.cc index f9ddb7cd3d..cbc1abd17d 100644 --- a/src/analyzer/protocol/mime/MIME.cc +++ b/src/analyzer/protocol/mime/MIME.cc @@ -245,11 +245,16 @@ int MIME_get_field_name(int len, const char* data, data_chunk_t* name) } // See RFC 2045, page 12. -int MIME_is_tspecial (char ch) +int MIME_is_tspecial (char ch, bool is_boundary = false) { - return ch == '(' || ch == ')' || ch == '<' || ch == '>' || ch == '@' || - ch == ',' || ch == ';' || ch == ':' || ch == '\\' || ch == '"' || - ch == '/' || ch == '[' || ch == ']' || ch == '?' || ch == '='; + if ( is_boundary ) + return ch == '(' || ch == ')' || ch == '@' || + ch == ',' || ch == ';' || ch == ':' || ch == '\\' || ch == '"' || + ch == '/' || ch == '[' || ch == ']' || ch == '?' || ch == '='; + else + return ch == '(' || ch == ')' || ch == '<' || ch == '>' || ch == '@' || + ch == ',' || ch == ';' || ch == ':' || ch == '\\' || ch == '"' || + ch == '/' || ch == '[' || ch == ']' || ch == '?' || ch == '='; } int MIME_is_field_name_char (char ch) @@ -257,26 +262,27 @@ int MIME_is_field_name_char (char ch) return ch >= 33 && ch <= 126 && ch != ':'; } -int MIME_is_token_char (char ch) +int MIME_is_token_char (char ch, bool is_boundary = false) { - return ch >= 33 && ch <= 126 && ! MIME_is_tspecial(ch); + return ch >= 33 && ch <= 126 && ! MIME_is_tspecial(ch, is_boundary); } // See RFC 2045, page 12. // A token is composed of characters that are not SPACE, CTLs or tspecials -int MIME_get_token(int len, const char* data, data_chunk_t* token) +int MIME_get_token(int len, const char* data, data_chunk_t* token, + bool is_boundary) { int i = MIME_skip_lws_comments(len, data); while ( i < len ) { int j; - if ( MIME_is_token_char(data[i]) ) + if ( MIME_is_token_char(data[i], is_boundary) ) { token->data = (data + i); for ( j = i; j < len; ++j ) { - if ( ! MIME_is_token_char(data[j]) ) + if ( ! MIME_is_token_char(data[j], is_boundary) ) break; } @@ -358,7 +364,7 @@ int MIME_get_quoted_string(int len, const char* data, data_chunk_t* str) return -1; } -int MIME_get_value(int len, const char* data, BroString*& buf) +int MIME_get_value(int len, const char* data, BroString*& buf, bool is_boundary) { int offset = MIME_skip_lws_comments(len, data); @@ -379,7 +385,7 @@ int MIME_get_value(int len, const char* data, BroString*& buf) else { data_chunk_t str; - int end = MIME_get_token(len, data, &str); + int end = MIME_get_token(len, data, &str, is_boundary); if ( end < 0 ) return -1; @@ -862,8 +868,22 @@ int MIME_Entity::ParseFieldParameters(int len, const char* data) len -= offset; BroString* val = 0; - // token or quoted-string - offset = MIME_get_value(len, data, val); + + if ( current_field_type == MIME_CONTENT_TYPE && + content_type == CONTENT_TYPE_MULTIPART && + strcasecmp_n(attr, "boundary") == 0 ) + { + // token or quoted-string (and some lenience for characters + // not explicitly allowed by the RFC, but encountered in the wild) + offset = MIME_get_value(len, data, val, true); + data_chunk_t vd = get_data_chunk(val); + multipart_boundary = new BroString((const u_char*)vd.data, + vd.length, 1); + } + else + // token or quoted-string + offset = MIME_get_value(len, data, val); + if ( offset < 0 ) { IllegalFormat("value not found in parameter specification"); @@ -873,8 +893,6 @@ int MIME_Entity::ParseFieldParameters(int len, const char* data) data += offset; len -= offset; - - ParseParameter(attr, get_data_chunk(val)); delete val; } @@ -919,24 +937,6 @@ void MIME_Entity::ParseContentEncoding(data_chunk_t encoding_mechanism) content_encoding = i; } -void MIME_Entity::ParseParameter(data_chunk_t attr, data_chunk_t val) - { - switch ( current_field_type ) { - case MIME_CONTENT_TYPE: - if ( content_type == CONTENT_TYPE_MULTIPART && - strcasecmp_n(attr, "boundary") == 0 ) - multipart_boundary = new BroString((const u_char*)val.data, val.length, 1); - break; - - case MIME_CONTENT_TRANSFER_ENCODING: - break; - - default: - break; - } - } - - int MIME_Entity::CheckBoundaryDelimiter(int len, const char* data) { if ( ! multipart_boundary ) diff --git a/src/analyzer/protocol/mime/MIME.h b/src/analyzer/protocol/mime/MIME.h index a33779557c..8c7fdd4326 100644 --- a/src/analyzer/protocol/mime/MIME.h +++ b/src/analyzer/protocol/mime/MIME.h @@ -117,7 +117,6 @@ protected: void ParseContentType(data_chunk_t type, data_chunk_t sub_type); void ParseContentEncoding(data_chunk_t encoding_mechanism); - void ParseParameter(data_chunk_t attr, data_chunk_t val); void BeginBody(); void NewDataLine(int len, const char* data, int trailing_CRLF); @@ -276,9 +275,11 @@ extern int MIME_count_leading_lws(int len, const char* data); extern int MIME_count_trailing_lws(int len, const char* data); extern int MIME_skip_comments(int len, const char* data); extern int MIME_skip_lws_comments(int len, const char* data); -extern int MIME_get_token(int len, const char* data, data_chunk_t* token); +extern int MIME_get_token(int len, const char* data, data_chunk_t* token, + bool is_boundary = false); extern int MIME_get_slash_token_pair(int len, const char* data, data_chunk_t* first, data_chunk_t* second); -extern int MIME_get_value(int len, const char* data, BroString*& buf); +extern int MIME_get_value(int len, const char* data, BroString*& buf, + bool is_boundary = false); extern int MIME_get_field_name(int len, const char* data, data_chunk_t* name); extern BroString* MIME_decode_quoted_pairs(data_chunk_t buf); diff --git a/src/analyzer/protocol/ssl/dtls-analyzer.pac b/src/analyzer/protocol/ssl/dtls-analyzer.pac index c29c5a785f..c38ebdf029 100644 --- a/src/analyzer/protocol/ssl/dtls-analyzer.pac +++ b/src/analyzer/protocol/ssl/dtls-analyzer.pac @@ -55,7 +55,7 @@ refine connection SSL_Conn += { if ( length > MAX_DTLS_HANDSHAKE_RECORD ) { - bro_analyzer()->ProtocolViolation(fmt("DTLS record length %lld larger than allowed maximum.", length)); + bro_analyzer()->ProtocolViolation(fmt("DTLS record length %" PRId64 " larger than allowed maximum.", length)); return true; } diff --git a/src/analyzer/protocol/tcp/TCP_Endpoint.cc b/src/analyzer/protocol/tcp/TCP_Endpoint.cc index 2e8d6e593b..846eb6d9d1 100644 --- a/src/analyzer/protocol/tcp/TCP_Endpoint.cc +++ b/src/analyzer/protocol/tcp/TCP_Endpoint.cc @@ -201,13 +201,18 @@ int TCP_Endpoint::DataSent(double t, uint64 seq, int len, int caplen, { int status = 0; - if ( contents_processor && caplen >= len ) - status = contents_processor->DataSent(t, seq, len, data); + if ( contents_processor ) + { + if ( caplen >= len ) + status = contents_processor->DataSent(t, seq, len, data); + else + TCP()->Weird("truncated_tcp_payload"); + } if ( caplen <= 0 ) return status; - if ( contents_file && ! contents_processor && + if ( contents_file && ! contents_processor && seq + len > contents_start_seq ) { int64 under_seq = contents_start_seq - seq; diff --git a/src/analyzer/protocol/tcp/TCP_Reassembler.cc b/src/analyzer/protocol/tcp/TCP_Reassembler.cc index 16bb9cc56d..bbcd9cb43a 100644 --- a/src/analyzer/protocol/tcp/TCP_Reassembler.cc +++ b/src/analyzer/protocol/tcp/TCP_Reassembler.cc @@ -42,6 +42,9 @@ TCP_Reassembler::TCP_Reassembler(analyzer::Analyzer* arg_dst_analyzer, seq_to_skip = 0; in_delivery = false; + if ( tcp_max_old_segments ) + SetMaxOldBlocks(tcp_max_old_segments); + if ( tcp_contents ) { // Val dst_port_val(ntohs(Conn()->RespPort()), TYPE_PORT); diff --git a/src/analyzer/protocol/zip/ZIP.cc b/src/analyzer/protocol/zip/ZIP.cc index 132515f29a..d14df95673 100644 --- a/src/analyzer/protocol/zip/ZIP.cc +++ b/src/analyzer/protocol/zip/ZIP.cc @@ -22,10 +22,9 @@ ZIP_Analyzer::ZIP_Analyzer(Connection* conn, bool orig, Method arg_method) zip->next_in = 0; zip->avail_in = 0; - // "15" here means maximum compression. "32" is a gross overload - // hack that means "check it for whether it's a gzip file". Sheesh. - zip_status = inflateInit2(zip, 15 + 32); - if ( zip_status != Z_OK ) + // "32" is a gross overload hack that means "check it + // for whether it's a gzip file". Sheesh. + if ( inflateInit2(zip, MAX_WBITS + 32) != Z_OK ) { Weird("inflate_init_failed"); delete zip; @@ -56,38 +55,63 @@ void ZIP_Analyzer::DeliverStream(int len, const u_char* data, bool orig) static unsigned int unzip_size = 4096; Bytef unzipbuf[unzip_size]; + int allow_restart = 1; + zip->next_in = (Bytef*) data; zip->avail_in = len; - do + Bytef *orig_next_in = zip->next_in; + size_t orig_avail_in = zip->avail_in; + + while ( true ) { zip->next_out = unzipbuf; zip->avail_out = unzip_size; zip_status = inflate(zip, Z_SYNC_FLUSH); - if ( zip_status != Z_STREAM_END && - zip_status != Z_OK && - zip_status != Z_BUF_ERROR ) + if ( zip_status == Z_STREAM_END || + zip_status == Z_OK ) + { + allow_restart = 0; + + int have = unzip_size - zip->avail_out; + if ( have ) + ForwardStream(have, unzipbuf, IsOrig()); + + if ( zip_status == Z_STREAM_END ) + { + inflateEnd(zip); + return; + } + + if ( zip->avail_in == 0 ) + return; + + } + + else if ( allow_restart && zip_status == Z_DATA_ERROR ) + { + // Some servers seem to not generate zlib headers, + // so this is an attempt to fix and continue anyway. + inflateEnd(zip); + + if ( inflateInit2(zip, -MAX_WBITS) != Z_OK ) + { + Weird("inflate_init_failed"); + return; + } + + zip->next_in = orig_next_in; + zip->avail_in = orig_avail_in; + allow_restart = 0; + continue; + } + + else { Weird("inflate_failed"); - inflateEnd(zip); - break; + return; } - - int have = unzip_size - zip->avail_out; - if ( have ) - ForwardStream(have, unzipbuf, IsOrig()); - - if ( zip_status == Z_STREAM_END ) - { - inflateEnd(zip); - delete zip; - zip = 0; - break; - } - - zip_status = Z_OK; } - while ( zip->avail_out == 0 ); } diff --git a/src/bro.bif b/src/bro.bif index 037b236cc0..629abe7735 100644 --- a/src/bro.bif +++ b/src/bro.bif @@ -22,6 +22,7 @@ #include "util.h" #include "file_analysis/Manager.h" #include "iosource/Manager.h" +#include "iosource/Packet.h" using namespace std; @@ -3238,11 +3239,10 @@ const char* conn_id_string(Val* c) ## .. bro:see:: dump_packet get_current_packet send_current_packet function dump_current_packet%(file_name: string%) : bool %{ - const struct pcap_pkthdr* hdr; - const u_char* pkt; + const Packet* pkt; if ( ! current_pktsrc || - ! current_pktsrc->GetCurrentPacket(&hdr, &pkt) ) + ! current_pktsrc->GetCurrentPacket(&pkt) ) return new Val(0, TYPE_BOOL); if ( ! addl_pkt_dumper ) @@ -3250,13 +3250,10 @@ function dump_current_packet%(file_name: string%) : bool if ( addl_pkt_dumper ) { - iosource::PktDumper::Packet p; - p.hdr = hdr; - p.data = pkt; - addl_pkt_dumper->Dump(&p); + addl_pkt_dumper->Dump(pkt); } - return new Val(! addl_pkt_dumper->IsError(), TYPE_BOOL); + return new Val( addl_pkt_dumper && ! addl_pkt_dumper->IsError(), TYPE_BOOL); %} ## Returns the currently processed PCAP packet. @@ -3267,26 +3264,27 @@ function dump_current_packet%(file_name: string%) : bool ## .. bro:see:: dump_current_packet dump_packet send_current_packet function get_current_packet%(%) : pcap_packet %{ - const struct pcap_pkthdr* hdr; - const u_char* data; + const Packet* p; RecordVal* pkt = new RecordVal(pcap_packet); if ( ! current_pktsrc || - ! current_pktsrc->GetCurrentPacket(&hdr, &data) ) + ! current_pktsrc->GetCurrentPacket(&p) ) { pkt->Assign(0, new Val(0, TYPE_COUNT)); pkt->Assign(1, new Val(0, TYPE_COUNT)); pkt->Assign(2, new Val(0, TYPE_COUNT)); pkt->Assign(3, new Val(0, TYPE_COUNT)); pkt->Assign(4, new StringVal("")); + pkt->Assign(5, new EnumVal(BifEnum::LINK_UNKNOWN, BifType::Enum::link_encap)); return pkt; } - pkt->Assign(0, new Val(uint32(hdr->ts.tv_sec), TYPE_COUNT)); - pkt->Assign(1, new Val(uint32(hdr->ts.tv_usec), TYPE_COUNT)); - pkt->Assign(2, new Val(hdr->caplen, TYPE_COUNT)); - pkt->Assign(3, new Val(hdr->len, TYPE_COUNT)); - pkt->Assign(4, new StringVal(hdr->caplen, (const char*) data)); + pkt->Assign(0, new Val(uint32(p->ts.tv_sec), TYPE_COUNT)); + pkt->Assign(1, new Val(uint32(p->ts.tv_usec), TYPE_COUNT)); + pkt->Assign(2, new Val(p->cap_len, TYPE_COUNT)); + pkt->Assign(3, new Val(p->len, TYPE_COUNT)); + pkt->Assign(4, new StringVal(p->cap_len, (const char*)p->data)); + pkt->Assign(5, new EnumVal(p->link_type, BifType::Enum::link_encap)); return pkt; %} @@ -3302,26 +3300,29 @@ function get_current_packet%(%) : pcap_packet ## .. bro:see:: get_current_packet dump_current_packet send_current_packet function dump_packet%(pkt: pcap_packet, file_name: string%) : bool %{ - struct pcap_pkthdr hdr; - const val_list* pkt_vl = pkt->AsRecord(); - - hdr.ts.tv_sec = (*pkt_vl)[0]->AsCount(); - hdr.ts.tv_usec = (*pkt_vl)[1]->AsCount(); - hdr.caplen = (*pkt_vl)[2]->AsCount(); - hdr.len = (*pkt_vl)[3]->AsCount(); - if ( ! addl_pkt_dumper ) addl_pkt_dumper = iosource_mgr->OpenPktDumper(file_name->CheckString(), true); if ( addl_pkt_dumper ) { - iosource::PktDumper::Packet p; - p.hdr = &hdr; - p.data = (*pkt_vl)[4]->AsString()->Bytes(); + struct timeval ts; + uint32 caplen, len, link_type; + u_char *data; + + const val_list* pkt_vl = pkt->AsRecord(); + + ts.tv_sec = (*pkt_vl)[0]->AsCount(); + ts.tv_usec = (*pkt_vl)[1]->AsCount(); + caplen = (*pkt_vl)[2]->AsCount(); + len = (*pkt_vl)[3]->AsCount(); + data = (*pkt_vl)[4]->AsString()->Bytes(); + link_type = (*pkt_vl)[5]->AsEnum(); + Packet p(link_type, &ts, caplen, len, data, true); + addl_pkt_dumper->Dump(&p); } - return new Val(addl_pkt_dumper->IsError(), TYPE_BOOL); + return new Val(addl_pkt_dumper && ! addl_pkt_dumper->IsError(), TYPE_BOOL); %} %%{ @@ -4800,20 +4801,16 @@ function send_ping%(p: event_peer, seq: count%) : bool ## dump_packet dump_current_packet get_current_packet function send_current_packet%(p: event_peer%) : bool %{ - Packet pkt(""); + const Packet* pkt; if ( ! current_pktsrc || - ! current_pktsrc->GetCurrentPacket(&pkt.hdr, &pkt.pkt) ) + ! current_pktsrc->GetCurrentPacket(&pkt) ) return new Val(0, TYPE_BOOL); RemoteSerializer::PeerID id = p->AsRecordVal()->Lookup(0)->AsCount(); - pkt.time = pkt.hdr->ts.tv_sec + double(pkt.hdr->ts.tv_usec) / 1e6; - pkt.hdr_size = current_pktsrc->HdrSize(); - pkt.link_type = current_pktsrc->LinkType(); - SerialInfo info(remote_serializer); - return new Val(remote_serializer->SendPacket(&info, id, pkt), TYPE_BOOL); + return new Val(remote_serializer->SendPacket(&info, id, *pkt), TYPE_BOOL); %} ## Returns the peer who generated the last event. diff --git a/src/event.bif b/src/event.bif index 6531bef6d8..456de20b3a 100644 --- a/src/event.bif +++ b/src/event.bif @@ -225,17 +225,31 @@ event udp_session_done%(u: connection%); ## ``ANALYZER_*`` constants right now. event scheduled_analyzer_applied%(c: connection, a: Analyzer::Tag%); -## Generated for every packet Bro sees. This is a very low-level and expensive -## event that should be avoided when at all possible. It's usually infeasible to -## handle when processing even medium volumes of traffic in real-time. That -## said, if you work from a trace and want to do some packet-level analysis, -## it may come in handy. +## Generated for every packet Bro sees that have a valid link-layer header. This +## is a very very low-level and expensive event that should be avoided when at all +## possible. It's usually infeasible to handle when processing even medium volumes +## of traffic in real-time. That said, if you work from a trace and want to do some +## packet-level analysis, it may come in handy. +## +## p: Information from the header of the packet that triggered the event. +## +## .. bro:see:: new_packet packet_contents +event raw_packet%(p: raw_pkt_hdr%); + +## Generated for all packets that make it into Bro's connection processing. In +## contrast to :bro:id:`raw_packet` this filters out some more packets that don't +## pass certain sanity checks. +## +## This is a very low-level and expensive event that should be avoided when at all +## possible. It's usually infeasible to handle when processing even medium volumes +## of traffic in real-time. That said, if you work from a trace and want to do some +## packet-level analysis, it may come in handy. ## ## c: The connection the packet is part of. ## ## p: Information from the header of the packet that triggered the event. ## -## .. bro:see:: tcp_packet packet_contents +## .. bro:see:: tcp_packet packet_contents raw_packet event new_packet%(c: connection, p: pkt_hdr%); ## Generated for every IPv6 packet that contains extension headers. @@ -282,7 +296,8 @@ event packet_contents%(c: connection, contents: string%); ## reassembling a TCP stream, Bro buffers all payload until it sees the ## responder acking it. If during that time, the sender resends a chunk of ## payload but with different content than originally, this event will be -## raised. +## raised. In addition, if :bro:id:`tcp_max_old_segments` is larger than zero, +## mismatches with that older still-buffered data will likewise trigger the event. ## ## c: The connection showing the inconsistency. ## diff --git a/src/iosource/CMakeLists.txt b/src/iosource/CMakeLists.txt index a36667aee7..b1de9bddaf 100644 --- a/src/iosource/CMakeLists.txt +++ b/src/iosource/CMakeLists.txt @@ -12,6 +12,7 @@ set(iosource_SRCS BPF_Program.cc Component.cc Manager.cc + Packet.cc PktDumper.cc PktSrc.cc ) diff --git a/src/iosource/Packet.cc b/src/iosource/Packet.cc new file mode 100644 index 0000000000..2ff910ed52 --- /dev/null +++ b/src/iosource/Packet.cc @@ -0,0 +1,477 @@ + +#include "Packet.h" +#include "Sessions.h" +#include "iosource/Manager.h" + +extern "C" { +#ifdef HAVE_NET_ETHERNET_H +#include +#elif defined(HAVE_SYS_ETHERNET_H) +#include +#elif defined(HAVE_NETINET_IF_ETHER_H) +#include +#elif defined(HAVE_NET_ETHERTYPES_H) +#include +#endif +} + +void Packet::Init(int arg_link_type, struct timeval *arg_ts, uint32 arg_caplen, + uint32 arg_len, const u_char *arg_data, int arg_copy, + std::string arg_tag) + { + if ( data && copy ) + delete [] data; + + link_type = arg_link_type; + ts = *arg_ts; + cap_len = arg_caplen; + len = arg_len; + tag = arg_tag; + + copy = arg_copy; + + if ( arg_data && arg_copy ) + { + data = new u_char[arg_caplen]; + memcpy(const_cast(data), arg_data, arg_caplen); + } + else + data = arg_data; + + time = ts.tv_sec + double(ts.tv_usec) / 1e6; + hdr_size = GetLinkHeaderSize(arg_link_type); + l3_proto = L3_UNKNOWN; + eth_type = 0; + vlan = 0; + + l2_valid = false; + + if ( data ) + ProcessLayer2(); + } + +void Packet::Weird(const char* name) + { + sessions->Weird(name, this); + l2_valid = false; + } + +int Packet::GetLinkHeaderSize(int link_type) + { + switch ( link_type ) { + case DLT_NULL: + return 4; + + case DLT_EN10MB: + return 14; + + case DLT_FDDI: + return 13 + 8; // fddi_header + LLC + +#ifdef DLT_LINUX_SLL + case DLT_LINUX_SLL: + return 16; +#endif + + case DLT_PPP_SERIAL: // PPP_SERIAL + return 4; + + case DLT_RAW: + return 0; + } + + return -1; + } + +void Packet::ProcessLayer2() + { + l2_valid = true; + + // Unfortunately some packets on the link might have MPLS labels + // while others don't. That means we need to ask the link-layer if + // labels are in place. + bool have_mpls = false; + + const u_char* pdata = data; + + switch ( link_type ) { + case DLT_NULL: + { + int protocol = (pdata[3] << 24) + (pdata[2] << 16) + (pdata[1] << 8) + pdata[0]; + pdata += GetLinkHeaderSize(link_type); + + // From the Wireshark Wiki: "AF_INET6, unfortunately, has + // different values in {NetBSD,OpenBSD,BSD/OS}, + // {FreeBSD,DragonFlyBSD}, and {Darwin/Mac OS X}, so an IPv6 + // packet might have a link-layer header with 24, 28, or 30 + // as the AF_ value." As we may be reading traces captured on + // platforms other than what we're running on, we accept them + // all here. + + if ( protocol == AF_INET ) + l3_proto = L3_IPV4; + else if ( protocol == 24 || protocol == 28 || protocol == 30 ) + l3_proto = L3_IPV6; + else + { + Weird("non_ip_packet_in_null_transport"); + return; + } + + break; + } + + case DLT_EN10MB: + { + // Get protocol being carried from the ethernet frame. + int protocol = (pdata[12] << 8) + pdata[13]; + pdata += GetLinkHeaderSize(link_type); + eth_type = protocol; + + switch ( protocol ) + { + // MPLS carried over the ethernet frame. + case 0x8847: + have_mpls = true; + break; + + // VLAN carried over the ethernet frame. + // 802.1q / 802.1ad + case 0x8100: + case 0x9100: + vlan = ((pdata[0] << 8) + pdata[1]) & 0xfff; + protocol = ((pdata[2] << 8) + pdata[3]); + pdata += 4; // Skip the vlan header + + // Check for MPLS in VLAN. + if ( protocol == 0x8847 ) + { + have_mpls = true; + break; + } + + // Check for double-tagged (802.1ad) + if ( protocol == 0x8100 || protocol == 0x9100 ) + { + protocol = ((pdata[2] << 8) + pdata[3]); + pdata += 4; // Skip the vlan header + } + + eth_type = protocol; + break; + + // PPPoE carried over the ethernet frame. + case 0x8864: + protocol = (pdata[6] << 8) + pdata[7]; + pdata += 8; // Skip the PPPoE session and PPP header + + if ( protocol == 0x0021 ) + l3_proto = L3_IPV4; + else if ( protocol == 0x0057 ) + l3_proto = L3_IPV6; + else + { + // Neither IPv4 nor IPv6. + Weird("non_ip_packet_in_pppoe_encapsulation"); + return; + } + + break; + } + + // Normal path to determine Layer 3 protocol. + if ( ! have_mpls && l3_proto == L3_UNKNOWN ) + { + if ( protocol == 0x800 ) + l3_proto = L3_IPV4; + else if ( protocol == 0x86dd ) + l3_proto = L3_IPV6; + else if ( protocol == 0x0806 || protocol == 0x8035 ) + l3_proto = L3_ARP; + else + { + // Neither IPv4 nor IPv6. + Weird("non_ip_packet_in_ethernet"); + return; + } + } + + break; + } + + case DLT_PPP_SERIAL: + { + // Get PPP protocol. + int protocol = (pdata[2] << 8) + pdata[3]; + pdata += GetLinkHeaderSize(link_type); + + if ( protocol == 0x0281 ) + { + // MPLS Unicast. Remove the pdata link layer and + // denote a header size of zero before the IP header. + have_mpls = true; + } + else if ( protocol == 0x0021 ) + l3_proto = L3_IPV4; + else if ( protocol == 0x0057 ) + l3_proto = L3_IPV6; + else + { + // Neither IPv4 nor IPv6. + Weird("non_ip_packet_in_ppp_encapsulation"); + return; + } + break; + } + + default: + { + // Assume we're pointing at IP. Just figure out which version. + pdata += GetLinkHeaderSize(link_type); + const struct ip* ip = (const struct ip *)pdata; + + if ( ip->ip_v == 4 ) + l3_proto = L3_IPV4; + else if ( ip->ip_v == 6 ) + l3_proto = L3_IPV6; + else + { + // Neither IPv4 nor IPv6. + Weird("non_ip_packet"); + return; + } + + break; + } + } + + if ( have_mpls ) + { + // Skip the MPLS label stack. + bool end_of_stack = false; + + while ( ! end_of_stack ) + { + end_of_stack = *(pdata + 2) & 0x01; + pdata += 4; + + if ( pdata >= pdata + cap_len ) + { + Weird("no_mpls_payload"); + return; + } + } + + // We assume that what remains is IP + if ( pdata + sizeof(struct ip) >= data + cap_len ) + { + Weird("no_ip_in_mpls_payload"); + return; + } + + const struct ip* ip = (const struct ip *)pdata; + + if ( ip->ip_v == 4 ) + l3_proto = L3_IPV4; + else if ( ip->ip_v == 6 ) + l3_proto = L3_IPV6; + else + { + // Neither IPv4 nor IPv6. + Weird("no_ip_in_mpls_payload"); + return; + } + } + + else if ( encap_hdr_size ) + { + // Blanket encapsulation. We assume that what remains is IP. + pdata += encap_hdr_size; + if ( pdata + sizeof(struct ip) >= data + cap_len ) + { + Weird("no_ip_left_after_encap"); + return; + } + + const struct ip* ip = (const struct ip *)pdata; + + if ( ip->ip_v == 4 ) + l3_proto = L3_IPV4; + else if ( ip->ip_v == 6 ) + l3_proto = L3_IPV6; + else + { + // Neither IPv4 nor IPv6. + Weird("no_ip_in_encap"); + return; + } + + } + + // We've now determined (a) L3_IPV4 vs (b) L3_IPV6 vs + // (c) L3_ARP vs (d) L3_UNKNOWN. + l3_proto = l3_proto; + + // Calculate how much header we've used up. + hdr_size = (pdata - data); +} + +RecordVal* Packet::BuildPktHdrVal() const + { + static RecordType* l2_hdr_type = 0; + static RecordType* raw_pkt_hdr_type = 0; + + if ( ! raw_pkt_hdr_type ) + { + raw_pkt_hdr_type = internal_type("raw_pkt_hdr")->AsRecordType(); + l2_hdr_type = internal_type("l2_hdr")->AsRecordType(); + } + + RecordVal* pkt_hdr = new RecordVal(raw_pkt_hdr_type); + RecordVal* l2_hdr = new RecordVal(l2_hdr_type); + + int is_ethernet = (link_type == DLT_EN10MB) ? 1 : 0; + + int l3 = BifEnum::L3_UNKNOWN; + + if ( l3_proto == L3_IPV4 ) + l3 = BifEnum::L3_IPV4; + + else if ( l3_proto == L3_IPV6 ) + l3 = BifEnum::L3_IPV6; + + else if ( l3_proto == L3_ARP ) + l3 = BifEnum::L3_ARP; + + // l2_hdr layout: + // encap: link_encap; ##< L2 link encapsulation + // len: count; ##< Total frame length on wire + // cap_len: count; ##< Captured length + // src: string &optional; ##< L2 source (if ethernet) + // dst: string &optional; ##< L2 destination (if ethernet) + // vlan: count &optional; ##< VLAN tag if any (and ethernet) + // ethertype: count &optional; ##< If ethernet + // proto: layer3_proto; ##< L3 proto + + if ( is_ethernet ) + { + // Ethernet header layout is: + // dst[6bytes] src[6bytes] ethertype[2bytes]... + l2_hdr->Assign(0, new EnumVal(BifEnum::LINK_ETHERNET, BifType::Enum::link_encap)); + l2_hdr->Assign(3, FmtEUI48(data + 6)); // src + l2_hdr->Assign(4, FmtEUI48(data)); // dst + + if ( vlan ) + l2_hdr->Assign(5, new Val(vlan, TYPE_COUNT)); + + l2_hdr->Assign(6, new Val(eth_type, TYPE_COUNT)); + + if ( eth_type == ETHERTYPE_ARP || eth_type == ETHERTYPE_REVARP ) + // We also identify ARP for L3 over ethernet + l3 = BifEnum::L3_ARP; + } + else + l2_hdr->Assign(0, new EnumVal(BifEnum::LINK_UNKNOWN, BifType::Enum::link_encap)); + + l2_hdr->Assign(1, new Val(len, TYPE_COUNT)); + l2_hdr->Assign(2, new Val(cap_len, TYPE_COUNT)); + + l2_hdr->Assign(7, new EnumVal(l3, BifType::Enum::layer3_proto)); + + pkt_hdr->Assign(0, l2_hdr); + + if ( l3_proto == L3_IPV4 ) + { + IP_Hdr ip_hdr((const struct ip*)(data + hdr_size), false); + return ip_hdr.BuildPktHdrVal(pkt_hdr, 1); + } + + else if ( l3_proto == L3_IPV6 ) + { + IP_Hdr ip6_hdr((const struct ip6_hdr*)(data + hdr_size), false, cap_len); + return ip6_hdr.BuildPktHdrVal(pkt_hdr, 1); + } + + else + return pkt_hdr; + } + +Val *Packet::FmtEUI48(const u_char *mac) const + { + char buf[20]; + snprintf(buf, sizeof buf, "%02x:%02x:%02x:%02x:%02x:%02x", + mac[0], mac[1], mac[2], mac[3], mac[4], mac[5]); + return new StringVal(buf); + } + +void Packet::Describe(ODesc* d) const + { + const IP_Hdr ip = IP(); + d->Add(ip.SrcAddr()); + d->Add("->"); + d->Add(ip.DstAddr()); + } + +bool Packet::Serialize(SerialInfo* info) const + { + return SERIALIZE(uint32(ts.tv_sec)) && + SERIALIZE(uint32(ts.tv_usec)) && + SERIALIZE(uint32(len)) && + SERIALIZE(link_type) && + info->s->Write(tag.c_str(), tag.length(), "tag") && + info->s->Write((const char*)data, cap_len, "data"); + } + +#ifdef DEBUG +static iosource::PktDumper* dump = 0; +#endif + +Packet* Packet::Unserialize(UnserialInfo* info) + { + struct timeval ts; + uint32 len, link_type; + + if ( ! (UNSERIALIZE((uint32 *)&ts.tv_sec) && + UNSERIALIZE((uint32 *)&ts.tv_usec) && + UNSERIALIZE(&len) && + UNSERIALIZE(&link_type)) ) + return 0; + + char* tag; + if ( ! info->s->Read((char**) &tag, 0, "tag") ) + return 0; + + const u_char* pkt; + int caplen; + if ( ! info->s->Read((char**) &pkt, &caplen, "data") ) + { + delete [] tag; + return 0; + } + + Packet *p = new Packet(link_type, &ts, caplen, len, pkt, true, + std::string(tag)); + delete [] tag; + + // For the global timer manager, we take the global network_time as the + // packet's timestamp for feeding it into our packet loop. + if ( p->tag == "" ) + p->time = timer_mgr->Time(); + else + p->time = p->ts.tv_sec + double(p->ts.tv_usec) / 1e6; + +#ifdef DEBUG + if ( debug_logger.IsEnabled(DBG_TM) ) + { + if ( ! dump ) + dump = iosource_mgr->OpenPktDumper("tm.pcap", true); + + if ( dump ) + { + dump->Dump(p); + } + } +#endif + + return p; + } diff --git a/src/iosource/Packet.h b/src/iosource/Packet.h new file mode 100644 index 0000000000..eaa1b90210 --- /dev/null +++ b/src/iosource/Packet.h @@ -0,0 +1,202 @@ +#ifndef packet_h +#define packet_h + +#include "Desc.h" +#include "IP.h" +#include "NetVar.h" + +/** + * The Layer 3 type of a packet, as determined by the parsing code in Packet. + */ +enum Layer3Proto { + L3_UNKNOWN = -1, /// Layer 3 type could not be determined. + L3_IPV4 = 1, /// Layer 3 is IPv4. + L3_IPV6 = 2, /// Layer 3 is IPv6. + L3_ARP = 3, /// Layer 3 is ARP. +}; + +/** + * A link-layer packet. + * + * Note that for serialization we don't use much of the support provided by + * the serialization framework. Serialize/Unserialize do all the work by + * themselves. In particular, Packets aren't derived from SerialObj. They are + * completely seperate and self-contained entities, and we don't need any of + * the sophisticated features like object caching. + */ +class Packet { +public: + /** + * Construct and initialize from packet data. + * + * @param link_type The link type in the form of a \c DLT_* constant. + * + * @param ts The timestamp associated with the packet. + * + * @param caplen The number of bytes valid in *data*. + * + * @param len The wire length of the packet, which must be more or + * equal *caplen* (but can't be less). + * + * @param data A pointer to the raw packet data, starting with the + * layer 2 header. The pointer must remain valid for the lifetime of + * the Packet instance, unless *copy* is true. + * + * @param copy If true, the constructor will make an internal copy of + * *data*, so that the caller can release its version. + * + * @param tag A textual tag to associate with the packet for + * differentiating the input streams. + */ + Packet(int link_type, struct timeval *ts, uint32 caplen, + uint32 len, const u_char *data, int copy = false, + std::string tag = std::string("")) : data(0) + { + Init(link_type, ts, caplen, len, data, copy, tag); + } + + /** + * Default constructor. For internal use only. + */ + Packet() : data(0) + { + struct timeval ts = {0, 0}; + Init(0, &ts, 0, 0, 0); + } + + /** + * Destructor. + */ + ~Packet() + { + if ( copy ) + delete [] data; + } + + /** + * (Re-)initialize from packet data. + * + * @param link_type The link type in the form of a \c DLT_* constant. + * + * @param ts The timestamp associated with the packet. + * + * @param caplen The number of bytes valid in *data*. + * + * @param len The wire length of the packet, which must be more or + * equal *caplen* (but can't be less). + * + * @param data A pointer to the raw packet data, starting with the + * layer 2 header. The pointer must remain valid for the lifetime of + * the Packet instance, unless *copy* is true. + * + * @param copy If true, the constructor will make an internal copy of + * *data*, so that the caller can release its version. + * + * @param tag A textual tag to associate with the packet for + * differentiating the input streams. + */ + void Init(int link_type, struct timeval *ts, uint32 caplen, + uint32 len, const u_char *data, int copy = false, + std::string tag = std::string("")); + + /** + * Returns true if parsing the layer 2 fields failed, including when + * no data was passed into the constructor in the first place. + */ + bool Layer2Valid() + { + return l2_valid; + } + + /** + * Interprets the Layer 3 of the packet as IP and returns a + * correspondign object. + */ + const IP_Hdr IP() const + { return IP_Hdr((struct ip *) (data + hdr_size), false); } + + /** + * Returns a \c raw_pkt_hdr RecordVal, which includes layer 2 and + * also everything in IP_Hdr (i.e., IP4/6 + TCP/UDP/ICMP). + */ + RecordVal* BuildPktHdrVal() const; + + /** + * Static method returning the link-layer header size for a given + * link type. + * + * @param link_type The link tyoe. + * + * @return The header size in bytes, or -1 if not known. + */ + static int GetLinkHeaderSize(int link_type); + + /** + * Describes the packet, with standard signature. + */ + void Describe(ODesc* d) const; + + /** + * Serializes the packet, with standard signature. + */ + bool Serialize(SerialInfo* info) const; + + /** + * Unserializes the packet, with standard signature. + */ + static Packet* Unserialize(UnserialInfo* info); + + // These are passed in through the constructor. + std::string tag; /// Used in serialization + double time; /// Timestamp reconstituted as float + struct timeval ts; /// Capture timestamp + const u_char* data; /// Packet data. + uint32 len; /// Actual length on wire + uint32 cap_len; /// Captured packet length + uint32 link_type; /// pcap link_type (DLT_EN10MB, DLT_RAW, etc) + + // These are computed from Layer 2 data. These fields are only valid if + // Layer2Valid() returns true. + + /** + * Layer 2 header size. Valid iff Layer2Valid() returns true. + */ + uint32 hdr_size; + + /** + * Layer 3 protocol identified (if any). Valid iff Layer2Valid() + * returns true. + */ + Layer3Proto l3_proto; /// + + /** + * If layer 2 is Ethernet, innermost ethertype field. Valid iff + * Layer2Valid() returns true. + */ + uint32 eth_type; /// + + /** + * (Outermost) VLAN tag if any, else 0. Valid iff Layer2Valid() + * returns true. + */ + uint32 vlan; /// + +private: + // Calculate layer 2 attributes. Sets + void ProcessLayer2(); + + // Wrapper to generate a packet-level weird. + void Weird(const char* name); + + // Renders an MAC address into its ASCII representation. + Val *FmtEUI48(const u_char *mac) const; + + // True if we need to delete associated packet memory upon + // destruction. + bool copy; + + // True if L2 processing succeeded. + bool l2_valid; +}; + +#endif // packet_h diff --git a/src/iosource/PktDumper.h b/src/iosource/PktDumper.h index 56555c247a..dcfda2030b 100644 --- a/src/iosource/PktDumper.h +++ b/src/iosource/PktDumper.h @@ -3,6 +3,7 @@ #ifndef IOSOURCE_PKTSRC_PKTDUMPER_H #define IOSOURCE_PKTSRC_PKTDUMPER_H +#include "Packet.h" #include "IOSource.h" namespace iosource { @@ -12,21 +13,6 @@ namespace iosource { */ class PktDumper { public: - /** - * Structure describing a packet. - */ - struct Packet { - /** - * The pcap header associated with the packet. - */ - const struct pcap_pkthdr* hdr; - - /** - * The full content of the packet. - */ - const unsigned char* data; - }; - /** * Constructor. */ diff --git a/src/iosource/PktSrc.cc b/src/iosource/PktSrc.cc index 84929b08d0..8012f79f1b 100644 --- a/src/iosource/PktSrc.cc +++ b/src/iosource/PktSrc.cc @@ -17,7 +17,6 @@ PktSrc::Properties::Properties() { selectable_fd = -1; link_type = -1; - hdr_size = -1; netmask = NETMASK_UNKNOWN; is_live = false; } @@ -67,11 +66,6 @@ bool PktSrc::IsError() const return ErrorMsg(); } -int PktSrc::HdrSize() const - { - return IsOpen() ? props.hdr_size : -1; - } - int PktSrc::SnapLen() const { return snaplen; // That's a global. Change? @@ -98,7 +92,7 @@ double PktSrc::CurrentPacketWallClock() void PktSrc::Opened(const Properties& arg_props) { - if ( arg_props.hdr_size < 0 ) + if ( Packet::GetLinkHeaderSize(arg_props.link_type) < 0 ) { char buf[512]; safe_snprintf(buf, sizeof(buf), @@ -147,7 +141,7 @@ void PktSrc::Info(const std::string& msg) void PktSrc::Weird(const std::string& msg, const Packet* p) { - sessions->Weird(msg.c_str(), p->hdr, p->data, 0); + sessions->Weird(msg.c_str(), p, 0); } void PktSrc::InternalError(const std::string& msg) @@ -160,33 +154,6 @@ void PktSrc::ContinueAfterSuspend() current_wallclock = current_time(true); } -int PktSrc::GetLinkHeaderSize(int link_type) - { - switch ( link_type ) { - case DLT_NULL: - return 4; - - case DLT_EN10MB: - return 14; - - case DLT_FDDI: - return 13 + 8; // fddi_header + LLC - -#ifdef DLT_LINUX_SLL - case DLT_LINUX_SLL: - return 16; -#endif - - case DLT_PPP_SERIAL: // PPP_SERIAL - return 4; - - case DLT_RAW: - return 0; - } - - return -1; - } - double PktSrc::CheckPseudoTime() { if ( ! IsOpen() ) @@ -197,20 +164,20 @@ double PktSrc::CheckPseudoTime() if ( remote_trace_sync_interval ) { - if ( next_sync_point == 0 || current_packet.ts >= next_sync_point ) + if ( next_sync_point == 0 || current_packet.time >= next_sync_point ) { int n = remote_serializer->SendSyncPoint(); next_sync_point = first_timestamp + n * remote_trace_sync_interval; remote_serializer->Log(RemoteSerializer::LogInfo, fmt("stopping at packet %.6f, next sync-point at %.6f", - current_packet.ts, next_sync_point)); + current_packet.time, next_sync_point)); return 0; } } - double pseudo_time = current_packet.ts - first_timestamp; + double pseudo_time = current_packet.time - first_timestamp; double ct = (current_time(true) - first_wallclock) * pseudo_realtime; return pseudo_time <= ct ? bro_start_time + pseudo_time : 0; @@ -273,7 +240,7 @@ double PktSrc::NextTimestamp(double* local_network_time) return -1.0; } - return current_packet.ts; + return current_packet.time; } void PktSrc::Process() @@ -284,145 +251,20 @@ void PktSrc::Process() if ( ! ExtractNextPacketInternal() ) return; - int pkt_hdr_size = props.hdr_size; - - // Unfortunately some packets on the link might have MPLS labels - // while others don't. That means we need to ask the link-layer if - // labels are in place. - bool have_mpls = false; - - int protocol = 0; - const u_char* data = current_packet.data; - - switch ( props.link_type ) { - case DLT_NULL: + if ( current_packet.Layer2Valid() ) { - protocol = (data[3] << 24) + (data[2] << 16) + (data[1] << 8) + data[0]; - - // From the Wireshark Wiki: "AF_INET6, unfortunately, has - // different values in {NetBSD,OpenBSD,BSD/OS}, - // {FreeBSD,DragonFlyBSD}, and {Darwin/Mac OS X}, so an IPv6 - // packet might have a link-layer header with 24, 28, or 30 - // as the AF_ value." As we may be reading traces captured on - // platforms other than what we're running on, we accept them - // all here. - if ( protocol != AF_INET - && protocol != AF_INET6 - && protocol != 24 - && protocol != 28 - && protocol != 30 ) + if ( pseudo_realtime ) { - Weird("non_ip_packet_in_null_transport", ¤t_packet); - goto done; + current_pseudo = CheckPseudoTime(); + net_packet_dispatch(current_pseudo, ¤t_packet, this); + if ( ! first_wallclock ) + first_wallclock = current_time(true); } - break; + else + net_packet_dispatch(current_packet.time, ¤t_packet, this); } - case DLT_EN10MB: - { - // Get protocol being carried from the ethernet frame. - protocol = (data[12] << 8) + data[13]; - - switch ( protocol ) - { - // MPLS carried over the ethernet frame. - case 0x8847: - // Remove the data link layer and denote a - // header size of zero before the IP header. - have_mpls = true; - data += GetLinkHeaderSize(props.link_type); - pkt_hdr_size = 0; - break; - - // VLAN carried over the ethernet frame. - case 0x8100: - data += GetLinkHeaderSize(props.link_type); - - // Check for MPLS in VLAN. - if ( ((data[2] << 8) + data[3]) == 0x8847 ) - have_mpls = true; - - data += 4; // Skip the vlan header - pkt_hdr_size = 0; - - // Check for 802.1ah (Q-in-Q) containing IP. - // Only do a second layer of vlan tag - // stripping because there is no - // specification that allows for deeper - // nesting. - if ( ((data[2] << 8) + data[3]) == 0x0800 ) - data += 4; - - break; - - // PPPoE carried over the ethernet frame. - case 0x8864: - data += GetLinkHeaderSize(props.link_type); - protocol = (data[6] << 8) + data[7]; - data += 8; // Skip the PPPoE session and PPP header - pkt_hdr_size = 0; - - if ( protocol != 0x0021 && protocol != 0x0057 ) - { - // Neither IPv4 nor IPv6. - Weird("non_ip_packet_in_pppoe_encapsulation", ¤t_packet); - goto done; - } - break; - } - - break; - } - - case DLT_PPP_SERIAL: - { - // Get PPP protocol. - protocol = (data[2] << 8) + data[3]; - - if ( protocol == 0x0281 ) - { - // MPLS Unicast. Remove the data link layer and - // denote a header size of zero before the IP header. - have_mpls = true; - data += GetLinkHeaderSize(props.link_type); - pkt_hdr_size = 0; - } - - else if ( protocol != 0x0021 && protocol != 0x0057 ) - { - // Neither IPv4 nor IPv6. - Weird("non_ip_packet_in_ppp_encapsulation", ¤t_packet); - goto done; - } - break; - } - } - - if ( have_mpls ) - { - // Skip the MPLS label stack. - bool end_of_stack = false; - - while ( ! end_of_stack ) - { - end_of_stack = *(data + 2) & 0x01; - data += 4; - } - } - - if ( pseudo_realtime ) - { - current_pseudo = CheckPseudoTime(); - net_packet_dispatch(current_pseudo, current_packet.hdr, data, pkt_hdr_size, this); - if ( ! first_wallclock ) - first_wallclock = current_time(true); - } - - else - net_packet_dispatch(current_packet.ts, current_packet.hdr, data, pkt_hdr_size, this); - -done: have_packet = 0; DoneWithPacket(); } @@ -453,7 +295,7 @@ bool PktSrc::ExtractNextPacketInternal() if ( ExtractNextPacket(¤t_packet) ) { if ( ! first_timestamp ) - first_timestamp = current_packet.ts; + first_timestamp = current_packet.time; SetIdle(false); have_packet = true; @@ -536,12 +378,11 @@ bool PktSrc::ApplyBPFFilter(int index, const struct pcap_pkthdr *hdr, const u_ch return pcap_offline_filter(code->GetProgram(), hdr, pkt); } -bool PktSrc::GetCurrentPacket(const pcap_pkthdr** hdr, const u_char** pkt) +bool PktSrc::GetCurrentPacket(const Packet** pkt) { if ( ! have_packet ) return false; - *hdr = current_packet.hdr; - *pkt = current_packet.data; + *pkt = ¤t_packet; return true; } diff --git a/src/iosource/PktSrc.h b/src/iosource/PktSrc.h index 2400219fd0..bf4c811dca 100644 --- a/src/iosource/PktSrc.h +++ b/src/iosource/PktSrc.h @@ -6,6 +6,7 @@ #include "IOSource.h" #include "BPF_Program.h" #include "Dict.h" +#include "Packet.h" declare(PDict,BPF_Program); @@ -165,14 +166,12 @@ public: /** * Returns the packet currently being processed, if available. * - * @param hdr A pointer to pass the header of the current packet back. - * * @param pkt A pointer to pass the content of the current packet * back. * * @return True if the current packet is available, or false if not. */ - bool GetCurrentPacket(const pcap_pkthdr** hdr, const u_char** pkt); + bool GetCurrentPacket(const Packet** hdr); // PacketSource interace for derived classes to override. @@ -216,15 +215,6 @@ public: */ virtual void Statistics(Stats* stats) = 0; - /** - * Helper method to return the header size for a given link tyoe. - * - * @param link_type The link tyoe. - * - * @return The header size in bytes. - */ - static int GetLinkHeaderSize(int link_type); - protected: friend class Manager; @@ -252,13 +242,6 @@ protected: */ int link_type; - /** - * The size of the link-layer header for packets from this - * source. \a GetLinkHeaderSize() may be used to derive this - * value. - */ - int hdr_size; - /** * Returns the netmask associated with the source, or \c * NETMASK_UNKNOWN if unknown. @@ -274,26 +257,6 @@ protected: Properties(); }; - /** - * Structure describing a packet. - */ - struct Packet { - /** - * Time associated with the packet. - */ - double ts; - - /** - * The pcap header associated with the packet. - */ - const struct ::pcap_pkthdr* hdr; - - /** - * The full content of the packet. - */ - const u_char* data; - }; - /** * Called from the implementations of \a Open() to signal that the * source has been successully opened. diff --git a/src/iosource/pcap/Dumper.cc b/src/iosource/pcap/Dumper.cc index 5d0b5e599b..5bea6231f7 100644 --- a/src/iosource/pcap/Dumper.cc +++ b/src/iosource/pcap/Dumper.cc @@ -79,7 +79,7 @@ void PcapDumper::Open() } props.open_time = network_time; - props.hdr_size = PktSrc::GetLinkHeaderSize(pcap_datalink(pd)); + props.hdr_size = Packet::GetLinkHeaderSize(pcap_datalink(pd)); Opened(props); } @@ -101,8 +101,12 @@ bool PcapDumper::Dump(const Packet* pkt) if ( ! dumper ) return false; - pcap_dump((u_char*) dumper, pkt->hdr, pkt->data); + // Reconstitute the pcap_pkthdr. + const struct pcap_pkthdr phdr = { + .ts = pkt->ts, .caplen = pkt->cap_len, .len = pkt->len + }; + pcap_dump((u_char*) dumper, &phdr, pkt->data); return true; } diff --git a/src/iosource/pcap/Source.cc b/src/iosource/pcap/Source.cc index 7645903c2a..bebe02c018 100644 --- a/src/iosource/pcap/Source.cc +++ b/src/iosource/pcap/Source.cc @@ -5,6 +5,7 @@ #include "config.h" #include "Source.h" +#include "iosource/Packet.h" #ifdef HAVE_PCAP_INT_H #include @@ -167,9 +168,8 @@ bool PcapSource::ExtractNextPacket(Packet* pkt) return false; } - pkt->ts = current_hdr.ts.tv_sec + double(current_hdr.ts.tv_usec) / 1e6; - pkt->hdr = ¤t_hdr; - pkt->data = last_data = data; + last_data = data; + pkt->Init(props.link_type, ¤t_hdr.ts, current_hdr.caplen, current_hdr.len, data); if ( current_hdr.len == 0 || current_hdr.caplen == 0 ) { @@ -275,7 +275,6 @@ void PcapSource::SetHdrSize() char errbuf[PCAP_ERRBUF_SIZE]; props.link_type = pcap_datalink(pd); - props.hdr_size = GetLinkHeaderSize(props.link_type); } iosource::PktSrc* PcapSource::Instantiate(const std::string& path, bool is_live) diff --git a/src/types.bif b/src/types.bif index 180112dd8c..f2a895f57f 100644 --- a/src/types.bif +++ b/src/types.bif @@ -184,6 +184,18 @@ type EncapsulatingConn: record; module GLOBAL; +enum link_encap %{ + LINK_ETHERNET, + LINK_UNKNOWN, +%} + +enum layer3_proto %{ + L3_IPV4, + L3_IPV6, + L3_ARP, + L3_UNKNOWN, +%} + type gtpv1_hdr: record; type gtp_create_pdp_ctx_request_elements: record; type gtp_create_pdp_ctx_response_elements: record; diff --git a/testing/btest/Baseline/core.raw_packet/output b/testing/btest/Baseline/core.raw_packet/output new file mode 100644 index 0000000000..eeca545bf6 --- /dev/null +++ b/testing/btest/Baseline/core.raw_packet/output @@ -0,0 +1,44 @@ +[l2=[encap=LINK_ETHERNET, len=215, cap_len=215, src=e8:de:27:ff:c0:78, dst=ff:ff:ff:ff:ff:ff, vlan=, eth_type=2048, proto=L3_IPV4], ip=[hl=20, tos=0, len=201, id=0, ttl=64, p=17, src=192.168.1.1, dst=255.255.255.255], ip6=, tcp=, udp=[sport=40190/udp, dport=7437/udp, ulen=181], icmp=] +[l2=[encap=LINK_ETHERNET, len=68, cap_len=68, src=60:f8:1d:c9:8c:fa, dst=e8:de:27:ff:c0:78, vlan=, eth_type=2048, proto=L3_IPV4], ip=[hl=20, tos=0, len=54, id=52261, ttl=64, p=6, src=192.168.1.103, dst=64.4.23.176], ip6=, tcp=[sport=65493/tcp, dport=40031/tcp, seq=2642773190, ack=2891276360, hl=32, dl=2, flags=24, win=4096], udp=, icmp=] +[l2=[encap=LINK_ETHERNET, len=78, cap_len=78, src=60:f8:1d:c9:8c:fa, dst=e8:de:27:ff:c0:78, vlan=, eth_type=2048, proto=L3_IPV4], ip=[hl=20, tos=0, len=64, id=32575, ttl=64, p=17, src=192.168.1.103, dst=192.168.1.1], ip6=, tcp=, udp=[sport=65170/udp, dport=53/udp, ulen=44], icmp=] +[l2=[encap=LINK_ETHERNET, len=78, cap_len=78, src=60:f8:1d:c9:8c:fa, dst=e8:de:27:ff:c0:78, vlan=, eth_type=2048, proto=L3_IPV4], ip=[hl=20, tos=0, len=64, id=55466, ttl=64, p=17, src=192.168.1.103, dst=192.168.1.1], ip6=, tcp=, udp=[sport=53129/udp, dport=53/udp, ulen=44], icmp=] +[l2=[encap=LINK_ETHERNET, len=92, cap_len=92, src=60:f8:1d:c9:8c:fa, dst=e8:de:27:ff:c0:78, vlan=, eth_type=2048, proto=L3_IPV4], ip=[hl=20, tos=0, len=78, id=32240, ttl=64, p=17, src=192.168.1.103, dst=192.168.1.1], ip6=, tcp=, udp=[sport=53129/udp, dport=53/udp, ulen=58], icmp=] +[l2=[encap=LINK_ETHERNET, len=85, cap_len=85, src=60:f8:1d:c9:8c:fa, dst=e8:de:27:ff:c0:78, vlan=, eth_type=2048, proto=L3_IPV4], ip=[hl=20, tos=0, len=71, id=53895, ttl=64, p=17, src=192.168.1.103, dst=192.168.1.1], ip6=, tcp=, udp=[sport=57932/udp, dport=53/udp, ulen=51], icmp=] +[l2=[encap=LINK_ETHERNET, len=42, cap_len=42, src=00:50:56:3e:93:6b, dst=ff:ff:ff:ff:ff:ff, vlan=, eth_type=2054, proto=L3_ARP], ip=, ip6=, tcp=, udp=, icmp=] +[l2=[encap=LINK_ETHERNET, len=42, cap_len=42, src=00:50:56:3e:93:6b, dst=ff:ff:ff:ff:ff:ff, vlan=, eth_type=2054, proto=L3_ARP], ip=, ip6=, tcp=, udp=, icmp=] +[l2=[encap=LINK_ETHERNET, len=307, cap_len=307, src=e8:de:27:ff:c0:78, dst=01:00:5e:7f:ff:fa, vlan=, eth_type=2048, proto=L3_IPV4], ip=[hl=20, tos=0, len=293, id=0, ttl=4, p=17, src=192.168.1.1, dst=239.255.255.250], ip6=, tcp=, udp=[sport=45335/udp, dport=1900/udp, ulen=273], icmp=] +[l2=[encap=LINK_ETHERNET, len=316, cap_len=316, src=e8:de:27:ff:c0:78, dst=01:00:5e:7f:ff:fa, vlan=, eth_type=2048, proto=L3_IPV4], ip=[hl=20, tos=0, len=302, id=0, ttl=4, p=17, src=192.168.1.1, dst=239.255.255.250], ip6=, tcp=, udp=[sport=45335/udp, dport=1900/udp, ulen=282], icmp=] +[l2=[encap=LINK_ETHERNET, len=379, cap_len=379, src=e8:de:27:ff:c0:78, dst=01:00:5e:7f:ff:fa, vlan=, eth_type=2048, proto=L3_IPV4], ip=[hl=20, tos=0, len=365, id=0, ttl=4, p=17, src=192.168.1.1, dst=239.255.255.250], ip6=, tcp=, udp=[sport=45335/udp, dport=1900/udp, ulen=345], icmp=] +[l2=[encap=LINK_ETHERNET, len=371, cap_len=371, src=e8:de:27:ff:c0:78, dst=01:00:5e:7f:ff:fa, vlan=, eth_type=2048, proto=L3_IPV4], ip=[hl=20, tos=0, len=357, id=0, ttl=4, p=17, src=192.168.1.1, dst=239.255.255.250], ip6=, tcp=, udp=[sport=45335/udp, dport=1900/udp, ulen=337], icmp=] +[l2=[encap=LINK_ETHERNET, len=355, cap_len=355, src=e8:de:27:ff:c0:78, dst=01:00:5e:7f:ff:fa, vlan=, eth_type=2048, proto=L3_IPV4], ip=[hl=20, tos=0, len=341, id=0, ttl=4, p=17, src=192.168.1.1, dst=239.255.255.250], ip6=, tcp=, udp=[sport=45335/udp, dport=1900/udp, ulen=321], icmp=] +[l2=[encap=LINK_ETHERNET, len=42, cap_len=42, src=00:50:56:3e:93:6b, dst=ff:ff:ff:ff:ff:ff, vlan=, eth_type=2054, proto=L3_ARP], ip=, ip6=, tcp=, udp=, icmp=] +[l2=[encap=LINK_ETHERNET, len=387, cap_len=387, src=e8:de:27:ff:c0:78, dst=01:00:5e:7f:ff:fa, vlan=, eth_type=2048, proto=L3_IPV4], ip=[hl=20, tos=0, len=373, id=0, ttl=4, p=17, src=192.168.1.1, dst=239.255.255.250], ip6=, tcp=, udp=[sport=45335/udp, dport=1900/udp, ulen=353], icmp=] +[l2=[encap=LINK_ETHERNET, len=316, cap_len=316, src=e8:de:27:ff:c0:78, dst=01:00:5e:7f:ff:fa, vlan=, eth_type=2048, proto=L3_IPV4], ip=[hl=20, tos=0, len=302, id=0, ttl=4, p=17, src=192.168.1.1, dst=239.255.255.250], ip6=, tcp=, udp=[sport=45335/udp, dport=1900/udp, ulen=282], icmp=] +[l2=[encap=LINK_ETHERNET, len=375, cap_len=375, src=e8:de:27:ff:c0:78, dst=01:00:5e:7f:ff:fa, vlan=, eth_type=2048, proto=L3_IPV4], ip=[hl=20, tos=0, len=361, id=0, ttl=4, p=17, src=192.168.1.1, dst=239.255.255.250], ip6=, tcp=, udp=[sport=45335/udp, dport=1900/udp, ulen=341], icmp=] +[l2=[encap=LINK_ETHERNET, len=369, cap_len=369, src=e8:de:27:ff:c0:78, dst=01:00:5e:7f:ff:fa, vlan=, eth_type=2048, proto=L3_IPV4], ip=[hl=20, tos=0, len=355, id=0, ttl=4, p=17, src=192.168.1.1, dst=239.255.255.250], ip6=, tcp=, udp=[sport=45335/udp, dport=1900/udp, ulen=335], icmp=] +[l2=[encap=LINK_ETHERNET, len=316, cap_len=316, src=e8:de:27:ff:c0:78, dst=01:00:5e:7f:ff:fa, vlan=, eth_type=2048, proto=L3_IPV4], ip=[hl=20, tos=0, len=302, id=0, ttl=4, p=17, src=192.168.1.1, dst=239.255.255.250], ip6=, tcp=, udp=[sport=45335/udp, dport=1900/udp, ulen=282], icmp=] +[l2=[encap=LINK_ETHERNET, len=371, cap_len=371, src=e8:de:27:ff:c0:78, dst=01:00:5e:7f:ff:fa, vlan=, eth_type=2048, proto=L3_IPV4], ip=[hl=20, tos=0, len=357, id=0, ttl=4, p=17, src=192.168.1.1, dst=239.255.255.250], ip6=, tcp=, udp=[sport=45335/udp, dport=1900/udp, ulen=337], icmp=] +[l2=[encap=LINK_ETHERNET, len=381, cap_len=381, src=e8:de:27:ff:c0:78, dst=01:00:5e:7f:ff:fa, vlan=, eth_type=2048, proto=L3_IPV4], ip=[hl=20, tos=0, len=367, id=0, ttl=4, p=17, src=192.168.1.1, dst=239.255.255.250], ip6=, tcp=, udp=[sport=45335/udp, dport=1900/udp, ulen=347], icmp=] +[l2=[encap=LINK_ETHERNET, len=215, cap_len=215, src=e8:de:27:ff:c0:78, dst=ff:ff:ff:ff:ff:ff, vlan=, eth_type=2048, proto=L3_IPV4], ip=[hl=20, tos=0, len=201, id=0, ttl=64, p=17, src=192.168.1.1, dst=255.255.255.255], ip6=, tcp=, udp=[sport=40190/udp, dport=7437/udp, ulen=181], icmp=] +[l2=[encap=LINK_ETHERNET, len=98, cap_len=98, src=00:50:56:3e:93:6b, dst=e8:de:27:ff:c0:78, vlan=, eth_type=2048, proto=L3_IPV4], ip=[hl=20, tos=0, len=84, id=29257, ttl=64, p=1, src=192.168.1.104, dst=192.168.1.1], ip6=, tcp=, udp=, icmp=[icmp_type=8]] +[l2=[encap=LINK_ETHERNET, len=98, cap_len=98, src=e8:de:27:ff:c0:78, dst=00:50:56:3e:93:6b, vlan=, eth_type=2048, proto=L3_IPV4], ip=[hl=20, tos=0, len=84, id=3684, ttl=64, p=1, src=192.168.1.1, dst=192.168.1.104], ip6=, tcp=, udp=, icmp=[icmp_type=0]] +[l2=[encap=LINK_ETHERNET, len=112, cap_len=112, src=60:f8:1d:c9:8c:fa, dst=e8:de:27:ff:c0:78, vlan=, eth_type=2048, proto=L3_IPV4], ip=[hl=20, tos=0, len=98, id=56893, ttl=64, p=6, src=192.168.1.103, dst=74.125.21.138], ip6=, tcp=[sport=49171/tcp, dport=443/tcp, seq=3725176031, ack=445274592, hl=32, dl=46, flags=24, win=4096], udp=, icmp=] +[l2=[encap=LINK_ETHERNET, len=66, cap_len=66, src=60:f8:1d:c9:8c:fa, dst=e8:de:27:ff:c0:78, vlan=, eth_type=2048, proto=L3_IPV4], ip=[hl=20, tos=0, len=52, id=22643, ttl=64, p=6, src=192.168.1.103, dst=74.125.21.138], ip6=, tcp=[sport=49171/tcp, dport=443/tcp, seq=3725176077, ack=445274652, hl=32, dl=0, flags=16, win=4094], udp=, icmp=] +[l2=[encap=LINK_ETHERNET, len=112, cap_len=112, src=60:f8:1d:c9:8c:fa, dst=e8:de:27:ff:c0:78, vlan=, eth_type=2048, proto=L3_IPV4], ip=[hl=20, tos=0, len=98, id=85, ttl=64, p=6, src=192.168.1.103, dst=74.125.21.138], ip6=, tcp=[sport=49171/tcp, dport=443/tcp, seq=3725176077, ack=445274652, hl=32, dl=46, flags=24, win=4096], udp=, icmp=] +[l2=[encap=LINK_ETHERNET, len=97, cap_len=97, src=60:f8:1d:c9:8c:fa, dst=e8:de:27:ff:c0:78, vlan=, eth_type=2048, proto=L3_IPV4], ip=[hl=20, tos=0, len=83, id=28558, ttl=64, p=6, src=192.168.1.103, dst=74.125.21.138], ip6=, tcp=[sport=49171/tcp, dport=443/tcp, seq=3725176123, ack=445274652, hl=32, dl=31, flags=24, win=4096], udp=, icmp=] +[l2=[encap=LINK_ETHERNET, len=66, cap_len=66, src=60:f8:1d:c9:8c:fa, dst=e8:de:27:ff:c0:78, vlan=, eth_type=2048, proto=L3_IPV4], ip=[hl=20, tos=0, len=52, id=36529, ttl=64, p=6, src=192.168.1.103, dst=74.125.21.138], ip6=, tcp=[sport=49171/tcp, dport=443/tcp, seq=3725176154, ack=445274652, hl=32, dl=0, flags=17, win=4096], udp=, icmp=] +[l2=[encap=LINK_ETHERNET, len=64, cap_len=64, src=00:19:06:ea:b8:c1, dst=ff:ff:ff:ff:ff:ff, vlan=123, eth_type=2054, proto=L3_ARP], ip=, ip6=, tcp=, udp=, icmp=] +[l2=[encap=LINK_ETHERNET, len=64, cap_len=64, src=00:18:73:de:57:c1, dst=ff:ff:ff:ff:ff:ff, vlan=123, eth_type=2054, proto=L3_ARP], ip=, ip6=, tcp=, udp=, icmp=] +[l2=[encap=LINK_ETHERNET, len=64, cap_len=64, src=00:18:73:de:57:c1, dst=ff:ff:ff:ff:ff:ff, vlan=123, eth_type=2054, proto=L3_ARP], ip=, ip6=, tcp=, udp=, icmp=] +[l2=[encap=LINK_ETHERNET, len=64, cap_len=64, src=00:19:06:ea:b8:c1, dst=00:18:73:de:57:c1, vlan=123, eth_type=2054, proto=L3_ARP], ip=, ip6=, tcp=, udp=, icmp=] +[l2=[encap=LINK_ETHERNET, len=118, cap_len=118, src=00:18:73:de:57:c1, dst=00:19:06:ea:b8:c1, vlan=123, eth_type=2048, proto=L3_IPV4], ip=[hl=20, tos=0, len=100, id=5, ttl=255, p=1, src=192.168.123.2, dst=192.168.123.1], ip6=, tcp=, udp=, icmp=[icmp_type=8]] +[l2=[encap=LINK_ETHERNET, len=64, cap_len=64, src=00:19:06:ea:b8:c1, dst=ff:ff:ff:ff:ff:ff, vlan=123, eth_type=2054, proto=L3_ARP], ip=, ip6=, tcp=, udp=, icmp=] +[l2=[encap=LINK_ETHERNET, len=64, cap_len=64, src=00:18:73:de:57:c1, dst=00:19:06:ea:b8:c1, vlan=123, eth_type=2054, proto=L3_ARP], ip=, ip6=, tcp=, udp=, icmp=] +[l2=[encap=LINK_ETHERNET, len=118, cap_len=118, src=00:18:73:de:57:c1, dst=00:19:06:ea:b8:c1, vlan=123, eth_type=2048, proto=L3_IPV4], ip=[hl=20, tos=0, len=100, id=6, ttl=255, p=1, src=192.168.123.2, dst=192.168.123.1], ip6=, tcp=, udp=, icmp=[icmp_type=8]] +[l2=[encap=LINK_ETHERNET, len=118, cap_len=118, src=00:19:06:ea:b8:c1, dst=00:18:73:de:57:c1, vlan=123, eth_type=2048, proto=L3_IPV4], ip=[hl=20, tos=0, len=100, id=6, ttl=255, p=1, src=192.168.123.1, dst=192.168.123.2], ip6=, tcp=, udp=, icmp=[icmp_type=0]] +[l2=[encap=LINK_ETHERNET, len=118, cap_len=118, src=00:18:73:de:57:c1, dst=00:19:06:ea:b8:c1, vlan=123, eth_type=2048, proto=L3_IPV4], ip=[hl=20, tos=0, len=100, id=7, ttl=255, p=1, src=192.168.123.2, dst=192.168.123.1], ip6=, tcp=, udp=, icmp=[icmp_type=8]] +[l2=[encap=LINK_ETHERNET, len=118, cap_len=118, src=00:19:06:ea:b8:c1, dst=00:18:73:de:57:c1, vlan=123, eth_type=2048, proto=L3_IPV4], ip=[hl=20, tos=0, len=100, id=7, ttl=255, p=1, src=192.168.123.1, dst=192.168.123.2], ip6=, tcp=, udp=, icmp=[icmp_type=0]] +[l2=[encap=LINK_ETHERNET, len=118, cap_len=118, src=00:18:73:de:57:c1, dst=00:19:06:ea:b8:c1, vlan=123, eth_type=2048, proto=L3_IPV4], ip=[hl=20, tos=0, len=100, id=8, ttl=255, p=1, src=192.168.123.2, dst=192.168.123.1], ip6=, tcp=, udp=, icmp=[icmp_type=8]] +[l2=[encap=LINK_ETHERNET, len=118, cap_len=118, src=00:19:06:ea:b8:c1, dst=00:18:73:de:57:c1, vlan=123, eth_type=2048, proto=L3_IPV4], ip=[hl=20, tos=0, len=100, id=8, ttl=255, p=1, src=192.168.123.1, dst=192.168.123.2], ip6=, tcp=, udp=, icmp=[icmp_type=0]] +[l2=[encap=LINK_ETHERNET, len=118, cap_len=118, src=00:18:73:de:57:c1, dst=00:19:06:ea:b8:c1, vlan=123, eth_type=2048, proto=L3_IPV4], ip=[hl=20, tos=0, len=100, id=9, ttl=255, p=1, src=192.168.123.2, dst=192.168.123.1], ip6=, tcp=, udp=, icmp=[icmp_type=8]] +[l2=[encap=LINK_ETHERNET, len=118, cap_len=118, src=00:19:06:ea:b8:c1, dst=00:18:73:de:57:c1, vlan=123, eth_type=2048, proto=L3_IPV4], ip=[hl=20, tos=0, len=100, id=9, ttl=255, p=1, src=192.168.123.1, dst=192.168.123.2], ip6=, tcp=, udp=, icmp=[icmp_type=0]] diff --git a/testing/btest/Baseline/core.reassembly/output b/testing/btest/Baseline/core.reassembly/output new file mode 100644 index 0000000000..79922b43c4 --- /dev/null +++ b/testing/btest/Baseline/core.reassembly/output @@ -0,0 +1,32 @@ +---------------------- +flow weird, excessively_small_fragment, 164.1.123.163, 164.1.123.61 +flow weird, fragment_size_inconsistency, 164.1.123.163, 164.1.123.61 +flow weird, fragment_inconsistency, 164.1.123.163, 164.1.123.61 +flow weird, fragment_inconsistency, 164.1.123.163, 164.1.123.61 +flow weird, dns_unmatched_msg, 164.1.123.163, 164.1.123.61 +---------------------- +flow weird, excessively_small_fragment, 164.1.123.163, 164.1.123.61 +flow weird, excessively_small_fragment, 164.1.123.163, 164.1.123.61 +flow weird, fragment_overlap, 164.1.123.163, 164.1.123.61 +---------------------- +flow weird, fragment_with_DF, 210.54.213.247, 131.243.1.10 +flow weird, fragment_with_DF, 210.54.213.247, 131.243.1.10 +flow weird, fragment_with_DF, 210.54.213.247, 131.243.1.10 +flow weird, fragment_with_DF, 210.54.213.247, 131.243.1.10 +flow weird, fragment_with_DF, 210.54.213.247, 131.243.1.10 +---------------------- +flow weird, excessively_small_fragment, 128.32.46.142, 10.0.0.1 +flow weird, excessively_small_fragment, 128.32.46.142, 10.0.0.1 +flow weird, fragment_inconsistency, 128.32.46.142, 10.0.0.1 +---------------------- +net_weird, truncated_IP +net_weird, truncated_IP +net_weird, truncated_IP +net_weird, truncated_IP +rexmit_inconsistency, [orig_h=63.193.213.194, orig_p=2564/tcp, resp_h=128.3.97.175, resp_p=80/tcp], nlkmlpjfjjnoomfnqmdqgrdsgpefslhjrdjghsshrmosrkosidknnieiggpmnggelfhlkflfqojpjrsmeqghklmjlkdskjollmensjiqosemknoehellhlsspjfjpddfgqkemghskqosrksmkpsdomfoghllfokilshsisgpjhjoosidirlnmespjhdogdidoemejrnjjrookfrmiqllllqhlqfgolfqssfjrhrjhgfkpdnigiilrmnespjspeqjfedjhrkisjdhoofqdfeqnmihrelmildkngirkqorjslhmglripdojfedjjngjnpikoliqhdipgpshenekqiphmrsqmemghklodqnqoeggfkdqngrfollhjmddjreeghdqflohgrhqhelqsmdghgihpifpnikrddpmdfejhrhgfdfdlepmmhlhrnrslepqgmkopmdfogpoljeepqoemisfeksdeddiplnkfjddjioqhojlnmlirehidipdhqlddssssgpgikieeldsmfrkidpldsngdkidkoshkrofnonrrehghlmgmqshkedgpkpgjjkoneigsfjdlgjsngepfkndqoefqmsssrgegspromqepdpdeglmmegjljlmljeeorhhfmrohjeregpfshqjsqkekrihjdpfdjflgspepqjrqfemsjffmjfkhejdkrokmgdrhojgmgjpldjeiphroeheipolfmshoglkfnllfnhlflhlpddjflekhiqilefjpfqepdrrdokkjiekmelkhdpjlqjdlnfjemqdrksirdnjlrhrdijgqjhdqlidpfdisgrmnlfnsdlishlpfkshhglpdiqhpgmhpjdrpednjljfsqknsiqpfeqhlphgqdphflglpmqfkkhdjeodkelinkfpmfedidhphldmqjqggrljlhriehqqemeimkjhoqnsrdgengmgjokpeiijgrseppeoiflngggomdfjkndpqedhgnkiqlodkpjfkqoifidjmrdhhmglledkomllhpehdfjfdspmklkjdnhkdgpgqephfdfdrfplmepoegsekmrnikknelnprdpslmfkhghhooknieksjjhdeelidikndedijqqhfmphdondndpehmfoqelqigdpgioeljhedhfoeqlinriemqjigerkphgepqmiiidqlhriqioimpglonlsgomeloipndiihqqfiekkeriokrsjlmsjqiehqsrqkhdjlddjrrllirqkidqiggdrjpjirssgqepnqmhigfsqlekiqdddllnsjmroiofkieqnghddpjnhdjkfloilheljofddrkherkrieeoijrlfghiikmhpfdhekdjloejlmpperkgrhomedpfOOOOOOOOOOOOOOOOOOOOOOOOOOOO, nlkmlpjfjjnoomfnqmdqgrdsgpefslhjrdjghsshrmosrkosidknnieiggpmnggelfhlkflfqojpjrsmeqghklmjlkdskjollmensjiqosemknoehellhlsspjfjpddfgqkemghskqosrksmkpsdomfoghllfokilshsisgpjhjoosidirlnmespjhdogdidoemejrnjjrookfrmiqllllqhlqfgolfqssfjrhrjhgfkpdnigiilrmnespjspeqjfedjhrkisjdhoofqdfeqnmihrelmildkngirkqorjslhmglripdojfedjjngjnpikoliqhdipgpshenekqiphmrsqmemghklodqnqoeggfkdqngrfollhjmddjreeghdqflohgrhqhelqsmdghgihpifpnikrddpmdfejhrhgfdfdlepmmhlhrnrslepqgmkopmdfogpoljeepqoemisfeksdeddiplnkfjddjioqhojlnmlirehidipdhqlddssssgpgikieeldsmfrkidpldsngdkidkoshkrofnonrrehghlmgmqshkedgpkpgjjkoneigsfjdlgjsngepfkndqoefqmsssrgegspromqepdpdeglmmegjljlmljeeorhhfmrohjeregpfshqjsqkekrihjdpfdjflgspepqjrqfemsjffmjfkhejdkrokmgdrhojgmgjpldjeiphroeheipolfmshoglkfnllfnhlflhlpddjflekhiqilefjpfqepdrrdokkjiekmelkhdpjlqjdlnfjemqdrksirdnjlrhrdijgqjhdqlidpfdisgrmnlfnsdlishlpfkshhglpdiqhpgmhpjdrpednjljfsqknsiqpfeqhlphgqdphflglpmqfkkhdjeodkelinkfpmfedidhphldmqjqggrljlhriehqqemeimkjhoqnsrdgengmgjokpeiijgrseppeoiflngggomdfjkndpqedhgnkiqlodkpjfkqoifidjmrdhhmglledkomllhpehdfjfdspmklkjdnhkdgpgqephfdfdrfplmepoegsekmrnikknelnprdpslmfkhghhooknieksjjhdeelidikndedijqqhfmphdondndpehmfoqelqigdpgioeljhedhfoeqlinriemqjigerkphgepqmiiidqlhriqioimpglonlsgomeloipndiihqqfiekkeriokrsjlmsjqiehqsrqkhdjlddjrrllirqkidqiggdrjpjirssgqepnqmhigfsqlekiqdddllnsjmroiofkieqnghddpjnhdjkfloilheljofddrkherkrieeoijrlfghiikmhpfdhekdjloejlmpperkgrhomedpfqkrodjdmrqfpiodgphidfliidlhd +rexmit_inconsistency, [orig_h=63.193.213.194, orig_p=2564/tcp, resp_h=128.3.97.175, resp_p=80/tcp], dgphrodofqhq, orgmmpelofil +rexmit_inconsistency, [orig_h=63.193.213.194, orig_p=2564/tcp, resp_h=128.3.97.175, resp_p=80/tcp], lenhfdqhqfgs, dfpqssidkpdg +rexmit_inconsistency, [orig_h=63.193.213.194, orig_p=2564/tcp, resp_h=128.3.97.175, resp_p=80/tcp], nlkmlpjfjjnoomfnqmdqgrdsgpefslhjrdjghsshrmosrkosidknnieiggpmnggelfhlkflfqojpjrsmeqghklmjlkdskjollmensjiqosemknoehellhlsspjfjpddfgqkemghskqosrksmkpsdomfoghllfokilshsisgpjhjoosidirlnmespjhdogdidoemejrnjjrookfrmiqllllqhlqfgolfqssfjrhrjhgfkpdnigiilrmnespjspeqjfedjhrkisjdhoofqdfeqnmihrelmildkngirkqorjslhmglripdojfedjjngjnpikoliqhdipgpshenekqiphmrsqmemghklodqnqoeggfkdqngrfollhjmddjreeghdqflohgrhqhelqsmdghgihpifpnikrddpmdfejhrhgfdfdlepmmhlhrnrslepqgmkopmdfogpoljeepqoemisfeksdeddiplnkfjddjioqhojlnmlirehidipdhqlddssssgpgikieeldsmfrkidpldsngdkidkoshkrofnonrrehghlmgmqshkedgpkpgjjkoneigsfjdlgjsngepfkndqoefqmsssrgegspromqepdpdeglmmegjljlmljeeorhhfmrohjeregpfshqjsqkekrihjdpfdjflgspepqjrqfemsjffmjfkhejdkrokmgdrhojgmgjpldjeiphroeheipolfmshoglkfnllfnhlflhlpddjflekhiqilefjpfqepdrrdokkjiekmelkhdpjlqjdlnfjemqdrksirdnjlrhrdijgqjhdqlidpfdisgrmnlfnsdlishlpfkshhglpdiqhpgmhpjdrpednjljfsqknsiqpfeqhlphgqdphflglpmqfkkhdjeodkelinkfpmfedidhphldmqjqggrljlhriehqqemeimkjhoqnsrdgengmgjokpeiijgrseppeoiflngggomdfjkndpqedhgnkiqlodkpjfkqoifidjmrdhhmglledkomllhpehdfjfdspmklkjdnhkdgpgqephfdfdrfplmepoegsekmrnikknelnprdpslmfkhghhooknieksjjhdeelidikndedijqqhfmphdondndpehmfoqelqigdpgioeljhedhfoeqlinriemqjigerkphgepqmiiidqlhriqioimpglonlsgomeloipndiihqqfiekkeriokrsjlmsjqiehqsrqkhdjlddjrrllirqkidqiggdrjpjirssgqepnqmhigfsqlekiqdddllnsjmroiofkieqnghddpjnhdjkfloilheljofddrkherkrieeoijrlfghiikmhpfdhekdjloejlmpperkgrhomedpfOOOOOOOOOOOOOOOOOOOOOOOOOOOO, nlkmlpjfjjnoomfnqmdqgrdsgpefslhjrdjghsshrmosrkosidknnieiggpmnggelfhlkflfqojpjrsmeqghklmjlkdskjollmensjiqosemknoehellhlsspjfjpddfgqkemghskqosrksmkpsdomfoghllfokilshsisgpjhjoosidirlnmespjhdogdidoemejrnjjrookfrmiqllllqhlqfgolfqssfjrhrjhgfkpdnigiilrmnespjspeqjfedjhrkisjdhoofqdfeqnmihrelmildkngirkqorjslhmglripdojfedjjngjnpikoliqhdipgpshenekqiphmrsqmemghklodqnqoeggfkdqngrfollhjmddjreeghdqflohgrhqhelqsmdghgihpifpnikrddpmdfejhrhgfdfdlepmmhlhrnrslepqgmkopmdfogpoljeepqoemisfeksdeddiplnkfjddjioqhojlnmlirehidipdhqlddssssgpgikieeldsmfrkidpldsngdkidkoshkrofnonrrehghlmgmqshkedgpkpgjjkoneigsfjdlgjsngepfkndqoefqmsssrgegspromqepdpdeglmmegjljlmljeeorhhfmrohjeregpfshqjsqkekrihjdpfdjflgspepqjrqfemsjffmjfkhejdkrokmgdrhojgmgjpldjeiphroeheipolfmshoglkfnllfnhlflhlpddjflekhiqilefjpfqepdrrdokkjiekmelkhdpjlqjdlnfjemqdrksirdnjlrhrdijgqjhdqlidpfdisgrmnlfnsdlishlpfkshhglpdiqhpgmhpjdrpednjljfsqknsiqpfeqhlphgqdphflglpmqfkkhdjeodkelinkfpmfedidhphldmqjqggrljlhriehqqemeimkjhoqnsrdgengmgjokpeiijgrseppeoiflngggomdfjkndpqedhgnkiqlodkpjfkqoifidjmrdhhmglledkomllhpehdfjfdspmklkjdnhkdgpgqephfdfdrfplmepoegsekmrnikknelnprdpslmfkhghhooknieksjjhdeelidikndedijqqhfmphdondndpehmfoqelqigdpgioeljhedhfoeqlinriemqjigerkphgepqmiiidqlhriqioimpglonlsgomeloipndiihqqfiekkeriokrsjlmsjqiehqsrqkhdjlddjrrllirqkidqiggdrjpjirssgqepnqmhigfsqlekiqdddllnsjmroiofkieqnghddpjnhdjkfloilheljofddrkherkrieeoijrlfghiikmhpfdhekdjloejlmpperkgrhomedpfqkrodjdmrqfpiodgphidfliislrr +rexmit_inconsistency, [orig_h=63.193.213.194, orig_p=2564/tcp, resp_h=128.3.97.175, resp_p=80/tcp], iokgedlsdkjkiefgmeqkfjoh, ggdeolssksemrhedoledddml +net_weird, truncated_IP +rexmit_inconsistency, [orig_h=63.193.213.194, orig_p=2564/tcp, resp_h=128.3.97.175, resp_p=80/tcp], OOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOO HTTP/1.1\x0d\x0aHost: 127.0.0.1\x0d\x0aContent-Type: text/xml\x0d\x0aContent-length: 1\x0d\x0a\x0d\x0aO\x0d\x0aoutput +# @TEST-EXEC: bro -b -r $TRACES/icmp_dot1q.trace %INPUT >>output +# @TEST-EXEC: btest-diff output + +event raw_packet(p: raw_pkt_hdr) + { + print p; + } + diff --git a/testing/btest/core/reassembly.bro b/testing/btest/core/reassembly.bro new file mode 100644 index 0000000000..cb04ace71b --- /dev/null +++ b/testing/btest/core/reassembly.bro @@ -0,0 +1,26 @@ +# @TEST-EXEC: bro -C -r $TRACES/ipv4/fragmented-1.pcap %INPUT >>output +# @TEST-EXEC: bro -C -r $TRACES/ipv4/fragmented-2.pcap %INPUT >>output +# @TEST-EXEC: bro -C -r $TRACES/ipv4/fragmented-3.pcap %INPUT >>output +# @TEST-EXEC: bro -C -r $TRACES/ipv4/fragmented-4.pcap %INPUT >>output +# @TEST-EXEC: bro -C -r $TRACES/tcp/reassembly.pcap %INPUT >>output +# @TEST-EXEC: btest-diff output + +event bro_init() + { + print "----------------------"; + } + +event flow_weird(name: string, src: addr, dst: addr) + { + print "flow weird", name, src, dst; + } + +event net_weird(name: string) + { + print "net_weird", name; + } + +event rexmit_inconsistency(c: connection, t1: string, t2: string) + { + print "rexmit_inconsistency", c$id, t1, t2 ; + } diff --git a/testing/btest/core/tcp/quantum-insert.bro b/testing/btest/core/tcp/quantum-insert.bro new file mode 100644 index 0000000000..3fff5cd911 --- /dev/null +++ b/testing/btest/core/tcp/quantum-insert.bro @@ -0,0 +1,12 @@ +# @TEST-EXEC: bro -b -r $TRACES/tcp/qi_internet_SYNACK_curl_jsonip.pcap %INPUT +# @TEST-EXEC: btest-diff .stdout + +# Quantum Insert like attack, overlapping TCP packet with different content +redef tcp_max_old_segments = 10; +event rexmit_inconsistency(c: connection, t1: string, t2: string) + { + print "----- rexmit_inconsistency -----"; + print fmt("%.6f c: %s", network_time(), c$id); + print fmt("%.6f t1: %s", network_time(), t1); + print fmt("%.6f t2: %s", network_time(), t2); + } diff --git a/testing/btest/plugins/pktdumper-plugin/src/Foo.cc b/testing/btest/plugins/pktdumper-plugin/src/Foo.cc index fdd364b034..c68eec809a 100644 --- a/testing/btest/plugins/pktdumper-plugin/src/Foo.cc +++ b/testing/btest/plugins/pktdumper-plugin/src/Foo.cc @@ -29,8 +29,8 @@ void Foo::Close() bool Foo::Dump(const Packet* pkt) { - double t = double(pkt->hdr->ts.tv_sec) + double(pkt->hdr->ts.tv_usec) / 1e6; - fprintf(stdout, "Dumping to %s: %.6f len %u\n", props.path.c_str(), t, (unsigned int)pkt->hdr->len); + double t = double(pkt->ts.tv_sec) + double(pkt->ts.tv_usec) / 1e6; + fprintf(stdout, "Dumping to %s: %.6f len %u\n", props.path.c_str(), t, (unsigned int)pkt->len); return true; } diff --git a/testing/btest/plugins/pktsrc-plugin/src/Foo.cc b/testing/btest/plugins/pktsrc-plugin/src/Foo.cc index b08bc51d72..af752a20bf 100644 --- a/testing/btest/plugins/pktsrc-plugin/src/Foo.cc +++ b/testing/btest/plugins/pktsrc-plugin/src/Foo.cc @@ -17,7 +17,6 @@ Foo::Foo(const std::string& path, bool is_live) props.path = path; props.selectable_fd = open("/bin/sh", O_RDONLY); // any fd is fine. props.link_type = DLT_RAW; - props.hdr_size = 0; props.netmask = 0; props.is_live = 0; } @@ -45,12 +44,9 @@ bool Foo::ExtractNextPacket(Packet* pkt) return false; } - hdr.ts.tv_sec = 1409193037; - hdr.ts.tv_usec = 0; - hdr.caplen = hdr.len = packet.size(); - pkt->ts = hdr.ts.tv_sec; - pkt->hdr = &hdr; - pkt->data = (const u_char *)packet.c_str(); + struct timeval ts = { 1409193037, 0 }; + pkt->Init(props.link_type, &ts, packet.size(), packet.size(), + (const u_char *)packet.c_str()); return true; } diff --git a/testing/btest/plugins/pktsrc-plugin/src/Foo.h b/testing/btest/plugins/pktsrc-plugin/src/Foo.h index 902ac0e37a..922b300b61 100644 --- a/testing/btest/plugins/pktsrc-plugin/src/Foo.h +++ b/testing/btest/plugins/pktsrc-plugin/src/Foo.h @@ -26,7 +26,6 @@ protected: private: Properties props; string packet; - struct pcap_pkthdr hdr; }; } diff --git a/testing/btest/scripts/base/protocols/http/missing-zlib-header.bro b/testing/btest/scripts/base/protocols/http/missing-zlib-header.bro new file mode 100644 index 0000000000..25923f70da --- /dev/null +++ b/testing/btest/scripts/base/protocols/http/missing-zlib-header.bro @@ -0,0 +1,6 @@ +# This tests an issue where some web servers don't +# include an appropriate ZLIB header on deflated +# content. +# +# @TEST-EXEC: bro -r $TRACES/http/missing-zlib-header.pcap %INPUT +# @TEST-EXEC: btest-diff http.log diff --git a/testing/btest/scripts/policy/frameworks/intel/seen/certs.bro b/testing/btest/scripts/policy/frameworks/intel/seen/certs.bro index afddc6b2d9..2ab4c6a50a 100644 --- a/testing/btest/scripts/policy/frameworks/intel/seen/certs.bro +++ b/testing/btest/scripts/policy/frameworks/intel/seen/certs.bro @@ -16,3 +16,13 @@ www.dresdner-privat.de Intel::DOMAIN source1 test entry http://some-data-distrib redef Intel::read_files += { "intel.dat" }; +event bro_init() + { + suspend_processing(); + } + +event Input::end_of_data(name: string, source: string) + { + continue_processing(); + } + diff --git a/testing/external/subdir-btest.cfg b/testing/external/subdir-btest.cfg index b2b36bf84f..4315ade850 100644 --- a/testing/external/subdir-btest.cfg +++ b/testing/external/subdir-btest.cfg @@ -15,6 +15,7 @@ TEST_DIFF_CANONIFIER=%(testbase)s/../../scripts/diff-canonifier-external TEST_DIFF_BRIEF=1 TRACES=%(testbase)s/Traces SCRIPTS=%(testbase)s/../scripts +SCRIPTS_LOCAL=%(testbase)s/scripts DIST=%(testbase)s/../../.. BUILD=%(testbase)s/../../../build BRO_PROFILER_FILE=%(testbase)s/.tmp/script-coverage.XXXXXX