From 93d9dde969a9bc3983b4bcb656970453730ae3eb Mon Sep 17 00:00:00 2001 From: Robin Sommer Date: Tue, 10 Dec 2013 16:26:34 -0800 Subject: [PATCH 01/25] IOSource reorg. A bunch of infrastructure work to move IOSource, IOSourceRegistry (now iosource::Manager) and PktSrc/PktDumper code into iosource/, and over to a plugin structure. Other IOSources aren't touched yet, they are still in src/*. It compiles and does something with a small trace, but that's all I've tested so far. There are quite certainly a number of problems left, as well as various TODOs and cleanup; and nothing's cast in stone yet. Will continue to work on this. --- TODO.iosources | 9 + aux/broccoli | 2 +- aux/broctl | 2 +- src/CMakeLists.txt | 7 +- src/DNS_Mgr.cc | 13 +- src/DNS_Mgr.h | 6 +- src/DebugLogger.cc | 3 +- src/DebugLogger.h | 1 + src/FlowSrc.cc | 15 +- src/FlowSrc.h | 4 +- src/Net.cc | 119 +-- src/Net.h | 22 +- src/PacketSort.cc | 2 +- src/PacketSort.h | 8 +- src/PktSrc.cc | 796 ------------------ src/PktSrc.h | 258 ------ src/RemoteSerializer.cc | 34 +- src/RemoteSerializer.h | 4 +- src/Serializer.cc | 19 +- src/Serializer.h | 4 +- src/Sessions.cc | 24 +- src/Sessions.h | 4 +- src/bro.bif | 73 +- src/iosource/CMakeLists.txt | 23 + src/iosource/Component.cc | 44 + src/iosource/Component.h | 56 ++ src/{ => iosource}/IOSource.h | 76 +- src/{IOSource.cc => iosource/Manager.cc} | 151 +++- src/iosource/Manager.h | 75 ++ src/iosource/pktsrc/CMakeLists.txt | 2 + src/iosource/pktsrc/Component.cc | 130 +++ src/iosource/pktsrc/Component.h | 132 +++ src/iosource/pktsrc/PktDumper.cc | 79 ++ src/iosource/pktsrc/PktDumper.h | 57 ++ src/iosource/pktsrc/PktSrc.cc | 411 +++++++++ src/iosource/pktsrc/PktSrc.h | 140 +++ src/iosource/pktsrc/old-2ndary-code.h | 69 ++ src/{ => iosource/pktsrc/pcap}/BPF_Program.cc | 0 src/{ => iosource/pktsrc/pcap}/BPF_Program.h | 0 src/iosource/pktsrc/pcap/CMakeLists.txt | 8 + src/iosource/pktsrc/pcap/Dumper.cc | 111 +++ src/iosource/pktsrc/pcap/Dumper.h | 40 + src/iosource/pktsrc/pcap/Plugin.cc | 12 + src/iosource/pktsrc/pcap/Source.cc | 343 ++++++++ src/iosource/pktsrc/pcap/Source.h | 60 ++ src/main.cc | 16 +- src/plugin/Component.cc | 12 + src/plugin/Component.h | 7 +- src/plugin/Macros.h | 27 + src/threading/Manager.cc | 8 +- src/threading/Manager.h | 4 +- src/util.cc | 7 +- 52 files changed, 2223 insertions(+), 1306 deletions(-) create mode 100644 TODO.iosources delete mode 100644 src/PktSrc.cc delete mode 100644 src/PktSrc.h create mode 100644 src/iosource/CMakeLists.txt create mode 100644 src/iosource/Component.cc create mode 100644 src/iosource/Component.h rename src/{ => iosource}/IOSource.h (51%) rename src/{IOSource.cc => iosource/Manager.cc} (52%) create mode 100644 src/iosource/Manager.h create mode 100644 src/iosource/pktsrc/CMakeLists.txt create mode 100644 src/iosource/pktsrc/Component.cc create mode 100644 src/iosource/pktsrc/Component.h create mode 100644 src/iosource/pktsrc/PktDumper.cc create mode 100644 src/iosource/pktsrc/PktDumper.h create mode 100644 src/iosource/pktsrc/PktSrc.cc create mode 100644 src/iosource/pktsrc/PktSrc.h create mode 100644 src/iosource/pktsrc/old-2ndary-code.h rename src/{ => iosource/pktsrc/pcap}/BPF_Program.cc (100%) rename src/{ => iosource/pktsrc/pcap}/BPF_Program.h (100%) create mode 100644 src/iosource/pktsrc/pcap/CMakeLists.txt create mode 100644 src/iosource/pktsrc/pcap/Dumper.cc create mode 100644 src/iosource/pktsrc/pcap/Dumper.h create mode 100644 src/iosource/pktsrc/pcap/Plugin.cc create mode 100644 src/iosource/pktsrc/pcap/Source.cc create mode 100644 src/iosource/pktsrc/pcap/Source.h diff --git a/TODO.iosources b/TODO.iosources new file mode 100644 index 0000000000..7380c89b92 --- /dev/null +++ b/TODO.iosources @@ -0,0 +1,9 @@ +- Move the current_{iosrc,pkt_src,etc.} into manager +- Remove all 2ndary path code +- Remove all flow src code. +- Move pktsrc/*.{h,cc} up a level? Or create a subsublibrary there? +- Create a global Packet data structure and pass that around instead + of the pcap_* stuff? +- PktDumper: Move Dump() to public and remove Record() +- Wrap BPF_Program into namespace and clean up +- Tests, in particular the packet dumping needs testing. diff --git a/aux/broccoli b/aux/broccoli index e02ccc0a27..17ec437752 160000 --- a/aux/broccoli +++ b/aux/broccoli @@ -1 +1 @@ -Subproject commit e02ccc0a27e64b147f01e4c7deb5b897864d59d5 +Subproject commit 17ec437752837fb4214abfb0a2da49df74668d5d diff --git a/aux/broctl b/aux/broctl index 2e07720b4f..6e01d6972f 160000 --- a/aux/broctl +++ b/aux/broctl @@ -1 +1 @@ -Subproject commit 2e07720b4f129802e07ca99498e2aff4542c737a +Subproject commit 6e01d6972f02d68ee82d05f392d1a00725595b7f diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 8e22b504e4..1aede44934 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -151,10 +151,12 @@ list(APPEND BINPAC_OUTPUTS "${BINPAC_OUTPUT_CC}") set(bro_SUBDIR_LIBS CACHE INTERNAL "subdir libraries" FORCE) set(bro_PLUGIN_LIBS CACHE INTERNAL "plugin libraries" FORCE) +add_subdirectory(iosource) add_subdirectory(analyzer) add_subdirectory(file_analysis) add_subdirectory(probabilistic) add_subdirectory(broxygen) +add_subdirectory(iosource) set(bro_SUBDIRS ${bro_SUBDIR_LIBS} @@ -249,7 +251,8 @@ set(bro_SRCS Anon.cc Attr.cc Base64.cc - BPF_Program.cc + BroDoc.cc + BroDocObj.cc Brofiler.cc BroString.cc CCL.cc @@ -281,7 +284,6 @@ set(bro_SRCS Hash.cc ID.cc IntSet.cc - IOSource.cc IP.cc IPAddr.cc List.cc @@ -295,7 +297,6 @@ set(bro_SRCS PacketFilter.cc PacketSort.cc PersistenceSerializer.cc - PktSrc.cc PolicyFile.cc PrefixTable.cc PriorityQueue.cc diff --git a/src/DNS_Mgr.cc b/src/DNS_Mgr.cc index 17409a930b..4d96e21a3e 100644 --- a/src/DNS_Mgr.cc +++ b/src/DNS_Mgr.cc @@ -34,6 +34,7 @@ #include "Net.h" #include "Var.h" #include "Reporter.h" +#include "iosource/Manager.h" extern "C" { extern int select(int, fd_set *, fd_set *, fd_set *, struct timeval *); @@ -404,17 +405,17 @@ DNS_Mgr::~DNS_Mgr() delete [] dir; } -bool DNS_Mgr::Init() +void DNS_Mgr::Init() { if ( did_init ) - return true; + return; const char* cache_dir = dir ? dir : "."; if ( mode == DNS_PRIME && ! ensure_dir(cache_dir) ) { did_init = 0; - return false; + return; } cache_name = new char[strlen(cache_dir) + 64]; @@ -433,14 +434,12 @@ bool DNS_Mgr::Init() did_init = 1; - io_sources.Register(this, true); + iosource_mgr->Register(this, true); // We never set idle to false, having the main loop only calling us from // time to time. If we're issuing more DNS requests than we can handle // in this way, we are having problems anyway ... - idle = true; - - return true; + SetIdle(true); } TableVal* DNS_Mgr::LookupHost(const char* name) diff --git a/src/DNS_Mgr.h b/src/DNS_Mgr.h index bfcc70a5c2..069c7e1a2b 100644 --- a/src/DNS_Mgr.h +++ b/src/DNS_Mgr.h @@ -12,7 +12,7 @@ #include "BroList.h" #include "Dict.h" #include "EventHandler.h" -#include "IOSource.h" +#include "iosource/IOSource.h" #include "IPAddr.h" class Val; @@ -40,12 +40,12 @@ enum DNS_MgrMode { // Number of seconds we'll wait for a reply. #define DNS_TIMEOUT 5 -class DNS_Mgr : public IOSource { +class DNS_Mgr : public iosource::IOSource { public: DNS_Mgr(DNS_MgrMode mode); virtual ~DNS_Mgr(); - bool Init(); + void Init(); void Flush(); // Looks up the address or addresses of the given host, and returns diff --git a/src/DebugLogger.cc b/src/DebugLogger.cc index 78377eafcf..05f0f9e89f 100644 --- a/src/DebugLogger.cc +++ b/src/DebugLogger.cc @@ -17,7 +17,8 @@ DebugLogger::Stream DebugLogger::streams[NUM_DBGS] = { { "dpd", 0, false }, { "tm", 0, false }, { "logging", 0, false }, {"input", 0, false }, { "threading", 0, false }, { "file_analysis", 0, false }, - { "plugins", 0, false }, { "broxygen", 0, false } + { "plugins", 0, false }, { "broxygen", 0, false }, + { "pktio", 0, false} }; DebugLogger::DebugLogger(const char* filename) diff --git a/src/DebugLogger.h b/src/DebugLogger.h index d1f053788e..b098430a9a 100644 --- a/src/DebugLogger.h +++ b/src/DebugLogger.h @@ -29,6 +29,7 @@ enum DebugStream { DBG_FILE_ANALYSIS, // File analysis DBG_PLUGINS, DBG_BROXYGEN, + DBG_PKTIO, // Packet sources and dumpers. NUM_DBGS // Has to be last }; diff --git a/src/FlowSrc.cc b/src/FlowSrc.cc index 32aa4c4e3a..fb9676ab5b 100644 --- a/src/FlowSrc.cc +++ b/src/FlowSrc.cc @@ -15,7 +15,6 @@ FlowSrc::FlowSrc() { // TODO: v9. selectable_fd = -1; - idle = false; data = 0; pdu_len = -1; exporter_ip = 0; @@ -80,7 +79,7 @@ int FlowSocketSrc::ExtractNextPDU() reporter->Error("problem reading NetFlow data from socket"); data = 0; next_timestamp = -1.0; - closed = 1; + SetClosed(true); return 0; } @@ -115,7 +114,7 @@ FlowSocketSrc::FlowSocketSrc(const char* listen_parms) snprintf(errbuf, BRO_FLOW_ERRBUF_SIZE, "parsing your listen-spec went nuts: laddr='%s', port='%s'\n", laddr[0] ? laddr : "", port[0] ? port : ""); - closed = 1; + SetClosed(true); return; } @@ -131,7 +130,7 @@ FlowSocketSrc::FlowSocketSrc(const char* listen_parms) snprintf(errbuf, BRO_FLOW_ERRBUF_SIZE, "getaddrinfo(%s, %s, ...): %s", laddr, port, gai_strerror(ret)); - closed = 1; + SetClosed(true); return; } @@ -139,7 +138,7 @@ FlowSocketSrc::FlowSocketSrc(const char* listen_parms) { snprintf(errbuf, BRO_FLOW_ERRBUF_SIZE, "socket: %s", strerror(errno)); - closed = 1; + SetClosed(true); goto cleanup; } @@ -147,7 +146,7 @@ FlowSocketSrc::FlowSocketSrc(const char* listen_parms) { snprintf(errbuf, BRO_FLOW_ERRBUF_SIZE, "bind: %s", strerror(errno)); - closed = 1; + SetClosed(true); goto cleanup; } @@ -211,7 +210,7 @@ FlowFileSrc::FlowFileSrc(const char* readfile) selectable_fd = open(this->readfile, O_RDONLY); if ( selectable_fd < 0 ) { - closed = 1; + SetClosed(true); snprintf(errbuf, BRO_FLOW_ERRBUF_SIZE, "open: %s", strerror(errno)); } @@ -223,6 +222,6 @@ int FlowFileSrc::Error(int errlvl, const char* errmsg) "%s: %s", errmsg, strerror(errlvl)); data = 0; next_timestamp = -1.0; - closed = 1; + SetClosed(true); return 0; } diff --git a/src/FlowSrc.h b/src/FlowSrc.h index 03dda2761d..71c8b0cd11 100644 --- a/src/FlowSrc.h +++ b/src/FlowSrc.h @@ -5,7 +5,7 @@ #ifndef flowsrc_h #define flowsrc_h -#include "IOSource.h" +#include "iosource/IOSource.h" #include "NetVar.h" #include "binpac.h" @@ -28,7 +28,7 @@ namespace binpac { } } -class FlowSrc : public IOSource { +class FlowSrc : public iosource::IOSource { public: virtual ~FlowSrc(); diff --git a/src/Net.cc b/src/Net.cc index ac4dacf9b8..64baeff4aa 100644 --- a/src/Net.cc +++ b/src/Net.cc @@ -30,6 +30,9 @@ #include "PacketSort.h" #include "Serializer.h" #include "PacketDumper.h" +#include "iosource/Manager.h" +#include "iosource/pktsrc/PktSrc.h" +#include "iosource/pktsrc/PktDumper.h" extern "C" { #include "setsignal.h" @@ -39,10 +42,7 @@ extern "C" { extern int select(int, fd_set *, fd_set *, fd_set *, struct timeval *); } -PList(PktSrc) pkt_srcs; - -// FIXME: We should really merge PktDumper and PacketDumper. -PktDumper* pkt_dumper = 0; +iosource::PktDumper* pkt_dumper = 0; int reading_live = 0; int reading_traces = 0; @@ -65,8 +65,8 @@ const u_char* current_pkt = 0; int current_dispatched = 0; int current_hdr_size = 0; double current_timestamp = 0.0; -PktSrc* current_pktsrc = 0; -IOSource* current_iosrc; +iosource::PktSrc* current_pktsrc = 0; +iosource::IOSource* current_iosrc = 0; std::list files_scanned; std::vector sig_files; @@ -115,8 +115,8 @@ RETSIGTYPE watchdog(int /* signo */) // saving the packet which caused the // watchdog to trigger may be helpful, // so we'll save that one nevertheless. - pkt_dumper = new PktDumper("watchdog-pkt.pcap"); - if ( pkt_dumper->IsError() ) + pkt_dumper = iosource_mgr->OpenPktDumper("watchdog-pkt.pcap", false); + if ( ! pkt_dumper || pkt_dumper->IsError() ) { reporter->Error("watchdog: can't open watchdog-pkt.pcap for writing\n"); delete pkt_dumper; @@ -125,7 +125,12 @@ RETSIGTYPE watchdog(int /* signo */) } if ( pkt_dumper ) - pkt_dumper->Dump(current_hdr, current_pkt); + { + iosource::PktDumper::Packet p; + p.hdr = current_hdr; + p.data = current_pkt; + pkt_dumper->Record(&p); + } } net_get_final_stats(); @@ -157,18 +162,10 @@ void net_init(name_list& interfaces, name_list& readfiles, reading_traces = 1; for ( int i = 0; i < readfiles.length(); ++i ) - { - PktFileSrc* ps = new PktFileSrc(readfiles[i], filter); - - if ( ! ps->IsOpen() ) - reporter->FatalError("%s: problem with trace file %s - %s\n", - prog, readfiles[i], ps->ErrorMsg()); - else - { - pkt_srcs.append(ps); - io_sources.Register(ps); - } + iosource_mgr->OpenPktSrc(readfiles[i], filter, false); + } +#if 0 if ( secondary_filter ) { // We use a second PktFileSrc for the @@ -189,7 +186,6 @@ void net_init(name_list& interfaces, name_list& readfiles, ps->AddSecondaryTablePrograms(); } - } for ( int i = 0; i < flowfiles.length(); ++i ) { @@ -203,7 +199,7 @@ void net_init(name_list& interfaces, name_list& readfiles, io_sources.Register(fs); } } - } +#endif else if ((interfaces.length() > 0 || netflows.length() > 0)) { @@ -211,22 +207,13 @@ void net_init(name_list& interfaces, name_list& readfiles, reading_traces = 0; for ( int i = 0; i < interfaces.length(); ++i ) - { - PktSrc* ps; - ps = new PktInterfaceSrc(interfaces[i], filter); - - if ( ! ps->IsOpen() ) - reporter->FatalError("%s: problem with interface %s - %s\n", - prog, interfaces[i], ps->ErrorMsg()); - else - { - pkt_srcs.append(ps); - io_sources.Register(ps); - } + iosource_mgr->OpenPktSrc(interfaces[i], filter, true); + } +#if 0 if ( secondary_filter ) { - PktSrc* ps; + iosource::PktSrc* ps; ps = new PktInterfaceSrc(interfaces[i], filter, TYPE_FILTER_SECONDARY); @@ -258,8 +245,7 @@ void net_init(name_list& interfaces, name_list& readfiles, else io_sources.Register(fs); } - - } +#endif else // have_pending_timers = 1, possibly. We don't set @@ -270,12 +256,7 @@ void net_init(name_list& interfaces, name_list& readfiles, if ( writefile ) { - // ### This will fail horribly if there are multiple - // interfaces with different-lengthed media. - pkt_dumper = new PktDumper(writefile); - if ( pkt_dumper->IsError() ) - reporter->FatalError("%s: can't open write file \"%s\" - %s\n", - prog, writefile, pkt_dumper->ErrorMsg()); + pkt_dumper = iosource_mgr->OpenPktDumper(writefile, false); ID* id = global_scope()->Lookup("trace_output_file"); if ( ! id ) @@ -299,7 +280,7 @@ void net_init(name_list& interfaces, name_list& readfiles, } } -void expire_timers(PktSrc* src_ps) +void expire_timers(iosource::PktSrc* src_ps) { SegmentProfiler(segment_logger, "expiring-timers"); TimerMgr* tmgr = @@ -313,7 +294,7 @@ void expire_timers(PktSrc* src_ps) void net_packet_dispatch(double t, const struct pcap_pkthdr* hdr, const u_char* pkt, int hdr_size, - PktSrc* src_ps, PacketSortElement* pkt_elem) + iosource::PktSrc* src_ps, PacketSortElement* pkt_elem) { if ( ! bro_start_network_time ) bro_start_network_time = t; @@ -394,7 +375,7 @@ int process_packet_sorter(double latest_packet_time) void net_packet_arrival(double t, const struct pcap_pkthdr* hdr, const u_char* pkt, int hdr_size, - PktSrc* src_ps) + iosource::PktSrc* src_ps) { if ( packet_sorter ) { @@ -421,12 +402,12 @@ void net_run() { set_processing_status("RUNNING", "net_run"); - while ( io_sources.Size() || + while ( iosource_mgr->Size() || (packet_sorter && ! packet_sorter->Empty()) || (BifConst::exit_only_after_terminate && ! terminating) ) { double ts; - IOSource* src = io_sources.FindSoonest(&ts); + iosource::IOSource* src = iosource_mgr->FindSoonest(&ts); #ifdef DEBUG static int loop_counter = 0; @@ -535,16 +516,19 @@ void net_run() void net_get_final_stats() { - loop_over_list(pkt_srcs, i) + const iosource::Manager::PktSrcList& pkt_srcs(iosource_mgr->GetPktSrcs()); + + for ( iosource::Manager::PktSrcList::const_iterator i = pkt_srcs.begin(); + i != pkt_srcs.end(); i++ ) { - PktSrc* ps = pkt_srcs[i]; + iosource::PktSrc* ps = *i; if ( ps->IsLive() ) { - struct PktSrc::Stats s; + iosource::PktSrc::Stats s; ps->Statistics(&s); reporter->Info("%d packets received on interface %s, %d dropped\n", - s.received, ps->Interface(), s.dropped); + s.received, ps->Path().c_str(), s.dropped); } } } @@ -587,29 +571,6 @@ void net_delete() delete ip_anonymizer[i]; } -// net_packet_match -// -// Description: -// - Checks if a packet matches a filter. It just wraps up a call to -// [pcap.h's] bpf_filter(). -// -// Inputs: -// - fp: a BPF-compiled filter -// - pkt: a pointer to the packet -// - len: the original packet length -// - caplen: the captured packet length. This is pkt length -// -// Output: -// - return: 1 if the packet matches the filter, 0 otherwise - -int net_packet_match(BPF_Program* fp, const u_char* pkt, - u_int len, u_int caplen) - { - // NOTE: I don't like too much un-const'ing the pkt variable. - return bpf_filter(fp->GetProgram()->bf_insns, (u_char*) pkt, len, caplen); - } - - int _processing_suspended = 0; static double suspend_start = 0; @@ -627,8 +588,12 @@ void net_continue_processing() if ( _processing_suspended == 1 ) { reporter->Info("processing continued"); - loop_over_list(pkt_srcs, i) - pkt_srcs[i]->ContinueAfterSuspend(); + + const iosource::Manager::PktSrcList& pkt_srcs(iosource_mgr->GetPktSrcs()); + + for ( iosource::Manager::PktSrcList::const_iterator i = pkt_srcs.begin(); + i != pkt_srcs.end(); i++ ) + (*i)->ContinueAfterSuspend(); } --_processing_suspended; diff --git a/src/Net.h b/src/Net.h index 07c856d1dd..421bee5911 100644 --- a/src/Net.h +++ b/src/Net.h @@ -4,13 +4,18 @@ #define net_h #include "net_util.h" +<<<<<<< HEAD #include "util.h" #include "BPF_Program.h" +======= +>>>>>>> 5493253... Checkpoint. #include "List.h" -#include "PktSrc.h" #include "FlowSrc.h" #include "Func.h" #include "RemoteSerializer.h" +#include "iosource/IOSource.h" +#include "iosource/pktsrc/PktSrc.h" +#include "iosource/pktsrc/PktDumper.h" extern void net_init(name_list& interfaces, name_list& readfiles, name_list& netflows, name_list& flowfiles, @@ -22,10 +27,8 @@ extern void net_finish(int drain_events); extern void net_delete(); // Reclaim all memory, etc. extern void net_packet_arrival(double t, const struct pcap_pkthdr* hdr, const u_char* pkt, int hdr_size, - PktSrc* src_ps); -extern int net_packet_match(BPF_Program* fp, const u_char* pkt, - u_int len, u_int caplen); -extern void expire_timers(PktSrc* src_ps = 0); + iosource::PktSrc* src_ps); +extern void expire_timers(iosource::PktSrc* src_ps = 0); extern void termination_signal(); // Functions to temporarily suspend processing of live input (network packets @@ -82,13 +85,10 @@ extern const u_char* current_pkt; extern int current_dispatched; extern int current_hdr_size; extern double current_timestamp; -extern PktSrc* current_pktsrc; -extern IOSource* current_iosrc; +extern iosource::PktSrc* current_pktsrc; +extern iosource::IOSource* current_iosrc; -declare(PList,PktSrc); -extern PList(PktSrc) pkt_srcs; - -extern PktDumper* pkt_dumper; // where to save packets +extern iosource::PktDumper* pkt_dumper; // where to save packets extern char* writefile; diff --git a/src/PacketSort.cc b/src/PacketSort.cc index 429d8e2720..606d21b689 100644 --- a/src/PacketSort.cc +++ b/src/PacketSort.cc @@ -3,7 +3,7 @@ const bool DEBUG_packetsort = false; -PacketSortElement::PacketSortElement(PktSrc* arg_src, +PacketSortElement::PacketSortElement(iosource::PktSrc* arg_src, double arg_timestamp, const struct pcap_pkthdr* arg_hdr, const u_char* arg_pkt, int arg_hdr_size) { diff --git a/src/PacketSort.h b/src/PacketSort.h index 199da0732f..d61f66994e 100644 --- a/src/PacketSort.h +++ b/src/PacketSort.h @@ -16,16 +16,14 @@ enum { NUM_OF_PQ_LEVEL, }; -class PktSrc; - class PacketSortElement { public: - PacketSortElement(PktSrc* src, double timestamp, + PacketSortElement(iosource::PktSrc* src, double timestamp, const struct pcap_pkthdr* hdr, const u_char* pkt, int hdr_size); ~PacketSortElement(); - PktSrc* Src() const { return src; } + iosource::PktSrc* Src() const { return src; } double TimeStamp() const { return timestamp; } const struct pcap_pkthdr* Hdr() const { return &hdr; } const u_char* Pkt() const { return pkt; } @@ -33,7 +31,7 @@ public: const IP_Hdr* IPHdr() const { return ip_hdr; } protected: - PktSrc* src; + iosource::PktSrc* src; double timestamp; struct pcap_pkthdr hdr; u_char* pkt; diff --git a/src/PktSrc.cc b/src/PktSrc.cc deleted file mode 100644 index 9d6bce6fe9..0000000000 --- a/src/PktSrc.cc +++ /dev/null @@ -1,796 +0,0 @@ -// See the file "COPYING" in the main distribution directory for copyright. - -#include -#include - -#include "config.h" - -#include "util.h" -#include "PktSrc.h" -#include "Hash.h" -#include "Net.h" -#include "Sessions.h" - - -// ### This needs auto-confing. -#ifdef HAVE_PCAP_INT_H -#include -#endif - -PktSrc::PktSrc() - { - interface = readfile = 0; - data = last_data = 0; - memset(&hdr, 0, sizeof(hdr)); - hdr_size = 0; - datalink = 0; - netmask = 0xffffff00; - pd = 0; - idle = false; - - next_sync_point = 0; - first_timestamp = current_timestamp = next_timestamp = 0.0; - first_wallclock = current_wallclock = 0; - - stats.received = stats.dropped = stats.link = 0; - } - -PktSrc::~PktSrc() - { - Close(); - - loop_over_list(program_list, i) - delete program_list[i]; - - BPF_Program* code; - IterCookie* cookie = filters.InitForIteration(); - while ( (code = filters.NextEntry(cookie)) ) - delete code; - - delete [] interface; - delete [] readfile; - } - -void PktSrc::GetFds(int* read, int* write, int* except) - { - if ( pseudo_realtime ) - { - // Select would give erroneous results. But we simulate it - // by setting idle accordingly. - idle = CheckPseudoTime() == 0; - return; - } - - if ( selectable_fd >= 0 ) - *read = selectable_fd; - } - -int PktSrc::ExtractNextPacket() - { - // Don't return any packets if processing is suspended (except for the - // very first packet which we need to set up times). - if ( net_is_processing_suspended() && first_timestamp ) - { - idle = true; - return 0; - } - - data = last_data = pcap_next(pd, &hdr); - - if ( data && (hdr.len == 0 || hdr.caplen == 0) ) - { - sessions->Weird("empty_pcap_header", &hdr, data); - return 0; - } - - if ( data ) - next_timestamp = hdr.ts.tv_sec + double(hdr.ts.tv_usec) / 1e6; - - if ( pseudo_realtime ) - current_wallclock = current_time(true); - - if ( ! first_timestamp ) - first_timestamp = next_timestamp; - - idle = (data == 0); - - if ( data ) - ++stats.received; - - // Source has gone dry. If it's a network interface, this just means - // it's timed out. If it's a file, though, then the file has been - // exhausted. - if ( ! data && ! IsLive() ) - { - closed = true; - - if ( pseudo_realtime && using_communication ) - { - if ( remote_trace_sync_interval ) - remote_serializer->SendFinalSyncPoint(); - else - remote_serializer->Terminate(); - } - } - - return data != 0; - } - -double PktSrc::NextTimestamp(double* local_network_time) - { - if ( ! data && ! ExtractNextPacket() ) - return -1.0; - - if ( pseudo_realtime ) - { - // Delay packet if necessary. - double packet_time = CheckPseudoTime(); - if ( packet_time ) - return packet_time; - - idle = true; - return -1.0; - } - - return next_timestamp; - } - -void PktSrc::ContinueAfterSuspend() - { - current_wallclock = current_time(true); - } - -double PktSrc::CurrentPacketWallClock() - { - // We stop time when we are suspended. - if ( net_is_processing_suspended() ) - current_wallclock = current_time(true); - - return current_wallclock; - } - -double PktSrc::CheckPseudoTime() - { - if ( ! data && ! ExtractNextPacket() ) - return 0; - - if ( ! current_timestamp ) - return bro_start_time; - - if ( remote_trace_sync_interval ) - { - if ( next_sync_point == 0 || next_timestamp >= next_sync_point ) - { - int n = remote_serializer->SendSyncPoint(); - next_sync_point = first_timestamp + - n * remote_trace_sync_interval; - remote_serializer->Log(RemoteSerializer::LogInfo, - fmt("stopping at packet %.6f, next sync-point at %.6f", - current_timestamp, next_sync_point)); - - return 0; - } - } - - double pseudo_time = next_timestamp - first_timestamp; - double ct = (current_time(true) - first_wallclock) * pseudo_realtime; - - return pseudo_time <= ct ? bro_start_time + pseudo_time : 0; - } - -void PktSrc::Process() - { - if ( ! data && ! ExtractNextPacket() ) - return; - - current_timestamp = next_timestamp; - - int pkt_hdr_size = hdr_size; - - // Unfortunately some packets on the link might have MPLS labels - // while others don't. That means we need to ask the link-layer if - // labels are in place. - bool have_mpls = false; - - int protocol = 0; - - switch ( datalink ) { - case DLT_NULL: - { - protocol = (data[3] << 24) + (data[2] << 16) + (data[1] << 8) + data[0]; - - // From the Wireshark Wiki: "AF_INET6, unfortunately, has - // different values in {NetBSD,OpenBSD,BSD/OS}, - // {FreeBSD,DragonFlyBSD}, and {Darwin/Mac OS X}, so an IPv6 - // packet might have a link-layer header with 24, 28, or 30 - // as the AF_ value." As we may be reading traces captured on - // platforms other than what we're running on, we accept them - // all here. - if ( protocol != AF_INET - && protocol != AF_INET6 - && protocol != 24 - && protocol != 28 - && protocol != 30 ) - { - sessions->Weird("non_ip_packet_in_null_transport", &hdr, data); - data = 0; - return; - } - - break; - } - - case DLT_EN10MB: - { - // Get protocol being carried from the ethernet frame. - protocol = (data[12] << 8) + data[13]; - - switch ( protocol ) - { - // MPLS carried over the ethernet frame. - case 0x8847: - have_mpls = true; - break; - - // VLAN carried over the ethernet frame. - case 0x8100: - data += get_link_header_size(datalink); - data += 4; // Skip the vlan header - pkt_hdr_size = 0; - - // Check for 802.1ah (Q-in-Q) containing IP. - // Only do a second layer of vlan tag - // stripping because there is no - // specification that allows for deeper - // nesting. - if ( ((data[2] << 8) + data[3]) == 0x0800 ) - data += 4; - - break; - - // PPPoE carried over the ethernet frame. - case 0x8864: - data += get_link_header_size(datalink); - protocol = (data[6] << 8) + data[7]; - data += 8; // Skip the PPPoE session and PPP header - pkt_hdr_size = 0; - - if ( protocol != 0x0021 && protocol != 0x0057 ) - { - // Neither IPv4 nor IPv6. - sessions->Weird("non_ip_packet_in_pppoe_encapsulation", &hdr, data); - data = 0; - return; - } - break; - } - - break; - } - - case DLT_PPP_SERIAL: - { - // Get PPP protocol. - protocol = (data[2] << 8) + data[3]; - - if ( protocol == 0x0281 ) - // MPLS Unicast - have_mpls = true; - - else if ( protocol != 0x0021 && protocol != 0x0057 ) - { - // Neither IPv4 nor IPv6. - sessions->Weird("non_ip_packet_in_ppp_encapsulation", &hdr, data); - data = 0; - return; - } - break; - } - } - - if ( have_mpls ) - { - // Remove the data link layer - data += get_link_header_size(datalink); - - // Denote a header size of zero before the IP header - pkt_hdr_size = 0; - - // Skip the MPLS label stack. - bool end_of_stack = false; - - while ( ! end_of_stack ) - { - end_of_stack = *(data + 2) & 0x01; - data += 4; - } - } - - if ( pseudo_realtime ) - { - current_pseudo = CheckPseudoTime(); - net_packet_arrival(current_pseudo, &hdr, data, pkt_hdr_size, this); - if ( ! first_wallclock ) - first_wallclock = current_time(true); - } - - else - net_packet_arrival(current_timestamp, &hdr, data, pkt_hdr_size, this); - - data = 0; - } - -bool PktSrc::GetCurrentPacket(const struct pcap_pkthdr** arg_hdr, - const u_char** arg_pkt) - { - if ( ! last_data ) - return false; - - *arg_hdr = &hdr; - *arg_pkt = last_data; - return true; - } - -int PktSrc::PrecompileFilter(int index, const char* filter) - { - // Compile filter. - BPF_Program* code = new BPF_Program(); - - if ( ! code->Compile(pd, filter, netmask, errbuf, sizeof(errbuf)) ) - { - delete code; - return 0; - } - - // Store it in hash. - HashKey* hash = new HashKey(HashKey(bro_int_t(index))); - BPF_Program* oldcode = filters.Lookup(hash); - if ( oldcode ) - delete oldcode; - - filters.Insert(hash, code); - delete hash; - - return 1; - } - -int PktSrc::SetFilter(int index) - { - // We don't want load-level filters for the secondary path. - if ( filter_type == TYPE_FILTER_SECONDARY && index > 0 ) - return 1; - - HashKey* hash = new HashKey(HashKey(bro_int_t(index))); - BPF_Program* code = filters.Lookup(hash); - delete hash; - - if ( ! code ) - { - safe_snprintf(errbuf, sizeof(errbuf), - "No precompiled pcap filter for index %d", - index); - return 0; - } - - if ( pcap_setfilter(pd, code->GetProgram()) < 0 ) - { - safe_snprintf(errbuf, sizeof(errbuf), - "pcap_setfilter(%d): %s", - index, pcap_geterr(pd)); - return 0; - } - -#ifndef HAVE_LINUX - // Linux doesn't clear counters when resetting filter. - stats.received = stats.dropped = stats.link = 0; -#endif - - return 1; - } - -void PktSrc::SetHdrSize() - { - int dl = pcap_datalink(pd); - hdr_size = get_link_header_size(dl); - - if ( hdr_size < 0 ) - { - safe_snprintf(errbuf, sizeof(errbuf), - "unknown data link type 0x%x", dl); - Close(); - } - - datalink = dl; - } - -void PktSrc::Close() - { - if ( pd ) - { - pcap_close(pd); - pd = 0; - closed = true; - } - } - -void PktSrc::AddSecondaryTablePrograms() - { - BPF_Program* program; - - loop_over_list(secondary_path->EventTable(), i) - { - SecondaryEvent* se = secondary_path->EventTable()[i]; - program = new BPF_Program(); - - if ( ! program->Compile(snaplen, datalink, se->Filter(), - netmask, errbuf, sizeof(errbuf)) ) - { - delete program; - Close(); - return; - } - - SecondaryProgram* sp = new SecondaryProgram(program, se); - program_list.append(sp); - } - } - -void PktSrc::Statistics(Stats* s) - { - if ( reading_traces ) - s->received = s->dropped = s->link = 0; - - else - { - struct pcap_stat pstat; - if ( pcap_stats(pd, &pstat) < 0 ) - { - reporter->Error("problem getting packet filter statistics: %s", - ErrorMsg()); - s->received = s->dropped = s->link = 0; - } - - else - { - s->dropped = pstat.ps_drop; - s->link = pstat.ps_recv; - } - } - - s->received = stats.received; - - if ( pseudo_realtime ) - s->dropped = 0; - - stats.dropped = s->dropped; - } - -PktInterfaceSrc::PktInterfaceSrc(const char* arg_interface, const char* filter, - PktSrc_Filter_Type ft) -: PktSrc() - { - char tmp_errbuf[PCAP_ERRBUF_SIZE]; - filter_type = ft; - - // Determine interface if not specified. - if ( ! arg_interface && ! (arg_interface = pcap_lookupdev(tmp_errbuf)) ) - { - safe_snprintf(errbuf, sizeof(errbuf), - "pcap_lookupdev: %s", tmp_errbuf); - return; - } - - interface = copy_string(arg_interface); - - // Determine network and netmask. - uint32 net; - if ( pcap_lookupnet(interface, &net, &netmask, tmp_errbuf) < 0 ) - { - // ### The lookup can fail if no address is assigned to - // the interface; and libpcap doesn't have any useful notion - // of error codes, just error strings - how bogus - so we - // just kludge around the error :-(. - // sprintf(errbuf, "pcap_lookupnet %s", tmp_errbuf); - // return; - net = 0; - netmask = 0xffffff00; - } - - // We use the smallest time-out possible to return almost immediately if - // no packets are available. (We can't use set_nonblocking() as it's - // broken on FreeBSD: even when select() indicates that we can read - // something, we may get nothing if the store buffer hasn't filled up - // yet.) - pd = pcap_open_live(interface, snaplen, 1, 1, tmp_errbuf); - - if ( ! pd ) - { - safe_snprintf(errbuf, sizeof(errbuf), - "pcap_open_live: %s", tmp_errbuf); - closed = true; - return; - } - - // ### This needs autoconf'ing. -#ifdef HAVE_PCAP_INT_H - reporter->Info("pcap bufsize = %d\n", ((struct pcap *) pd)->bufsize); -#endif - -#ifdef HAVE_LINUX - if ( pcap_setnonblock(pd, 1, tmp_errbuf) < 0 ) - { - safe_snprintf(errbuf, sizeof(errbuf), - "pcap_setnonblock: %s", tmp_errbuf); - pcap_close(pd); - closed = true; - return; - } -#endif - selectable_fd = pcap_fileno(pd); - - if ( PrecompileFilter(0, filter) && SetFilter(0) ) - { - SetHdrSize(); - - if ( closed ) - // Couldn't get header size. - return; - - reporter->Info("listening on %s, capture length %d bytes\n", interface, snaplen); - } - else - closed = true; - } - - -PktFileSrc::PktFileSrc(const char* arg_readfile, const char* filter, - PktSrc_Filter_Type ft) -: PktSrc() - { - readfile = copy_string(arg_readfile); - - filter_type = ft; - - pd = pcap_open_offline((char*) readfile, errbuf); - - if ( pd && PrecompileFilter(0, filter) && SetFilter(0) ) - { - SetHdrSize(); - - if ( closed ) - // Unknown link layer type. - return; - - // We don't put file sources into non-blocking mode as - // otherwise we would not be able to identify the EOF. - - selectable_fd = fileno(pcap_file(pd)); - - if ( selectable_fd < 0 ) - reporter->InternalError("OS does not support selectable pcap fd"); - } - else - closed = true; - } - - -SecondaryPath::SecondaryPath() - { - filter = 0; - - // Glue together the secondary filter, if exists. - Val* secondary_fv = internal_val("secondary_filters"); - if ( secondary_fv->AsTableVal()->Size() == 0 ) - return; - - int did_first = 0; - const TableEntryValPDict* v = secondary_fv->AsTable(); - IterCookie* c = v->InitForIteration(); - TableEntryVal* tv; - HashKey* h; - - while ( (tv = v->NextEntry(h, c)) ) - { - // Get the index values. - ListVal* index = - secondary_fv->AsTableVal()->RecoverIndex(h); - - const char* str = - index->Index(0)->Ref()->AsString()->CheckString(); - - if ( ++did_first == 1 ) - { - filter = copy_string(str); - } - else - { - if ( strlen(filter) > 0 ) - { - char* tmp_f = new char[strlen(str) + strlen(filter) + 32]; - if ( strlen(str) == 0 ) - sprintf(tmp_f, "%s", filter); - else - sprintf(tmp_f, "(%s) or (%s)", filter, str); - delete [] filter; - filter = tmp_f; - } - } - - // Build secondary_path event table item and link it. - SecondaryEvent* se = - new SecondaryEvent(index->Index(0)->Ref()->AsString()->CheckString(), - tv->Value()->AsFunc() ); - - event_list.append(se); - - delete h; - Unref(index); - } - } - -SecondaryPath::~SecondaryPath() - { - loop_over_list(event_list, i) - delete event_list[i]; - - delete [] filter; - } - - -SecondaryProgram::~SecondaryProgram() - { - delete program; - } - -PktDumper::PktDumper(const char* arg_filename, bool arg_append) - { - filename[0] = '\0'; - is_error = false; - append = arg_append; - dumper = 0; - open_time = 0.0; - - // We need a pcap_t with a reasonable link-layer type. We try to get it - // from the packet sources. If not available, we fall back to Ethernet. - // FIXME: Perhaps we should make this configurable? - int linktype = -1; - - if ( pkt_srcs.length() ) - linktype = pkt_srcs[0]->LinkType(); - - if ( linktype < 0 ) - linktype = DLT_EN10MB; - - pd = pcap_open_dead(linktype, 8192); - if ( ! pd ) - { - Error("error for pcap_open_dead"); - return; - } - - if ( arg_filename ) - Open(arg_filename); - } - -bool PktDumper::Open(const char* arg_filename) - { - if ( ! arg_filename && ! *filename ) - { - Error("no filename given"); - return false; - } - - if ( arg_filename ) - { - if ( dumper && streq(arg_filename, filename) ) - // Already open. - return true; - - safe_strncpy(filename, arg_filename, FNBUF_LEN); - } - - if ( dumper ) - Close(); - - struct stat s; - int exists = 0; - - if ( append ) - { - // See if output file already exists (and is non-empty). - exists = stat(filename, &s); ; - - if ( exists < 0 && errno != ENOENT ) - { - Error(fmt("can't stat file %s: %s", filename, strerror(errno))); - return false; - } - } - - if ( ! append || exists < 0 || s.st_size == 0 ) - { - // Open new file. - dumper = pcap_dump_open(pd, filename); - if ( ! dumper ) - { - Error(pcap_geterr(pd)); - return false; - } - } - - else - { - // Old file and we need to append, which, unfortunately, - // is not supported by libpcap. So, we have to hack a - // little bit, knowing that pcap_dumpter_t is, in fact, - // a FILE ... :-( - dumper = (pcap_dumper_t*) fopen(filename, "a"); - if ( ! dumper ) - { - Error(fmt("can't open dump %s: %s", filename, strerror(errno))); - return false; - } - } - - open_time = network_time; - is_error = false; - return true; - } - -bool PktDumper::Close() - { - if ( dumper ) - { - pcap_dump_close(dumper); - dumper = 0; - is_error = false; - } - - return true; - } - -bool PktDumper::Dump(const struct pcap_pkthdr* hdr, const u_char* pkt) - { - if ( ! dumper ) - return false; - - if ( ! open_time ) - open_time = network_time; - - pcap_dump((u_char*) dumper, hdr, pkt); - - return true; - } - -void PktDumper::Error(const char* errstr) - { - safe_strncpy(errbuf, errstr, sizeof(errbuf)); - is_error = true; - } - -int get_link_header_size(int dl) - { - switch ( dl ) { - case DLT_NULL: - return 4; - - case DLT_EN10MB: - return 14; - - case DLT_FDDI: - return 13 + 8; // fddi_header + LLC - -#ifdef DLT_LINUX_SLL - case DLT_LINUX_SLL: - return 16; -#endif - - case DLT_PPP_SERIAL: // PPP_SERIAL - return 4; - - case DLT_RAW: - return 0; - } - - return -1; - } diff --git a/src/PktSrc.h b/src/PktSrc.h deleted file mode 100644 index 70eef4dd00..0000000000 --- a/src/PktSrc.h +++ /dev/null @@ -1,258 +0,0 @@ -// See the file "COPYING" in the main distribution directory for copyright. - -#ifndef pktsrc_h -#define pktsrc_h - -#include "Dict.h" -#include "Expr.h" -#include "BPF_Program.h" -#include "IOSource.h" -#include "RemoteSerializer.h" - -#define BRO_PCAP_ERRBUF_SIZE PCAP_ERRBUF_SIZE + 256 - -extern "C" { -#include -} - -declare(PDict,BPF_Program); - -// Whether a PktSrc object is used by the normal filter structure or the -// secondary-path structure. -typedef enum { - TYPE_FILTER_NORMAL, // the normal filter - TYPE_FILTER_SECONDARY, // the secondary-path filter -} PktSrc_Filter_Type; - - -// {filter,event} tuples conforming the secondary path. -class SecondaryEvent { -public: - SecondaryEvent(const char* arg_filter, Func* arg_event) - { - filter = arg_filter; - event = arg_event; - } - - const char* Filter() { return filter; } - Func* Event() { return event; } - -private: - const char* filter; - Func* event; -}; - -declare(PList,SecondaryEvent); -typedef PList(SecondaryEvent) secondary_event_list; - - - -class SecondaryPath { -public: - SecondaryPath(); - ~SecondaryPath(); - - secondary_event_list& EventTable() { return event_list; } - const char* Filter() { return filter; } - -private: - secondary_event_list event_list; - // OR'ed union of all SecondaryEvent filters - char* filter; -}; - -// Main secondary-path object. -extern SecondaryPath* secondary_path; - - -// {program, {filter,event}} tuple table. -class SecondaryProgram { -public: - SecondaryProgram(BPF_Program* arg_program, SecondaryEvent* arg_event) - { - program = arg_program; - event = arg_event; - } - - ~SecondaryProgram(); - - BPF_Program* Program() { return program; } - SecondaryEvent* Event() { return event; } - -private: - // Associated program. - BPF_Program *program; - - // Event that is run in case the program is matched. - SecondaryEvent* event; -}; - -declare(PList,SecondaryProgram); -typedef PList(SecondaryProgram) secondary_program_list; - - - -class PktSrc : public IOSource { -public: - ~PktSrc(); - - // IOSource interface - bool IsReady(); - void GetFds(int* read, int* write, int* except); - double NextTimestamp(double* local_network_time); - void Process(); - const char* Tag() { return "PktSrc"; } - - const char* ErrorMsg() const { return errbuf; } - void ClearErrorMsg() { *errbuf ='\0'; } - - // Returns the packet last processed; false if there is no - // current packet available. - bool GetCurrentPacket(const pcap_pkthdr** hdr, const u_char** pkt); - - int HdrSize() const { return hdr_size; } - int DataLink() const { return datalink; } - - void ConsumePacket() { data = 0; } - - int IsLive() const { return interface != 0; } - - pcap_t* PcapHandle() const { return pd; } - int LinkType() const { return pcap_datalink(pd); } - - const char* ReadFile() const { return readfile; } - const char* Interface() const { return interface; } - PktSrc_Filter_Type FilterType() const { return filter_type; } - void AddSecondaryTablePrograms(); - const secondary_program_list& ProgramTable() const - { return program_list; } - - // Signal packet source that processing was suspended and is now going - // to be continued. - void ContinueAfterSuspend(); - - // Only valid in pseudo-realtime mode. - double CurrentPacketTimestamp() { return current_pseudo; } - double CurrentPacketWallClock(); - - struct Stats { - unsigned int received; // pkts received (w/o drops) - unsigned int dropped; // pkts dropped - unsigned int link; // total packets on link - // (not always not available) - }; - - virtual void Statistics(Stats* stats); - - // Precompiles a filter and associates the given index with it. - // Returns true on success, 0 if a problem occurred. - virtual int PrecompileFilter(int index, const char* filter); - - // Activates the filter with the given index. - // Returns true on success, 0 if a problem occurred. - virtual int SetFilter(int index); - -protected: - PktSrc(); - - static const int PCAP_TIMEOUT = 20; - - void SetHdrSize(); - - virtual void Close(); - - // Returns 1 on success, 0 on time-out/gone dry. - virtual int ExtractNextPacket(); - - // Checks if the current packet has a pseudo-time <= current_time. - // If yes, returns pseudo-time, otherwise 0. - double CheckPseudoTime(); - - double current_timestamp; - double next_timestamp; - - // Only set in pseudo-realtime mode. - double first_timestamp; - double first_wallclock; - double current_wallclock; - double current_pseudo; - - struct pcap_pkthdr hdr; - const u_char* data; // contents of current packet - const u_char* last_data; // same, but unaffected by consuming - int hdr_size; - int datalink; - double next_sync_point; // For trace synchronziation in pseudo-realtime - - char* interface; // nil if not reading from an interface - char* readfile; // nil if not reading from a file - - pcap_t* pd; - int selectable_fd; - uint32 netmask; - char errbuf[BRO_PCAP_ERRBUF_SIZE]; - - Stats stats; - - PDict(BPF_Program) filters; // precompiled filters - - PktSrc_Filter_Type filter_type; // normal path or secondary path - secondary_program_list program_list; -}; - -class PktInterfaceSrc : public PktSrc { -public: - PktInterfaceSrc(const char* interface, const char* filter, - PktSrc_Filter_Type ft=TYPE_FILTER_NORMAL); -}; - -class PktFileSrc : public PktSrc { -public: - PktFileSrc(const char* readfile, const char* filter, - PktSrc_Filter_Type ft=TYPE_FILTER_NORMAL); -}; - - -extern int get_link_header_size(int dl); - -class PktDumper { -public: - PktDumper(const char* file = 0, bool append = false); - ~PktDumper() { Close(); } - - bool Open(const char* file = 0); - bool Close(); - bool Dump(const struct pcap_pkthdr* hdr, const u_char* pkt); - - pcap_dumper_t* PcapDumper() { return dumper; } - - const char* FileName() const { return filename; } - bool IsError() const { return is_error; } - const char* ErrorMsg() const { return errbuf; } - - // This heuristic will horribly fail if we're using packets - // with different link layers. (If we can't derive a reasonable value - // from the packet sources, our fall-back is Ethernet.) - int HdrSize() const - { return get_link_header_size(pcap_datalink(pd)); } - - // Network time when dump file was opened. - double OpenTime() const { return open_time; } - -private: - void InitPd(); - void Error(const char* str); - - static const int FNBUF_LEN = 1024; - char filename[FNBUF_LEN]; - - bool append; - pcap_dumper_t* dumper; - pcap_t* pd; - double open_time; - - bool is_error; - char errbuf[BRO_PCAP_ERRBUF_SIZE]; -}; - -#endif diff --git a/src/RemoteSerializer.cc b/src/RemoteSerializer.cc index c8cf03667b..b0db8fafe8 100644 --- a/src/RemoteSerializer.cc +++ b/src/RemoteSerializer.cc @@ -188,10 +188,11 @@ #include "File.h" #include "Conn.h" #include "Reporter.h" -#include "threading/SerialTypes.h" -#include "logging/Manager.h" #include "IPAddr.h" #include "bro_inet_ntop.h" +#include "threading/SerialTypes.h" +#include "logging/Manager.h" +#include "iosource/Manager.h" extern "C" { #include "setsignal.h" @@ -284,10 +285,10 @@ struct ping_args { \ if ( ! c ) \ { \ - idle = io->IsIdle();\ + SetIdle(io->IsIdle());\ return true; \ } \ - idle = false; \ + SetIdle(false); \ } static const char* msgToStr(int msg) @@ -536,7 +537,6 @@ RemoteSerializer::RemoteSerializer() current_sync_point = 0; syncing_times = false; io = 0; - closed = false; terminating = false; in_sync = 0; last_flush = 0; @@ -574,7 +574,7 @@ void RemoteSerializer::Init() Fork(); - io_sources.Register(this); + iosource_mgr->Register(this); Log(LogInfo, fmt("communication started, parent pid is %d, child pid is %d", getpid(), child_pid)); initialized = 1; @@ -1278,7 +1278,7 @@ bool RemoteSerializer::Listen(const IPAddr& ip, uint16 port, bool expect_ssl, return false; listening = true; - closed = false; + SetClosed(false); return true; } @@ -1347,7 +1347,7 @@ bool RemoteSerializer::StopListening() return false; listening = false; - closed = ! IsActive(); + SetClosed(! IsActive()); return true; } @@ -1385,7 +1385,7 @@ double RemoteSerializer::NextTimestamp(double* local_network_time) if ( received_logs > 0 ) { // If we processed logs last time, assume there's more. - idle = false; + SetIdle(false); received_logs = 0; return timer_mgr->Time(); } @@ -1400,7 +1400,7 @@ double RemoteSerializer::NextTimestamp(double* local_network_time) pt = timer_mgr->Time(); if ( packets.length() ) - idle = false; + SetIdle(false); if ( et >= 0 && (et < pt || pt < 0) ) return et; @@ -1479,7 +1479,7 @@ void RemoteSerializer::Process() } if ( packets.length() ) - idle = false; + SetIdle(false); } void RemoteSerializer::Finish() @@ -1511,7 +1511,7 @@ bool RemoteSerializer::Poll(bool may_block) } io->Flush(); - idle = false; + SetIdle(false); switch ( msgstate ) { case TYPE: @@ -1695,7 +1695,7 @@ bool RemoteSerializer::DoMessage() case MSG_TERMINATE: assert(terminating); - io_sources.Terminate(); + iosource_mgr->Terminate(); return true; case MSG_REMOTE_PRINT: @@ -1885,7 +1885,7 @@ void RemoteSerializer::RemovePeer(Peer* peer) delete peer->cache_out; delete peer; - closed = ! IsActive(); + SetClosed(! IsActive()); if ( in_sync == peer ) in_sync = 0; @@ -2850,7 +2850,7 @@ void RemoteSerializer::GotEvent(const char* name, double time, BufferedEvent* e = new BufferedEvent; // Our time, not the time when the event was generated. - e->time = pkt_srcs.length() ? + e->time = iosource_mgr->GetPktSrcs().size() ? time_t(network_time) : time_t(timer_mgr->Time()); e->src = current_peer->id; @@ -3094,7 +3094,7 @@ RecordVal* RemoteSerializer::GetPeerVal(PeerID id) void RemoteSerializer::ChildDied() { Log(LogError, "child died"); - closed = true; + SetClosed(true); child_pid = 0; // Shut down the main process as well. @@ -3188,7 +3188,7 @@ void RemoteSerializer::FatalError(const char* msg) Log(LogError, msg); reporter->Error("%s", msg); - closed = true; + SetClosed(true); if ( kill(child_pid, SIGQUIT) < 0 ) reporter->Warning("warning: cannot kill child pid %d, %s", child_pid, strerror(errno)); diff --git a/src/RemoteSerializer.h b/src/RemoteSerializer.h index 5ff7fff8d6..f8b306f002 100644 --- a/src/RemoteSerializer.h +++ b/src/RemoteSerializer.h @@ -6,7 +6,7 @@ #include "Dict.h" #include "List.h" #include "Serializer.h" -#include "IOSource.h" +#include "iosource/IOSource.h" #include "Stats.h" #include "File.h" #include "logging/WriterBackend.h" @@ -22,7 +22,7 @@ namespace threading { } // This class handles the communication done in Bro's main loop. -class RemoteSerializer : public Serializer, public IOSource { +class RemoteSerializer : public Serializer, public iosource::IOSource { public: RemoteSerializer(); virtual ~RemoteSerializer(); diff --git a/src/Serializer.cc b/src/Serializer.cc index 156ad67f2e..74740497a1 100644 --- a/src/Serializer.cc +++ b/src/Serializer.cc @@ -19,6 +19,7 @@ #include "Conn.h" #include "Timer.h" #include "RemoteSerializer.h" +#include "iosource/Manager.h" Serializer::Serializer(SerializationFormat* arg_format) { @@ -1045,7 +1046,7 @@ EventPlayer::EventPlayer(const char* file) Error(fmt("event replayer: cannot open %s", file)); if ( ReadHeader() ) - io_sources.Register(this); + iosource_mgr->Register(this); } EventPlayer::~EventPlayer() @@ -1085,7 +1086,7 @@ double EventPlayer::NextTimestamp(double* local_network_time) { UnserialInfo info(this); Unserialize(&info); - closed = io->Eof(); + SetClosed(io->Eof()); } if ( ! ne_time ) @@ -1142,7 +1143,7 @@ bool Packet::Serialize(SerialInfo* info) const static BroFile* profiling_output = 0; #ifdef DEBUG -static PktDumper* dump = 0; +static iosource::PktDumper* dump = 0; #endif Packet* Packet::Unserialize(UnserialInfo* info) @@ -1188,7 +1189,7 @@ Packet* Packet::Unserialize(UnserialInfo* info) p->hdr = hdr; p->pkt = (u_char*) pkt; p->tag = tag; - p->hdr_size = get_link_header_size(p->link_type); + p->hdr_size = iosource::PktSrc::GetLinkHeaderSize(p->link_type); delete [] tag; @@ -1213,9 +1214,15 @@ Packet* Packet::Unserialize(UnserialInfo* info) if ( debug_logger.IsEnabled(DBG_TM) ) { if ( ! dump ) - dump = new PktDumper("tm.pcap"); + dump = iosource_mgr->OpenPktDumper("tm.pcap", true); - dump->Dump(p->hdr, p->pkt); + if ( dump ) + { + iosource::PktDumper::Packet dp; + dp.hdr = p->hdr; + dp.data = p->pkt; + dump->Record(&dp); + } } #endif diff --git a/src/Serializer.h b/src/Serializer.h index af4878ccf5..3be2da5134 100644 --- a/src/Serializer.h +++ b/src/Serializer.h @@ -15,7 +15,7 @@ #include "SerialInfo.h" #include "IP.h" #include "Timer.h" -#include "IOSource.h" +#include "iosource/IOSource.h" #include "Reporter.h" class SerializationCache; @@ -350,7 +350,7 @@ public: }; // Plays a file of events back. -class EventPlayer : public FileSerializer, public IOSource { +class EventPlayer : public FileSerializer, public iosource::IOSource { public: EventPlayer(const char* file); virtual ~EventPlayer(); diff --git a/src/Sessions.cc b/src/Sessions.cc index acc306d277..c84c677db4 100644 --- a/src/Sessions.cc +++ b/src/Sessions.cc @@ -168,7 +168,7 @@ void NetSessions::Done() void NetSessions::DispatchPacket(double t, const struct pcap_pkthdr* hdr, const u_char* pkt, int hdr_size, - PktSrc* src_ps, PacketSortElement* pkt_elem) + iosource::PktSrc* src_ps, PacketSortElement* pkt_elem) { const struct ip* ip_hdr = 0; const u_char* ip_data = 0; @@ -185,10 +185,14 @@ void NetSessions::DispatchPacket(double t, const struct pcap_pkthdr* hdr, // Blanket encapsulation hdr_size += encap_hdr_size; +#if 0 if ( src_ps->FilterType() == TYPE_FILTER_NORMAL ) NextPacket(t, hdr, pkt, hdr_size, pkt_elem); else NextPacketSecondary(t, hdr, pkt, hdr_size, src_ps); +#else + NextPacket(t, hdr, pkt, hdr_size, pkt_elem); +#endif } void NetSessions::NextPacket(double t, const struct pcap_pkthdr* hdr, @@ -278,7 +282,7 @@ void NetSessions::NextPacket(double t, const struct pcap_pkthdr* hdr, void NetSessions::NextPacketSecondary(double /* t */, const struct pcap_pkthdr* hdr, const u_char* const pkt, int hdr_size, - const PktSrc* src_ps) + const iosource::PktSrc* src_ps) { SegmentProfiler(segment_logger, "processing-secondary-packet"); @@ -291,6 +295,7 @@ void NetSessions::NextPacketSecondary(double /* t */, const struct pcap_pkthdr* return; } +#if 0 const struct ip* ip = (const struct ip*) (pkt + hdr_size); if ( ip->ip_v == 4 ) { @@ -321,6 +326,7 @@ void NetSessions::NextPacketSecondary(double /* t */, const struct pcap_pkthdr* delete args; } } +#endif } int NetSessions::CheckConnectionTag(Connection* conn) @@ -1341,14 +1347,24 @@ void NetSessions::DumpPacket(const struct pcap_pkthdr* hdr, return; if ( len == 0 ) - pkt_dumper->Dump(hdr, pkt); + { + iosource::PktDumper::Packet p; + p.hdr = hdr; + p.data = pkt; + pkt_dumper->Record(&p); + } + else { struct pcap_pkthdr h = *hdr; h.caplen = len; if ( h.caplen > hdr->caplen ) reporter->InternalError("bad modified caplen"); - pkt_dumper->Dump(&h, pkt); + + iosource::PktDumper::Packet p; + p.hdr = &h; + p.data = pkt; + pkt_dumper->Record(&p); } } diff --git a/src/Sessions.h b/src/Sessions.h index 1788541f45..4f12bd1240 100644 --- a/src/Sessions.h +++ b/src/Sessions.h @@ -74,7 +74,7 @@ public: // employing the packet sorter first. void DispatchPacket(double t, const struct pcap_pkthdr* hdr, const u_char* const pkt, int hdr_size, - PktSrc* src_ps, PacketSortElement* pkt_elem); + iosource::PktSrc* src_ps, PacketSortElement* pkt_elem); void Done(); // call to drain events before destructing @@ -225,7 +225,7 @@ protected: void NextPacketSecondary(double t, const struct pcap_pkthdr* hdr, const u_char* const pkt, int hdr_size, - const PktSrc* src_ps); + const iosource::PktSrc* src_ps); // Record the given packet (if a dumper is active). If len=0 // then the whole packet is recorded, otherwise just the first diff --git a/src/bro.bif b/src/bro.bif index 24dff3c77c..2b94307143 100644 --- a/src/bro.bif +++ b/src/bro.bif @@ -21,6 +21,7 @@ #include "IPAddr.h" #include "util.h" #include "file_analysis/Manager.h" +#include "iosource/Manager.h" using namespace std; @@ -33,7 +34,7 @@ TableType* var_sizes; // and hence it's declared in NetVar.{h,cc}. extern RecordType* gap_info; -static PktDumper* addl_pkt_dumper = 0; +static iosource::PktDumper* addl_pkt_dumper = 0; bro_int_t parse_int(const char*& fmt) { @@ -1657,11 +1658,14 @@ function net_stats%(%): NetStats unsigned int drop = 0; unsigned int link = 0; - loop_over_list(pkt_srcs, i) - { - PktSrc* ps = pkt_srcs[i]; + const iosource::Manager::PktSrcList& pkt_srcs(iosource_mgr->GetPktSrcs()); - struct PktSrc::Stats stat; + for ( iosource::Manager::PktSrcList::const_iterator i = pkt_srcs.begin(); + i != pkt_srcs.end(); i++ ) + { + iosource::PktSrc* ps = *i; + + struct iosource::PktSrc::Stats stat; ps->Statistics(&stat); recv += stat.received; drop += stat.dropped; @@ -3206,10 +3210,15 @@ function dump_current_packet%(file_name: string%) : bool return new Val(0, TYPE_BOOL); if ( ! addl_pkt_dumper ) - addl_pkt_dumper = new PktDumper(0, true); + addl_pkt_dumper = iosource_mgr->OpenPktDumper(file_name->CheckString(), true); - addl_pkt_dumper->Open(file_name->CheckString()); - addl_pkt_dumper->Dump(hdr, pkt); + if ( addl_pkt_dumper ) + { + iosource::PktDumper::Packet p; + p.hdr = hdr; + p.data = pkt; + addl_pkt_dumper->Record(&p); + } return new Val(! addl_pkt_dumper->IsError(), TYPE_BOOL); %} @@ -3266,10 +3275,15 @@ function dump_packet%(pkt: pcap_packet, file_name: string%) : bool hdr.len = (*pkt_vl)[3]->AsCount(); if ( ! addl_pkt_dumper ) - addl_pkt_dumper = new PktDumper(0, true); + addl_pkt_dumper = iosource_mgr->OpenPktDumper(file_name->CheckString(), true); - addl_pkt_dumper->Open(file_name->CheckString()); - addl_pkt_dumper->Dump(&hdr, (*pkt_vl)[4]->AsString()->Bytes()); + if ( addl_pkt_dumper ) + { + iosource::PktDumper::Packet p; + p.hdr = &hdr; + p.data = (*pkt_vl)[4]->AsString()->Bytes(); + addl_pkt_dumper->Record(&p); + } return new Val(addl_pkt_dumper->IsError(), TYPE_BOOL); %} @@ -4030,14 +4044,14 @@ function rotate_file_by_name%(f: string%): rotate_info bool is_addl_pkt_dumper = false; // Special case: one of current dump files. - if ( pkt_dumper && streq(pkt_dumper->FileName(), f->CheckString()) ) + if ( pkt_dumper && streq(pkt_dumper->Path().c_str(), f->CheckString()) ) { is_pkt_dumper = true; pkt_dumper->Close(); } if ( addl_pkt_dumper && - streq(addl_pkt_dumper->FileName(), f->CheckString()) ) + streq(addl_pkt_dumper->Path().c_str(), f->CheckString()) ) { is_addl_pkt_dumper = true; addl_pkt_dumper->Close(); @@ -4156,15 +4170,18 @@ function precompile_pcap_filter%(id: PcapFilterID, s: string%): bool %{ bool success = true; - loop_over_list(pkt_srcs, i) - { - pkt_srcs[i]->ClearErrorMsg(); + const iosource::Manager::PktSrcList& pkt_srcs(iosource_mgr->GetPktSrcs()); - if ( ! pkt_srcs[i]->PrecompileFilter(id->ForceAsInt(), + for ( iosource::Manager::PktSrcList::const_iterator i = pkt_srcs.begin(); + i != pkt_srcs.end(); i++ ) + { + iosource::PktSrc* ps = *i; + + if ( ! ps->PrecompileFilter(id->ForceAsInt(), s->CheckString()) ) { reporter->Error("precompile_pcap_filter: %s", - pkt_srcs[i]->ErrorMsg()); + ps->ErrorMsg()); success = false; } } @@ -4194,11 +4211,14 @@ function install_pcap_filter%(id: PcapFilterID%): bool %{ bool success = true; - loop_over_list(pkt_srcs, i) - { - pkt_srcs[i]->ClearErrorMsg(); + const iosource::Manager::PktSrcList& pkt_srcs(iosource_mgr->GetPktSrcs()); - if ( ! pkt_srcs[i]->SetFilter(id->ForceAsInt()) ) + for ( iosource::Manager::PktSrcList::const_iterator i = pkt_srcs.begin(); + i != pkt_srcs.end(); i++ ) + { + iosource::PktSrc* ps = *i; + + if ( ! ps->SetFilter(id->ForceAsInt()) ) success = false; } @@ -4221,9 +4241,14 @@ function install_pcap_filter%(id: PcapFilterID%): bool ## uninstall_dst_net_filter function pcap_error%(%): string %{ - loop_over_list(pkt_srcs, i) + const iosource::Manager::PktSrcList& pkt_srcs(iosource_mgr->GetPktSrcs()); + + for ( iosource::Manager::PktSrcList::const_iterator i = pkt_srcs.begin(); + i != pkt_srcs.end(); i++ ) { - const char* err = pkt_srcs[i]->ErrorMsg(); + iosource::PktSrc* ps = *i; + + const char* err = ps->ErrorMsg(); if ( *err ) return new StringVal(err); } diff --git a/src/iosource/CMakeLists.txt b/src/iosource/CMakeLists.txt new file mode 100644 index 0000000000..a9246e8de9 --- /dev/null +++ b/src/iosource/CMakeLists.txt @@ -0,0 +1,23 @@ + +include(BroSubdir) + +include_directories(BEFORE + ${CMAKE_CURRENT_SOURCE_DIR} + ${CMAKE_CURRENT_BINARY_DIR} +) + +add_subdirectory(pktsrc) + + +set(iosource_SRCS + Component.cc + Manager.cc + + pktsrc/Component.cc + pktsrc/PktDumper.cc + pktsrc/PktSrc.cc +) + +bro_add_subdir_library(iosource ${iosource_SRCS}) +add_dependencies(bro_iosource generate_outputs) + diff --git a/src/iosource/Component.cc b/src/iosource/Component.cc new file mode 100644 index 0000000000..5f916c0a11 --- /dev/null +++ b/src/iosource/Component.cc @@ -0,0 +1,44 @@ + +#include "Component.h" + +#include "Desc.h" + +using namespace iosource; + +Component::Component(const std::string& arg_name) + : plugin::Component(plugin::component::IOSOURCE) + { + name = arg_name; + } + +Component::Component(plugin::component::Type type, const std::string& arg_name) + : plugin::Component(type) + { + name = arg_name; + } + +Component::Component(const Component& other) + : plugin::Component(other) + { + name = other.name; + } + +Component::~Component() + { + } + +void Component::Describe(ODesc* d) const + { + plugin::Component::Describe(d); + d->Add(name); + } + +Component& Component::operator=(const Component& other) + { + plugin::Component::operator=(other); + + if ( &other != this ) + name = other.name; + + return *this; + } diff --git a/src/iosource/Component.h b/src/iosource/Component.h new file mode 100644 index 0000000000..b56eeb038c --- /dev/null +++ b/src/iosource/Component.h @@ -0,0 +1,56 @@ +// See the file "COPYING" in the main distribution directory for copyright. + +#ifndef IOSOURCE_PLUGIN_COMPONENT_H +#define IOSOURCE_PLUGIN_COMPONENT_H + +#include "plugin/Component.h" + +namespace iosource { + +class IOSource; + +/** + * Component description for plugins providing IOSources. + */ +class Component : public plugin::Component { +public: + typedef IOSource* (*factory_callback)(); + + /** + * XXX + */ + Component(const std::string& name); + + /** + * Copy constructor. + */ + Component(const Component& other); + + /** + * Destructor. + */ + ~Component(); + + /** + * XXX + */ + virtual const char* Name() const { return name.c_str(); } + + /** + * Generates a human-readable description of the component. This goes + * into the output of \c "bro -NN". + */ + virtual void Describe(ODesc* d) const; + + Component& operator=(const Component& other); + +protected: + Component(plugin::component::Type type, const std::string& name); + +private: + std::string name; +}; + +} + +#endif diff --git a/src/IOSource.h b/src/iosource/IOSource.h similarity index 51% rename from src/IOSource.h rename to src/iosource/IOSource.h index db50bbd2a9..3419152a9a 100644 --- a/src/IOSource.h +++ b/src/iosource/IOSource.h @@ -1,13 +1,17 @@ -// Interface for classes providing/consuming data during Bro's main loop. +// See the file "COPYING" in the main distribution directory for copyright. -#ifndef iosource_h -#define iosource_h +#ifndef IOSOURCE_IOSOURCE_H +#define IOSOURCE_IOSOURCE_H + +#include -#include #include "Timer.h" -using namespace std; +namespace iosource { +/** + * Interface class for components providing/consuming data inside Bro's main loop. + */ class IOSource { public: IOSource() { idle = closed = false; } @@ -20,6 +24,12 @@ public: // Otherwise, source may be removed. bool IsOpen() const { return ! closed; } + // XXX + virtual void Init() { } + + // XXX + virtual void Done() { } + // Returns select'able fds (leaves args untouched if we don't have // selectable fds). virtual void GetFds(int* read, int* write, int* except) = 0; @@ -46,58 +56,18 @@ public: protected: // Derived classed are to set this to true if they have gone dry // temporarily. - bool idle; + void SetIdle(bool is_idle) { idle = is_idle; } + // Derived classed are to set this to true if they have gone dry - // permanently. + // temporarily. + void SetClosed(bool is_closed) { closed = is_closed; } + +private: + bool idle; bool closed; }; -class IOSourceRegistry { -public: - IOSourceRegistry() { call_count = 0; dont_counts = 0; } - ~IOSourceRegistry(); - - // If dont_count is true, this source does not contribute to the - // number of IOSources returned by Size(). The effect is that - // if all sources but the non-counting ones have gone dry, - // processing will shut down. - void Register(IOSource* src, bool dont_count = false); - - // This may block for some time. - IOSource* FindSoonest(double* ts); - - int Size() const { return sources.size() - dont_counts; } - - // Terminate IOSource processing immediately by removing all - // sources (and therefore returning a Size() of zero). - void Terminate() { RemoveAll(); } - -protected: - // When looking for a source with something to process, - // every SELECT_FREQUENCY calls we will go ahead and - // block on a select(). - static const int SELECT_FREQUENCY = 25; - - // Microseconds to wait in an empty select if no source is ready. - static const int SELECT_TIMEOUT = 50; - - void RemoveAll(); - - unsigned int call_count; - int dont_counts; - - struct Source { - IOSource* src; - int fd_read; - int fd_write; - int fd_except; - }; - - typedef list SourceList; - SourceList sources; -}; - -extern IOSourceRegistry io_sources; +} #endif diff --git a/src/IOSource.cc b/src/iosource/Manager.cc similarity index 52% rename from src/IOSource.cc rename to src/iosource/Manager.cc index d47007caad..9c14330868 100644 --- a/src/IOSource.cc +++ b/src/iosource/Manager.cc @@ -1,3 +1,4 @@ + #include #include #include @@ -5,26 +6,37 @@ #include -#include "util.h" +#include "Manager.h" #include "IOSource.h" +#include "pktsrc/PktSrc.h" +#include "pktsrc/PktDumper.h" +#include "pktsrc/Component.h" +#include "plugin/Manager.h" -IOSourceRegistry io_sources; +#include "util.h" -IOSourceRegistry::~IOSourceRegistry() +#define DEFAULT_PREFIX "pcap" + +using namespace iosource; + +Manager::~Manager() { for ( SourceList::iterator i = sources.begin(); i != sources.end(); ++i ) + { + (*i)->src->Done(); delete *i; + } sources.clear(); } -void IOSourceRegistry::RemoveAll() +void Manager::RemoveAll() { // We're cheating a bit here ... dont_counts = sources.size(); } -IOSource* IOSourceRegistry::FindSoonest(double* ts) +IOSource* Manager::FindSoonest(double* ts) { // Remove sources which have gone dry. For simplicity, we only // remove at most one each time. @@ -101,9 +113,9 @@ IOSource* IOSourceRegistry::FindSoonest(double* ts) FD_SET(src->fd_write, &fd_write); FD_SET(src->fd_except, &fd_except); - maxx = max(src->fd_read, maxx); - maxx = max(src->fd_write, maxx); - maxx = max(src->fd_except, maxx); + maxx = std::max(src->fd_read, maxx); + maxx = std::max(src->fd_write, maxx); + maxx = std::max(src->fd_except, maxx); } // We can't block indefinitely even when all sources are dry: @@ -166,11 +178,130 @@ finished: return soonest_src; } -void IOSourceRegistry::Register(IOSource* src, bool dont_count) +void Manager::Register(IOSource* src, bool dont_count) { + src->Init(); Source* s = new Source; s->src = src; if ( dont_count ) ++dont_counts; - return sources.push_back(s); + + sources.push_back(s); + } + +void Manager::Register(PktSrc* src) + { + pkt_srcs.push_back(src); + Register(src, false); + } + +static std::pair split_prefix(std::string path) + { + // See if the path comes with a prefix telling us which type of + // PktSrc to use. If not, choose default. + std::string prefix; + + std::string::size_type i = path.find(":"); + if ( i != std::string::npos ) + { + prefix = path.substr(0, i); + path = path.substr(++i, std::string::npos); + } + + else + prefix= DEFAULT_PREFIX; + + return std::make_pair(prefix, path); + } + +PktSrc* Manager::OpenPktSrc(const std::string& path, const std::string& filter, bool is_live) + { + std::pair t = split_prefix(path); + std::string prefix = t.first; + std::string npath = t.second; + + // Find the component providing packet sources of the requested prefix. + + pktsrc::SourceComponent* component = 0; + + std::list all_components = plugin_mgr->Components(); + + for ( std::list::const_iterator i = all_components.begin(); + i != all_components.end(); i++ ) + { + pktsrc::SourceComponent* c = *i; + + if ( c->Prefix() == prefix && + (( is_live && c->DoesLive() ) || + (! is_live && c->DoesTrace())) ) + { + component = c; + break; + } + } + + + if ( ! component ) + reporter->FatalError("type of packet source '%s' not recognized", prefix.c_str()); + + // Instantiate packet source. + + PktSrc* ps = (*component->Factory())(path, filter, is_live); + + if ( ! (ps && ps->IsOpen()) ) + { + string type = (is_live ? "interface" : "trace file"); + string pserr = ps->ErrorMsg() ? (string(" - ") + ps->ErrorMsg()) : ""; + + reporter->FatalError("%s: problem with %s %s%s\n", + prog, path.c_str(), type.c_str(), pserr.c_str()); + } + + DBG_LOG(DBG_PKTIO, "Created packet source of type %s for %s\n", component->Name(), path.c_str()); + + Register(ps); + return ps; + } + + +PktDumper* Manager::OpenPktDumper(const string& path, bool append) + { + std::pair t = split_prefix(path); + std::string prefix = t.first; + std::string npath = t.second; + + // Find the component providing packet dumpers of the requested prefix. + + pktsrc::DumperComponent* component = 0; + + std::list all_components = plugin_mgr->Components(); + + for ( std::list::const_iterator i = all_components.begin(); + i != all_components.end(); i++ ) + { + if ( (*i)->Prefix() == prefix ) + { + component = (*i); + break; + } + } + + if ( ! component ) + reporter->FatalError("type of packet dumper '%s' not recognized", prefix.c_str()); + + // Instantiate packet dumper. + + PktDumper* pd = (*component->Factory())(path, append); + + if ( ! (pd && pd->IsOpen()) ) + { + string pderr = pd->ErrorMsg().size() ? (string(" - ") + pd->ErrorMsg()) : ""; + + reporter->FatalError("%s: can't open write file \"%s\"%s\n", + prog, path.c_str(), pderr.c_str()); + } + + DBG_LOG(DBG_PKTIO, "Created packer dumper of type %s for %s\n", component->Name(), path.c_str()); + + return pd; } diff --git a/src/iosource/Manager.h b/src/iosource/Manager.h new file mode 100644 index 0000000000..5a3a58d798 --- /dev/null +++ b/src/iosource/Manager.h @@ -0,0 +1,75 @@ +// See the file "COPYING" in the main distribution directory for copyright. + +#ifndef IOSOURCE_MANAGER_H +#define IOSOURCE_MANAGER_H + +#include +#include + +namespace iosource { + +class IOSource; +class PktSrc; +class PktDumper; + +class Manager { +public: + Manager() { call_count = 0; dont_counts = 0; } + ~Manager(); + + // If dont_count is true, this source does not contribute to the + // number of IOSources returned by Size(). The effect is that + // if all sources but the non-counting ones have gone dry, + // processing will shut down. + void Register(IOSource* src, bool dont_count = false); + + // This may block for some time. + IOSource* FindSoonest(double* ts); + + int Size() const { return sources.size() - dont_counts; } + + typedef std::list PktSrcList; + const PktSrcList& GetPktSrcs() const { return pkt_srcs; } + + // Terminate IOSource processing immediately by removing all + // sources (and therefore returning a Size() of zero). + void Terminate() { RemoveAll(); } + + PktSrc* OpenPktSrc(const std::string& path, const std::string& filter, bool is_live); + PktDumper* OpenPktDumper(const std::string& path, bool append); + +protected: + void Register(PktSrc* src); + + // When looking for a source with something to process, + // every SELECT_FREQUENCY calls we will go ahead and + // block on a select(). + static const int SELECT_FREQUENCY = 25; + + // Microseconds to wait in an empty select if no source is ready. + static const int SELECT_TIMEOUT = 50; + + void RemoveAll(); + + unsigned int call_count; + int dont_counts; + + struct Source { + IOSource* src; + int fd_read; + int fd_write; + int fd_except; + }; + + typedef std::list SourceList; + SourceList sources; + + PktSrcList pkt_srcs; +}; + +} + +extern iosource::Manager* iosource_mgr; + +#endif + diff --git a/src/iosource/pktsrc/CMakeLists.txt b/src/iosource/pktsrc/CMakeLists.txt new file mode 100644 index 0000000000..07303b46a3 --- /dev/null +++ b/src/iosource/pktsrc/CMakeLists.txt @@ -0,0 +1,2 @@ + +add_subdirectory(pcap) diff --git a/src/iosource/pktsrc/Component.cc b/src/iosource/pktsrc/Component.cc new file mode 100644 index 0000000000..22c49feed0 --- /dev/null +++ b/src/iosource/pktsrc/Component.cc @@ -0,0 +1,130 @@ +// See the file "COPYING" in the main distribution directory for copyright. + +#include "Component.h" + +#include "../Desc.h" + +using namespace iosource::pktsrc; + +SourceComponent::SourceComponent(const std::string& arg_name, const std::string& arg_prefix, InputType arg_type, factory_callback arg_factory) + : iosource::Component(plugin::component::PKTSRC, arg_name) + { + prefix = arg_prefix; + type = arg_type; + factory = arg_factory; + } + +SourceComponent::SourceComponent(const SourceComponent& other) + : iosource::Component(other) + { + prefix = other.prefix; + type = other.type; + factory = other.factory; + } + +SourceComponent::~SourceComponent() + { + } + +const std::string& SourceComponent::Prefix() const + { + return prefix; + } + +bool SourceComponent::DoesLive() const + { + return type == LIVE || type == BOTH; + } + +bool SourceComponent::DoesTrace() const + { + return type == TRACE || type == BOTH; + } + +SourceComponent::factory_callback SourceComponent::Factory() const + { + return factory; + } + + +void SourceComponent::Describe(ODesc* d) const + { + iosource::Component::Describe(d); + + d->Add(" (interface prefix: "); + d->Add(prefix); + d->Add(")"); + } + +SourceComponent& SourceComponent::operator=(const SourceComponent& other) + { + iosource::Component::operator=(other); + + if ( &other != this ) + { + prefix = other.prefix; + type = other.type; + factory = other.factory; + } + + return *this; + } + +DumperComponent::DumperComponent(const std::string& arg_name, const std::string& arg_prefix, factory_callback arg_factory) + : plugin::Component(plugin::component::PKTDUMPER) + { + name = arg_name; + factory = arg_factory; + prefix = arg_prefix; + } + +DumperComponent::DumperComponent(const DumperComponent& other) + : plugin::Component(other) + { + name = other.name; + factory = other.factory; + prefix = other.prefix; + } + +DumperComponent::~DumperComponent() + { + } + +DumperComponent::factory_callback DumperComponent::Factory() const + { + return factory; + } + +const char* DumperComponent::Name() const + { + return name.c_str(); + } + +const std::string& DumperComponent::Prefix() const + { + return prefix; + } + +void DumperComponent::Describe(ODesc* d) const + { + plugin::Component::Describe(d); + + d->Add(name); + d->Add(" (dumper prefix: "); + d->Add(prefix); + d->Add(")"); + } + +DumperComponent& DumperComponent::operator=(const DumperComponent& other) + { + plugin::Component::operator=(other); + + if ( &other != this ) + { + name = other.name; + factory = other.factory; + prefix = other.prefix; + } + + return *this; + } diff --git a/src/iosource/pktsrc/Component.h b/src/iosource/pktsrc/Component.h new file mode 100644 index 0000000000..2a62fb5503 --- /dev/null +++ b/src/iosource/pktsrc/Component.h @@ -0,0 +1,132 @@ +// See the file "COPYING" in the main distribution directory for copyright. + +#ifndef IOSOURCE_PKTSRC_PLUGIN_COMPONENT_H +#define IOSOURCE_PKTSRC_PLUGIN_COMPONENT_H + +#include "../Component.h" + +namespace iosource { + +class PktSrc; +class PktDumper; + +namespace pktsrc { + +/** + * Component description for plugins providing a PktSrc for packet input. + */ +class SourceComponent : public iosource::Component { +public: + enum InputType { LIVE, TRACE, BOTH }; + + typedef PktSrc* (*factory_callback)(const std::string& path, const std::string& filter, bool is_live); + + /** + * XXX + */ + SourceComponent(const std::string& name, const std::string& prefix, InputType type, factory_callback factory); + + /** + * Copy constructor. + */ + SourceComponent(const SourceComponent& other); + + /** + * Destructor. + */ + virtual ~SourceComponent(); + + /** + * Returns the prefix passes to the constructor. + */ + const std::string& Prefix() const; + + /** + * Returns true if packet source instantiated by the component handle + * live traffic. + */ + bool DoesLive() const; + + /** + * Returns true if packet source instantiated by the component handle + * offline traces. + */ + bool DoesTrace() const; + + /** + * Returns the source's factory function. + */ + factory_callback Factory() const; + + /** + * Generates a human-readable description of the component. This goes + * into the output of \c "bro -NN". + */ + virtual void Describe(ODesc* d) const; + + SourceComponent& operator=(const SourceComponent& other); + +private: + std::string prefix; + InputType type; + factory_callback factory; +}; + +/** + * Component description for plugins providing a PktDumper for packet output. + * + * PktDumpers aren't IOSurces but we locate them here to keep them along with + * the PktSrc. + */ +class DumperComponent : public plugin::Component { +public: + typedef PktDumper* (*factory_callback)(const std::string& path, bool append); + + /** + * XXX + */ + DumperComponent(const std::string& name, const std::string& prefix, factory_callback factory); + + /** + * Copy constructor. + */ + DumperComponent(const DumperComponent& other); + + /** + * Destructor. + */ + ~DumperComponent(); + + /** + * XXX + */ + virtual const char* Name() const; + + /** + * Returns the prefix passes to the constructor. + */ + const std::string& Prefix() const; + + /** + * Returns the source's factory function. + */ + factory_callback Factory() const; + + /** + * Generates a human-readable description of the component. This goes + * into the output of \c "bro -NN". + */ + virtual void Describe(ODesc* d) const; + + DumperComponent& operator=(const DumperComponent& other); + +private: + std::string name; + std::string prefix; + factory_callback factory; +}; + +} +} + +#endif diff --git a/src/iosource/pktsrc/PktDumper.cc b/src/iosource/pktsrc/PktDumper.cc new file mode 100644 index 0000000000..21ad79b87d --- /dev/null +++ b/src/iosource/pktsrc/PktDumper.cc @@ -0,0 +1,79 @@ + +// See the file "COPYING" in the main distribution directory for copyright. + +#include +#include + +#include "config.h" + +#include "PktDumper.h" + +using namespace iosource; + +PktDumper::PktDumper() + { + is_open = false; + errmsg = ""; + } + +PktDumper::~PktDumper() + { + } + +const std::string& PktDumper::Path() const + { + return props.path; + } + +bool PktDumper::IsOpen() const + { + return is_open; + } + +double PktDumper::OpenTime() const + { + return is_open ? props.open_time : 0; + } + +bool PktDumper::IsError() const + { + return errmsg.size(); + } + +const std::string& PktDumper::ErrorMsg() const + { + return errmsg; + } + +int PktDumper::HdrSize() const + { + return is_open ? props.hdr_size : -1; + } + +bool PktDumper::Record(const Packet* pkt) + { + return Dump(pkt); + } + +void PktDumper::Opened(const Properties& arg_props) + { + is_open = true; + props = arg_props; + DBG_LOG(DBG_PKTIO, "Opened dumper %s", props.path.c_str()); + } + +void PktDumper::Closed() + { + is_open = false; + props.path = ""; + DBG_LOG(DBG_PKTIO, "Closed dumper %s", props.path.c_str()); + } + +void PktDumper::Error(const std::string& msg) + { + errmsg = msg; + + DBG_LOG(DBG_PKTIO, "Error with dumper %s: %s", + IsOpen() ? props.path.c_str() : "", + msg.c_str()); + } diff --git a/src/iosource/pktsrc/PktDumper.h b/src/iosource/pktsrc/PktDumper.h new file mode 100644 index 0000000000..b8f3595e32 --- /dev/null +++ b/src/iosource/pktsrc/PktDumper.h @@ -0,0 +1,57 @@ +// See the file "COPYING" in the main distribution directory for copyright. + +#ifndef IOSOURCE_PKTSRC_PKTDUMPER_H +#define IOSOURCE_PKTSRC_PKTDUMPER_H + +#include "../IOSource.h" + +namespace iosource { + +class PktDumper { +public: + struct Packet { + const struct pcap_pkthdr* hdr; + const u_char* data; + }; + + PktDumper(); + virtual ~PktDumper(); + + const std::string& Path() const; + bool IsOpen() const; + double OpenTime() const; + bool IsError() const; + const std::string& ErrorMsg() const; + int HdrSize() const; + bool Record(const Packet* pkt); + + virtual void Close() = 0; + virtual void Open() = 0; + +protected: + // Methods to use by derived classed. + // + struct Properties { + std::string path; + int hdr_size; + double open_time; + }; + + void Opened(const Properties& props); + void Closed(); + void Error(const std::string& msg); + + // PktSrc interface for derived classes to implement. + + virtual bool Dump(const Packet* pkt) = 0; + +private: + bool is_open; + Properties props; + + std::string errmsg; +}; + +} + +#endif diff --git a/src/iosource/pktsrc/PktSrc.cc b/src/iosource/pktsrc/PktSrc.cc new file mode 100644 index 0000000000..703a2d634b --- /dev/null +++ b/src/iosource/pktsrc/PktSrc.cc @@ -0,0 +1,411 @@ +// See the file "COPYING" in the main distribution directory for copyright. + +#include +#include + +#include "config.h" + +#include "util.h" +#include "PktSrc.h" +#include "Hash.h" +#include "Net.h" +#include "Sessions.h" + +using namespace iosource; + +PktSrc::PktSrc() + { + have_packet = false; + errbuf = ""; + + next_sync_point = 0; + first_timestamp = 0.0; + first_wallclock = current_wallclock = 0; + } + +PktSrc::~PktSrc() + { + } + +const std::string& PktSrc::Path() const + { + static std::string not_open("not open"); + return IsOpen() ? props.path : not_open; + } + +const char* PktSrc::ErrorMsg() const + { + return errbuf.c_str(); + } + +int PktSrc::LinkType() const + { + return IsOpen() ? props.link_type : -1; + } + +int PktSrc::HdrSize() const + { + return IsOpen() ? props.hdr_size : -1; + } + +int PktSrc::SnapLen() const + { + return snaplen; // That's a global. Change? + } + +bool PktSrc::IsLive() const + { + return props.is_live; + } + +double PktSrc::CurrentPacketTimestamp() + { + return current_pseudo; + } + +double PktSrc::CurrentPacketWallClock() + { + // We stop time when we are suspended. + if ( net_is_processing_suspended() ) + current_wallclock = current_time(true); + + return current_wallclock; + } + +void PktSrc::Opened(const Properties& arg_props) + { + props = arg_props; + SetClosed(false); + + DBG_LOG(DBG_PKTIO, "Opened source %s", props.path.c_str()); + } + +void PktSrc::Closed() + { + SetClosed(true); + + DBG_LOG(DBG_PKTIO, "Closed source %s", props.path.c_str()); + } + +void PktSrc::Error(const std::string& msg) + { + // We don't report this immediately, Bro will ask us for the error + // once it notices we aren't open. + errbuf = msg; + DBG_LOG(DBG_PKTIO, "Error with source %s: %s", + IsOpen() ? props.path.c_str() : "", + msg.c_str()); + } + +void PktSrc::Info(const std::string& msg) + { + reporter->Info("%s", msg.c_str()); + } + +void PktSrc::Weird(const std::string& msg, const Packet* p) + { + sessions->Weird(msg.c_str(), p->hdr, p->data, 0); + } + +void PktSrc::InternalError(const std::string& msg) + { + reporter->InternalError("%s", msg.c_str()); + } + +void PktSrc::ContinueAfterSuspend() + { + current_wallclock = current_time(true); + } + +int PktSrc::GetLinkHeaderSize(int link_type) + { + switch ( link_type ) { + case DLT_NULL: + return 4; + + case DLT_EN10MB: + return 14; + + case DLT_FDDI: + return 13 + 8; // fddi_header + LLC + +#ifdef DLT_LINUX_SLL + case DLT_LINUX_SLL: + return 16; +#endif + + case DLT_PPP_SERIAL: // PPP_SERIAL + return 4; + + case DLT_RAW: + return 0; + } + + return -1; + } + +double PktSrc::CheckPseudoTime() + { + if ( ! IsOpen() ) + return 0; + + if ( ! ExtractNextPacketInternal() ) + return 0; + + if ( remote_trace_sync_interval ) + { + if ( next_sync_point == 0 || current_packet.ts >= next_sync_point ) + { + int n = remote_serializer->SendSyncPoint(); + next_sync_point = first_timestamp + + n * remote_trace_sync_interval; + remote_serializer->Log(RemoteSerializer::LogInfo, + fmt("stopping at packet %.6f, next sync-point at %.6f", + current_packet.ts, next_sync_point)); + + return 0; + } + } + + double pseudo_time = current_packet.ts - first_timestamp; + double ct = (current_time(true) - first_wallclock) * pseudo_realtime; + + return pseudo_time <= ct ? bro_start_time + pseudo_time : 0; + } + +void PktSrc::Init() + { + Open(); + } + +void PktSrc::Done() + { + Close(); + } + +void PktSrc::GetFds(int* read, int* write, int* except) + { + if ( pseudo_realtime ) + { + // Select would give erroneous results. But we simulate it + // by setting idle accordingly. + SetIdle(CheckPseudoTime() == 0); + return; + } + + if ( IsOpen() && props.selectable_fd >= 0 ) + *read = props.selectable_fd; + } + +double PktSrc::NextTimestamp(double* local_network_time) + { + if ( ! IsOpen() ) + return -1.0; + + if ( ! ExtractNextPacketInternal() ) + return -1.0; + + if ( pseudo_realtime ) + { + // Delay packet if necessary. + double packet_time = CheckPseudoTime(); + if ( packet_time ) + return packet_time; + + SetIdle(true); + return -1.0; + } + + return current_packet.ts; + } + +void PktSrc::Process() + { + if ( ! IsOpen() ) + return; + + if ( ! ExtractNextPacketInternal() ) + return; + + int pkt_hdr_size = props.hdr_size; + + // Unfortunately some packets on the link might have MPLS labels + // while others don't. That means we need to ask the link-layer if + // labels are in place. + bool have_mpls = false; + + int protocol = 0; + const u_char* data = current_packet.data; + + switch ( props.link_type ) { + case DLT_NULL: + { + protocol = (data[3] << 24) + (data[2] << 16) + (data[1] << 8) + data[0]; + + // From the Wireshark Wiki: "AF_INET6, unfortunately, has + // different values in {NetBSD,OpenBSD,BSD/OS}, + // {FreeBSD,DragonFlyBSD}, and {Darwin/Mac OS X}, so an IPv6 + // packet might have a link-layer header with 24, 28, or 30 + // as the AF_ value." As we may be reading traces captured on + // platforms other than what we're running on, we accept them + // all here. + if ( protocol != AF_INET + && protocol != AF_INET6 + && protocol != 24 + && protocol != 28 + && protocol != 30 ) + { + Weird("non_ip_packet_in_null_transport", ¤t_packet); + data = 0; + return; + } + + break; + } + + case DLT_EN10MB: + { + // Get protocol being carried from the ethernet frame. + protocol = (data[12] << 8) + data[13]; + + switch ( protocol ) + { + // MPLS carried over the ethernet frame. + case 0x8847: + have_mpls = true; + break; + + // VLAN carried over the ethernet frame. + case 0x8100: + data += GetLinkHeaderSize(props.link_type); + data += 4; // Skip the vlan header + pkt_hdr_size = 0; + + // Check for 802.1ah (Q-in-Q) containing IP. + // Only do a second layer of vlan tag + // stripping because there is no + // specification that allows for deeper + // nesting. + if ( ((data[2] << 8) + data[3]) == 0x0800 ) + data += 4; + + break; + + // PPPoE carried over the ethernet frame. + case 0x8864: + data += GetLinkHeaderSize(props.link_type); + protocol = (data[6] << 8) + data[7]; + data += 8; // Skip the PPPoE session and PPP header + pkt_hdr_size = 0; + + if ( protocol != 0x0021 && protocol != 0x0057 ) + { + // Neither IPv4 nor IPv6. + Weird("non_ip_packet_in_pppoe_encapsulation", ¤t_packet); + data = 0; + return; + } + break; + } + + break; + } + + case DLT_PPP_SERIAL: + { + // Get PPP protocol. + protocol = (data[2] << 8) + data[3]; + + if ( protocol == 0x0281 ) + // MPLS Unicast + have_mpls = true; + + else if ( protocol != 0x0021 && protocol != 0x0057 ) + { + // Neither IPv4 nor IPv6. + Weird("non_ip_packet_in_ppp_encapsulation", ¤t_packet); + data = 0; + return; + } + break; + } + } + + if ( have_mpls ) + { + // Remove the data link layer + data += GetLinkHeaderSize(props.link_type); + + // Denote a header size of zero before the IP header + pkt_hdr_size = 0; + + // Skip the MPLS label stack. + bool end_of_stack = false; + + while ( ! end_of_stack ) + { + end_of_stack = *(data + 2) & 0x01; + data += 4; + } + } + + if ( pseudo_realtime ) + { + current_pseudo = CheckPseudoTime(); + net_packet_arrival(current_pseudo, current_packet.hdr, current_packet.data, pkt_hdr_size, this); + if ( ! first_wallclock ) + first_wallclock = current_time(true); + } + + else + net_packet_arrival(current_packet.ts, current_packet.hdr, current_packet.data, pkt_hdr_size, this); + + have_packet = 0; + DoneWithPacket(¤t_packet); + } + +const char* PktSrc::Tag() + { + return "PktSrc"; + } + +int PktSrc::ExtractNextPacketInternal() + { + if ( have_packet ) + return true; + + have_packet = false; + + // Don't return any packets if processing is suspended (except for the + // very first packet which we need to set up times). + if ( net_is_processing_suspended() && first_timestamp ) + { + SetIdle(true); + return 0; + } + + if ( pseudo_realtime ) + current_wallclock = current_time(true); + + if ( ExtractNextPacket(¤t_packet) ) + { + if ( ! first_timestamp ) + first_timestamp = current_packet.ts; + + have_packet = true; + return 1; + } + + if ( pseudo_realtime && using_communication && ! IsOpen() ) + { + // Source has gone dry, we're done. + if ( remote_trace_sync_interval ) + remote_serializer->SendFinalSyncPoint(); + else + remote_serializer->Terminate(); + } + + SetIdle(true); + return 0; + } + diff --git a/src/iosource/pktsrc/PktSrc.h b/src/iosource/pktsrc/PktSrc.h new file mode 100644 index 0000000000..3c3436bb19 --- /dev/null +++ b/src/iosource/pktsrc/PktSrc.h @@ -0,0 +1,140 @@ +// See the file "COPYING" in the main distribution directory for copyright. + +#ifndef IOSOURCE_PKTSRC_PKTSRC_H +#define IOSOURCE_PKTSRC_PKTSRC_H + +#include "../IOSource.h" + +struct pcap_pkthdr; + +namespace iosource { + +class PktSrc : public IOSource { +public: + struct Stats { + unsigned int received; // pkts received (w/o drops) + unsigned int dropped; // pkts dropped + unsigned int link; // total packets on link + // (not always not available) + // + Stats() { received = dropped = link = 0; } + }; + + PktSrc(); + virtual ~PktSrc(); + + const std::string& Path() const; + const std::string& Filter() const; + bool IsLive() const; + int LinkType() const; + const char* ErrorMsg() const; + int HdrSize() const; + int SnapLen() const; + + // Only valid in pseudo-realtime mode. + double CurrentPacketTimestamp(); + double CurrentPacketWallClock(); + + // Signal packet source that processing was suspended and is now + // going to be continued. + void ContinueAfterSuspend(); + + virtual void Statistics(Stats* stats) = 0; + + // Returns the packet last processed; false if there is no + // current packet available. + virtual bool GetCurrentPacket(const pcap_pkthdr** hdr, const u_char** pkt) = 0; + + // Precompiles a filter and associates the given index with it. + // Returns true on success, 0 if a problem occurred or filtering is + // not supported. + virtual int PrecompileFilter(int index, const std::string& filter) = 0; + + // Activates the filter with the given index. Returns true on + // success, 0 if a problem occurred or the filtering is not + // supported. + virtual int SetFilter(int index) = 0; + + static int GetLinkHeaderSize(int link_type); + +#if 0 + PktSrc_Filter_Type FilterType() const { return filter_type; } + + void AddSecondaryTablePrograms(); + const secondary_program_list& ProgramTable() const + { return program_list; } +#endif + +protected: + // Methods to use by derived classes. + + struct Properties { + std::string path; + std::string filter; // Maybe different than what's passed in if not (directly) supported. + int selectable_fd; + int link_type; + int hdr_size; + bool is_live; + }; + + struct Packet { + double ts; + const struct ::pcap_pkthdr* hdr; + const u_char* data; + }; + + void Opened(const Properties& props); + void Closed(); + void Info(const std::string& msg); + void Error(const std::string& msg); + void Weird(const std::string& msg, const Packet* pkt); + void InternalError(const std::string& msg); + + // PktSrc interface for derived classes to implement. + + virtual void Open() = 0; + virtual void Close() = 0; + // Returns 1 on success, 0 on time-out/gone dry. + virtual int ExtractNextPacket(Packet* pkt) = 0; + virtual void DoneWithPacket(Packet* pkt) = 0; + +private: + // Checks if the current packet has a pseudo-time <= current_time. + // If yes, returns pseudo-time, otherwise 0. + double CheckPseudoTime(); + + // XXX + int ExtractNextPacketInternal(); + + // IOSource interface implementation. + virtual void Init(); + virtual void Done(); + virtual void GetFds(int* read, int* write, int* except); + virtual double NextTimestamp(double* local_network_time); + virtual void Process(); + virtual const char* Tag(); + + Properties props; + + bool have_packet; + Packet current_packet; + + // Only set in pseudo-realtime mode. + double first_timestamp; + double first_wallclock; + double current_wallclock; + double current_pseudo; + double next_sync_point; // For trace synchronziation in pseudo-realtime + + std::string errbuf; + +#if 0 + PktSrc_Filter_Type filter_type; // normal path or secondary path + secondary_program_list program_list; +#endif +}; + +} + + +#endif diff --git a/src/iosource/pktsrc/old-2ndary-code.h b/src/iosource/pktsrc/old-2ndary-code.h new file mode 100644 index 0000000000..0b47cccdc5 --- /dev/null +++ b/src/iosource/pktsrc/old-2ndary-code.h @@ -0,0 +1,69 @@ +// Whether a PktSrc object is used by the normal filter structure or the +// secondary-path structure. +typedef enum { + TYPE_FILTER_NORMAL, // the normal filter + TYPE_FILTER_SECONDARY, // the secondary-path filter +} PktSrc_Filter_Type; + +// {filter,event} tuples conforming the secondary path. +class SecondaryEvent { +public: + SecondaryEvent(const char* arg_filter, Func* arg_event) + { + filter = arg_filter; + event = arg_event; + } + + const char* Filter() { return filter; } + Func* Event() { return event; } + +private: + const char* filter; + Func* event; +}; + +declare(PList,SecondaryEvent); +typedef PList(SecondaryEvent) secondary_event_list; + +class SecondaryPath { +public: + SecondaryPath(); + ~SecondaryPath(); + + secondary_event_list& EventTable() { return event_list; } + const char* Filter() { return filter; } + +private: + secondary_event_list event_list; + // OR'ed union of all SecondaryEvent filters + char* filter; +}; + +// Main secondary-path object. +extern SecondaryPath* secondary_path; + +// {program, {filter,event}} tuple table. +class SecondaryProgram { +public: + SecondaryProgram(BPF_Program* arg_program, SecondaryEvent* arg_event) + { + program = arg_program; + event = arg_event; + } + + ~SecondaryProgram(); + + BPF_Program* Program() { return program; } + SecondaryEvent* Event() { return event; } + +private: + // Associated program. + BPF_Program *program; + + // Event that is run in case the program is matched. + SecondaryEvent* event; +}; + +declare(PList,SecondaryProgram); +typedef PList(SecondaryProgram) secondary_program_list; + diff --git a/src/BPF_Program.cc b/src/iosource/pktsrc/pcap/BPF_Program.cc similarity index 100% rename from src/BPF_Program.cc rename to src/iosource/pktsrc/pcap/BPF_Program.cc diff --git a/src/BPF_Program.h b/src/iosource/pktsrc/pcap/BPF_Program.h similarity index 100% rename from src/BPF_Program.h rename to src/iosource/pktsrc/pcap/BPF_Program.h diff --git a/src/iosource/pktsrc/pcap/CMakeLists.txt b/src/iosource/pktsrc/pcap/CMakeLists.txt new file mode 100644 index 0000000000..b43d51b0ca --- /dev/null +++ b/src/iosource/pktsrc/pcap/CMakeLists.txt @@ -0,0 +1,8 @@ + +include(BroPlugin) + +include_directories(BEFORE ${CMAKE_CURRENT_SOURCE_DIR} ${CMAKE_CURRENT_BINARY_DIR}) + +bro_plugin_begin(Bro Pcap) +bro_plugin_cc(Source.cc Dumper.cc BPF_Program.cc Plugin.cc) +bro_plugin_end() diff --git a/src/iosource/pktsrc/pcap/Dumper.cc b/src/iosource/pktsrc/pcap/Dumper.cc new file mode 100644 index 0000000000..ad1ac2fa73 --- /dev/null +++ b/src/iosource/pktsrc/pcap/Dumper.cc @@ -0,0 +1,111 @@ +// See the file "COPYING" in the main distribution directory for copyright. + +#include +#include + +#include "Dumper.h" +#include "../PktSrc.h" + +using namespace iosource::pktsrc; + +PcapDumper::PcapDumper(const std::string& path, bool arg_append) + { + append = arg_append; + props.path = path; + dumper = 0; + pd = 0; + } + +PcapDumper::~PcapDumper() + { + } + +void PcapDumper::Open() + { + int linktype = -1; + + pd = pcap_open_dead(DLT_EN10MB, 8192); + if ( ! pd ) + { + Error("error for pcap_open_dead"); + return; + } + + if ( props.path.empty() ) + { + Error("no filename given"); + return; + } + + struct stat s; + int exists = 0; + + if ( append ) + { + // See if output file already exists (and is non-empty). + exists = stat(props.path.c_str(), &s); ; + + if ( exists < 0 && errno != ENOENT ) + { + Error(fmt("can't stat file %s: %s", props.path.c_str(), strerror(errno))); + return; + } + } + + if ( ! append || exists < 0 || s.st_size == 0 ) + { + // Open new file. + dumper = pcap_dump_open(pd, props.path.c_str()); + if ( ! dumper ) + { + Error(pcap_geterr(pd)); + return; + } + } + + else + { + // Old file and we need to append, which, unfortunately, + // is not supported by libpcap. So, we have to hack a + // little bit, knowing that pcap_dumpter_t is, in fact, + // a FILE ... :-( + dumper = (pcap_dumper_t*) fopen(props.path.c_str(), "a"); + if ( ! dumper ) + { + Error(fmt("can't open dump %s: %s", props.path.c_str(), strerror(errno))); + return; + } + } + + props.open_time = network_time; + props.hdr_size = PktSrc::GetLinkHeaderSize(pcap_datalink(pd)); + Opened(props); + } + +void PcapDumper::Close() + { + if ( ! dumper ) + return; + + pcap_dump_close(dumper); + pcap_close(pd); + dumper = 0; + pd = 0; + + Closed(); + } + +bool PcapDumper::Dump(const Packet* pkt) + { + if ( ! dumper ) + return false; + + pcap_dump((u_char*) dumper, pkt->hdr, pkt->data); + + return true; + } + +iosource::PktDumper* PcapDumper::Instantiate(const std::string& path, bool append) + { + return new PcapDumper(path, append); + } diff --git a/src/iosource/pktsrc/pcap/Dumper.h b/src/iosource/pktsrc/pcap/Dumper.h new file mode 100644 index 0000000000..c2762a2b04 --- /dev/null +++ b/src/iosource/pktsrc/pcap/Dumper.h @@ -0,0 +1,40 @@ + +#ifndef IOSOURCE_PKTSRC_PCAP_DUMPER_H +#define IOSOURCE_PKTSRC_PCAP_DUMPER_H + +extern "C" { +#include +} + +#include "../PktDumper.h" + +namespace iosource { +namespace pktsrc { + +class PcapDumper : public PktDumper { +public: + PcapDumper(const std::string& path, bool append); + virtual ~PcapDumper(); + + static PktDumper* Instantiate(const std::string& path, bool appen); + +protected: + // PktDumper interface. + virtual void Open(); + virtual void Close(); + virtual bool Dump(const Packet* pkt); + +private: + Properties props; + + bool append; + pcap_dumper_t* dumper; + pcap_t* pd; +}; + +} +} + +#endif + + diff --git a/src/iosource/pktsrc/pcap/Plugin.cc b/src/iosource/pktsrc/pcap/Plugin.cc new file mode 100644 index 0000000000..307f2da99f --- /dev/null +++ b/src/iosource/pktsrc/pcap/Plugin.cc @@ -0,0 +1,12 @@ + +#include "plugin/Plugin.h" +#include "iosource/pktsrc/Component.h" + +#include "Source.h" +#include "Dumper.h" + +BRO_PLUGIN_BEGIN(Bro, Pcap) + BRO_PLUGIN_DESCRIPTION("Packet I/O via libpcap"); + BRO_PLUGIN_PKTSRC("PcapReader", "pcap", SourceComponent::BOTH, PcapSource); + BRO_PLUGIN_PKTDUMPER("PcapTraceWriter", "pcap", PcapDumper); +BRO_PLUGIN_END diff --git a/src/iosource/pktsrc/pcap/Source.cc b/src/iosource/pktsrc/pcap/Source.cc new file mode 100644 index 0000000000..86c0273adf --- /dev/null +++ b/src/iosource/pktsrc/pcap/Source.cc @@ -0,0 +1,343 @@ + +#include "config.h" + +#include "Source.h" + +#ifdef HAVE_PCAP_INT_H +#include +#endif + +using namespace iosource::pktsrc; + +PcapSource::~PcapSource() + { + Close(); + } + +PcapSource::PcapSource(const std::string& path, const std::string& filter, bool is_live) + { + props.path = path; + props.filter = filter; + props.is_live = is_live; + last_data = 0; + } + +void PcapSource::Open() + { + if ( props.is_live ) + OpenLive(); + else + OpenOffline(); + } + +void PcapSource::Close() + { + if ( ! pd ) + return; + + BPF_Program* code; + IterCookie* cookie = filters.InitForIteration(); + while ( (code = filters.NextEntry(cookie)) ) + delete code; + + filters.Clear(); + + pcap_close(pd); + pd = 0; + last_data = 0; + + Closed(); + } + +void PcapSource::OpenLive() + { + char errbuf[PCAP_ERRBUF_SIZE]; + char tmp_errbuf[PCAP_ERRBUF_SIZE]; + +#if 0 + filter_type = ft; +#endif + + // Determine interface if not specified. + if ( props.path.empty() ) + props.path = pcap_lookupdev(tmp_errbuf); + + if ( props.path.empty() ) + { + safe_snprintf(errbuf, sizeof(errbuf), + "pcap_lookupdev: %s", tmp_errbuf); + Error(errbuf); + return; + } + + // Determine network and netmask. + uint32 net; + if ( pcap_lookupnet(props.path.c_str(), &net, &netmask, tmp_errbuf) < 0 ) + { + // ### The lookup can fail if no address is assigned to + // the interface; and libpcap doesn't have any useful notion + // of error codes, just error std::strings - how bogus - so we + // just kludge around the error :-(. + // sprintf(errbuf, "pcap_lookupnet %s", tmp_errbuf); + // return; + netmask = 0xffffff00; + } + + // We use the smallest time-out possible to return almost immediately if + // no packets are available. (We can't use set_nonblocking() as it's + // broken on FreeBSD: even when select() indicates that we can read + // something, we may get nothing if the store buffer hasn't filled up + // yet.) + pd = pcap_open_live(props.path.c_str(), SnapLen(), 1, 1, tmp_errbuf); + + if ( ! pd ) + { + safe_snprintf(errbuf, sizeof(errbuf), + "pcap_open_live: %s", tmp_errbuf); + Error(errbuf); + return; + } + + // ### This needs autoconf'ing. +#ifdef HAVE_PCAP_INT_H + Info("pcap bufsize = %d\n", ((struct pcap *) pd)->bufsize); +#endif + +#ifdef HAVE_LINUX + if ( pcap_setnonblock(pd, 1, tmp_errbuf) < 0 ) + { + safe_snprintf(errbuf, sizeof(errbuf), + "pcap_setnonblock: %s", tmp_errbuf); + Error(errbuf); + pcap_close(pd); + return; + } +#endif + + props.selectable_fd = pcap_fileno(pd); + + if ( PrecompileFilter(0, props.filter) && SetFilter(0) ) + { + SetHdrSize(); + + if ( ! pd ) + // Was closed, couldn't get header size. + return; + + Info(fmt("listening on %s, capture length %d bytes\n", props.path.c_str(), SnapLen())); + } + else + Close(); + + props.is_live = true; + Opened(props); + } + +void PcapSource::OpenOffline() + { + char errbuf[PCAP_ERRBUF_SIZE]; + char tmp_errbuf[PCAP_ERRBUF_SIZE]; + +#if 0 + filter_type = ft; +#endif + + pd = pcap_open_offline(props.path.c_str(), errbuf); + + if ( ! pd ) + { + safe_snprintf(errbuf, sizeof(errbuf), + "pcap_open_offline: %s", tmp_errbuf); + Error(errbuf); + return; + } + + if ( PrecompileFilter(0, props.filter) && SetFilter(0) ) + { + SetHdrSize(); + + if ( ! pd ) + // Was closed, unknown link layer type. + return; + + // We don't put file sources into non-blocking mode as + // otherwise we would not be able to identify the EOF. + + props.selectable_fd = fileno(pcap_file(pd)); + + if ( props.selectable_fd < 0 ) + InternalError("OS does not support selectable pcap fd"); + } + + else + Close(); + + props.is_live = false; + Opened(props); + } + +int PcapSource::ExtractNextPacket(Packet* pkt) + { + const u_char* data = pcap_next(pd, ¤t_hdr); + + if ( ! data ) + { + // Source has gone dry. If it's a network interface, this just means + // it's timed out. If it's a file, though, then the file has been + // exhausted. + if ( ! props.is_live ) + Close(); + + return 0; + } + + pkt->ts = current_hdr.ts.tv_sec + double(current_hdr.ts.tv_usec) / 1e6; + pkt->hdr = ¤t_hdr; + pkt->data = last_data = data; + + if ( current_hdr.len == 0 || current_hdr.caplen == 0 ) + { + Weird("empty_pcap_header", pkt); + return 0; + } + + last_hdr = current_hdr; + last_data = data; + ++stats.received; + return 1; + } + +void PcapSource::DoneWithPacket(Packet* pkt) + { + // Nothing to do. + } + +int PcapSource::PrecompileFilter(int index, const std::string& filter) + { + char errbuf[PCAP_ERRBUF_SIZE]; + + // Compile filter. + BPF_Program* code = new BPF_Program(); + + if ( ! code->Compile(pd, filter.c_str(), netmask, errbuf, sizeof(errbuf)) ) + { + PcapError(); + delete code; + return 0; + } + + // Store it in hash. + HashKey* hash = new HashKey(HashKey(bro_int_t(index))); + BPF_Program* oldcode = filters.Lookup(hash); + if ( oldcode ) + delete oldcode; + + filters.Insert(hash, code); + delete hash; + + return 1; + } + +int PcapSource::SetFilter(int index) + { + char errbuf[PCAP_ERRBUF_SIZE]; + +#if 0 + // We don't want load-level filters for the secondary path. + if ( filter_type == TYPE_FILTER_SECONDARY && index > 0 ) + return 1; +#endif + + HashKey* hash = new HashKey(HashKey(bro_int_t(index))); + BPF_Program* code = filters.Lookup(hash); + delete hash; + + if ( ! code ) + { + safe_snprintf(errbuf, sizeof(errbuf), + "No precompiled pcap filter for index %d", + index); + Error(errbuf); + return 0; + } + + if ( pcap_setfilter(pd, code->GetProgram()) < 0 ) + { + PcapError(); + return 0; + } + +#ifndef HAVE_LINUX + // Linux doesn't clear counters when resetting filter. + stats.received = stats.dropped = stats.link = 0; +#endif + + return 1; + } + +void PcapSource::Statistics(Stats* s) + { + char errbuf[PCAP_ERRBUF_SIZE]; + + if ( ! props.is_live ) + s->received = s->dropped = s->link = 0; + + else + { + struct pcap_stat pstat; + if ( pcap_stats(pd, &pstat) < 0 ) + { + PcapError(); + s->received = s->dropped = s->link = 0; + } + + else + { + s->dropped = pstat.ps_drop; + s->link = pstat.ps_recv; + } + } + + s->received = stats.received; + + if ( ! props.is_live ) + s->dropped = 0; + } + +bool PcapSource::GetCurrentPacket(const pcap_pkthdr** hdr, const u_char** pkt) + { + if ( ! last_data ) + return false; + + *hdr = &last_hdr; + *pkt = last_data; + return true; + } + +void PcapSource::PcapError() + { + assert(pd); + Error(fmt("pcap_error: %s", pcap_geterr(pd))); + Close(); + } + +void PcapSource::SetHdrSize() + { + char errbuf[PCAP_ERRBUF_SIZE]; + + props.link_type = pcap_datalink(pd); + props.hdr_size = GetLinkHeaderSize(props.link_type); + + if ( props.hdr_size < 0 ) + { + safe_snprintf(errbuf, sizeof(errbuf), + "unknown data link type 0x%x", props.link_type); + Error(errbuf); + Close(); + } + } + +iosource::PktSrc* PcapSource::Instantiate(const std::string& path, const std::string& filter, bool is_live) + { + return new PcapSource(path, filter, is_live); + } diff --git a/src/iosource/pktsrc/pcap/Source.h b/src/iosource/pktsrc/pcap/Source.h new file mode 100644 index 0000000000..9f1d7c7eb8 --- /dev/null +++ b/src/iosource/pktsrc/pcap/Source.h @@ -0,0 +1,60 @@ +// See the file "COPYING" in the main distribution directory for copyright. + +#ifndef IOSOURCE_PKTSRC_PCAP_SOURCE_H +#define IOSOURCE_PKTSRC_PCAP_SOURCE_H + +extern "C" { +#include +} + +#include "../PktSrc.h" +#include "BPF_Program.h" +#include "Dict.h" + +declare(PDict,BPF_Program); + +namespace iosource { +namespace pktsrc { + +class PcapSource : public iosource::PktSrc { +public: + // XXX + PcapSource(const std::string& path, const std::string& filter, bool is_live); + + virtual ~PcapSource(); + + static PktSrc* Instantiate(const std::string& path, const std::string& filter, bool is_live); + +protected: + // PktSrc interface. + virtual void Open(); + virtual void Close(); + virtual int ExtractNextPacket(Packet* pkt); + virtual void DoneWithPacket(Packet* pkt); + virtual int PrecompileFilter(int index, const std::string& filter); + virtual int SetFilter(int index); + virtual void Statistics(Stats* stats); + virtual bool GetCurrentPacket(const pcap_pkthdr** hdr, const u_char** pkt); + +private: + void OpenLive(); + void OpenOffline(); + void PcapError(); + void SetHdrSize(); + + Properties props; + Stats stats; + + pcap_t *pd; + uint32 netmask; + PDict(BPF_Program) filters; + + struct pcap_pkthdr current_hdr; + struct pcap_pkthdr last_hdr; + const u_char* last_data; +}; + +} +} + +#endif diff --git a/src/main.cc b/src/main.cc index 53a0cb20ee..cf4bd423b6 100644 --- a/src/main.cc +++ b/src/main.cc @@ -62,6 +62,7 @@ extern "C" void OPENSSL_add_all_algorithms_conf(void); #include "plugin/Manager.h" #include "file_analysis/Manager.h" #include "broxygen/Manager.h" +#include "iosource/Manager.h" #include "binpac_bro.h" @@ -100,6 +101,7 @@ plugin::Manager* plugin_mgr = 0; analyzer::Manager* analyzer_mgr = 0; file_analysis::Manager* file_mgr = 0; broxygen::Manager* broxygen_mgr = 0; +iosource::Manager* iosource_mgr = 0; Stmt* stmts; EventHandlerPtr net_done = 0; RuleMatcher* rule_matcher = 0; @@ -116,7 +118,10 @@ int signal_val = 0; int optimize = 0; int do_notice_analysis = 0; int rule_bench = 0; +int generate_documentation = 0; +#if 0 SecondaryPath* secondary_path = 0; +#endif extern char version[]; char* command_line_policy = 0; vector params; @@ -379,7 +384,9 @@ void terminate_bro() delete event_serializer; delete state_serializer; delete event_registry; +#if 0 delete secondary_path; +#endif delete remote_serializer; delete analyzer_mgr; delete log_mgr; @@ -845,6 +852,7 @@ int main(int argc, char** argv) input_mgr = new input::Manager(); plugin_mgr = new plugin::Manager(); file_mgr = new file_analysis::Manager(); + iosource_mgr = new iosource::Manager(); plugin_mgr->InitPreScript(); analyzer_mgr->InitPreScript(); @@ -967,13 +975,15 @@ int main(int argc, char** argv) snaplen = internal_val("snaplen")->AsCount(); +#if 0 // Initialize the secondary path, if it's needed. secondary_path = new SecondaryPath(); +#endif if ( dns_type != DNS_PRIME ) net_init(interfaces, read_files, netflows, flow_files, writefile, "", - secondary_path->Filter(), do_watchdog); + "", do_watchdog); BroFile::SetDefaultRotation(log_rotate_interval, log_max_size); @@ -1132,9 +1142,9 @@ int main(int argc, char** argv) have_pending_timers = ! reading_traces && timer_mgr->Size() > 0; - io_sources.Register(thread_mgr, true); + iosource_mgr->Register(thread_mgr, true); - if ( io_sources.Size() > 0 || + if ( iosource_mgr->Size() > 0 || have_pending_timers || BifConst::exit_only_after_terminate ) { diff --git a/src/plugin/Component.cc b/src/plugin/Component.cc index ee18d55cdf..94de2c5446 100644 --- a/src/plugin/Component.cc +++ b/src/plugin/Component.cc @@ -43,6 +43,18 @@ void Component::Describe(ODesc* d) const d->Add("File Analyzer"); break; + case component::IOSOURCE: + d->Add("I/O Source"); + break; + + case component::PKTSRC: + d->Add("Packet Source"); + break; + + case component::PKTDUMPER: + d->Add("Packet Dumper"); + break; + default: reporter->InternalWarning("unknown component type in plugin::Component::Describe"); d->Add(""); diff --git a/src/plugin/Component.h b/src/plugin/Component.h index ad02dc7e4b..04248f54a9 100644 --- a/src/plugin/Component.h +++ b/src/plugin/Component.h @@ -3,6 +3,8 @@ #ifndef PLUGIN_COMPONENT_H #define PLUGIN_COMPONENT_H +#include + class ODesc; namespace plugin { @@ -16,7 +18,10 @@ enum Type { READER, /// An input reader (not currently used). WRITER, /// An logging writer (not currenly used). ANALYZER, /// A protocol analyzer. - FILE_ANALYZER /// A file analyzer. + FILE_ANALYZER, /// A file analyzer. + IOSOURCE, /// An I/O source, excluding packet sources. + PKTSRC, /// A packet source. + PKTDUMPER /// A packet dumper. }; } diff --git a/src/plugin/Macros.h b/src/plugin/Macros.h index 9362642e91..c2f8a2ed7a 100644 --- a/src/plugin/Macros.h +++ b/src/plugin/Macros.h @@ -130,4 +130,31 @@ #define BRO_PLUGIN_SUPPORT_ANALYZER(tag) \ AddComponent(new ::analyzer::Component(tag, 0)); +/** + * Defines a component implementing an IOSource (excluding packet sources). + * + * XXXX + * + */ +#define BRO_PLUGIN_IOSOURCE(tag, cls) \ + AddComponent(new ::iosource::pktsrc::Component(tag, iosource::cls::Instantiate)); + +/** + * Defines a component implementing a PktSrc. + * + * XXXX + * + */ +#define BRO_PLUGIN_PKTSRC(tag, prefix, type, cls) \ + AddComponent(new ::iosource::pktsrc::SourceComponent(tag, prefix, iosource::pktsrc::type, iosource::pktsrc::cls::Instantiate)); + +/** + * Defines a component implementing a PktDumper. + * + * XXXX + * + */ +#define BRO_PLUGIN_PKTDUMPER(tag, prefix, cls) \ + AddComponent(new ::iosource::pktsrc::DumperComponent(tag, prefix, iosource::pktsrc::cls::Instantiate)); + #endif diff --git a/src/threading/Manager.cc b/src/threading/Manager.cc index 4491cd42b5..fdc422bd1f 100644 --- a/src/threading/Manager.cc +++ b/src/threading/Manager.cc @@ -11,7 +11,7 @@ Manager::Manager() did_process = true; next_beat = 0; terminating = false; - idle = true; + SetIdle(true); } Manager::~Manager() @@ -47,8 +47,8 @@ void Manager::Terminate() all_threads.clear(); msg_threads.clear(); - idle = true; - closed = true; + SetIdle(true); + SetClosed(true); terminating = false; } @@ -56,7 +56,7 @@ void Manager::AddThread(BasicThread* thread) { DBG_LOG(DBG_THREADING, "Adding thread %s ...", thread->Name()); all_threads.push_back(thread); - idle = false; + SetIdle(false); } void Manager::AddMsgThread(MsgThread* thread) diff --git a/src/threading/Manager.h b/src/threading/Manager.h index e839749a91..c94cc41aaa 100644 --- a/src/threading/Manager.h +++ b/src/threading/Manager.h @@ -4,7 +4,7 @@ #include -#include "IOSource.h" +#include "iosource/IOSource.h" #include "BasicThread.h" #include "MsgThread.h" @@ -21,7 +21,7 @@ namespace threading { * their outgoing message queue on a regular basis and feeds data sent into * the rest of Bro. It also triggers the regular heartbeats. */ -class Manager : public IOSource +class Manager : public iosource::IOSource { public: /** diff --git a/src/util.cc b/src/util.cc index ad55e3f75e..358e88b0e2 100644 --- a/src/util.cc +++ b/src/util.cc @@ -43,6 +43,7 @@ #include "NetVar.h" #include "Net.h" #include "Reporter.h" +#include "iosource/Manager.h" /** * Return IP address without enclosing brackets and any leading 0x. @@ -1351,11 +1352,13 @@ double current_time(bool real) double t = double(tv.tv_sec) + double(tv.tv_usec) / 1e6; - if ( ! pseudo_realtime || real || pkt_srcs.length() == 0 ) + const iosource::Manager::PktSrcList& pkt_srcs(iosource_mgr->GetPktSrcs()); + + if ( ! pseudo_realtime || real || pkt_srcs.empty() ) return t; // This obviously only works for a single source ... - PktSrc* src = pkt_srcs[0]; + iosource::PktSrc* src = pkt_srcs.front(); if ( net_is_processing_suspended() ) return src->CurrentPacketTimestamp(); From 61ee2b9172a10c3f0e483b90a0947fd54f3e989f Mon Sep 17 00:00:00 2001 From: Robin Sommer Date: Wed, 11 Dec 2013 20:55:17 -0800 Subject: [PATCH 02/25] Fixing rebase relicts. --- src/CMakeLists.txt | 3 --- src/Net.h | 4 ---- 2 files changed, 7 deletions(-) diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 1aede44934..516426ea02 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -151,7 +151,6 @@ list(APPEND BINPAC_OUTPUTS "${BINPAC_OUTPUT_CC}") set(bro_SUBDIR_LIBS CACHE INTERNAL "subdir libraries" FORCE) set(bro_PLUGIN_LIBS CACHE INTERNAL "plugin libraries" FORCE) -add_subdirectory(iosource) add_subdirectory(analyzer) add_subdirectory(file_analysis) add_subdirectory(probabilistic) @@ -251,8 +250,6 @@ set(bro_SRCS Anon.cc Attr.cc Base64.cc - BroDoc.cc - BroDocObj.cc Brofiler.cc BroString.cc CCL.cc diff --git a/src/Net.h b/src/Net.h index 421bee5911..b9320181f9 100644 --- a/src/Net.h +++ b/src/Net.h @@ -4,11 +4,7 @@ #define net_h #include "net_util.h" -<<<<<<< HEAD #include "util.h" -#include "BPF_Program.h" -======= ->>>>>>> 5493253... Checkpoint. #include "List.h" #include "FlowSrc.h" #include "Func.h" From 9a9451af00e5243ed29a2c305761e33ad9829577 Mon Sep 17 00:00:00 2001 From: Robin Sommer Date: Mon, 27 Jan 2014 15:19:32 -0800 Subject: [PATCH 03/25] A series of updates and tweaks to the new PktSrc interface. --- configure | 4 ++ src/Net.cc | 91 ++++++---------------------- src/iosource/Manager.cc | 6 +- src/iosource/pktsrc/CMakeLists.txt | 1 + src/iosource/pktsrc/Component.cc | 95 ++++++++++++++++++++++++++---- src/iosource/pktsrc/Component.h | 28 ++++++--- src/iosource/pktsrc/PktSrc.cc | 9 +++ src/iosource/pktsrc/PktSrc.h | 10 ++-- src/iosource/pktsrc/pcap/Plugin.cc | 4 +- src/iosource/pktsrc/pcap/Source.cc | 35 +++++++---- src/iosource/pktsrc/pcap/Source.h | 4 -- 11 files changed, 169 insertions(+), 118 deletions(-) diff --git a/configure b/configure index ba9bf58301..5a4596328c 100755 --- a/configure +++ b/configure @@ -64,6 +64,7 @@ Usage: $0 [OPTION]... [VAR=VALUE]... --with-dataseries=PATH path to DataSeries and Lintel libraries --with-xml2=PATH path to libxml2 installation (for DataSeries) --with-curl=PATH path to libcurl install root (for ElasticSearch) + --with-netmap=PATH path to netmap distribution Packaging Options (for developers): --binary-package toggle special logic for binary packaging @@ -247,6 +248,9 @@ while [ $# -ne 0 ]; do --with-curl=*) append_cache_entry LibCURL_ROOT_DIR PATH $optarg ;; + --with-netmap=*) + append_cache_entry NETMAP_ROOT_DIR PATH $optarg + ;; --binary-package) append_cache_entry BINARY_PACKAGING_MODE BOOL true ;; diff --git a/src/Net.cc b/src/Net.cc index 629f2398d0..a1278c709b 100644 --- a/src/Net.cc +++ b/src/Net.cc @@ -167,90 +167,33 @@ void net_init(name_list& interfaces, name_list& readfiles, reading_traces = 1; for ( int i = 0; i < readfiles.length(); ++i ) - iosource_mgr->OpenPktSrc(readfiles[i], filter, false); + { + iosource::PktSrc* ps = iosource_mgr->OpenPktSrc(readfiles[i], filter, false); + assert(ps); + + if ( ps->ErrorMsg() ) + reporter->FatalError("%s: problem with trace file %s - %s\n", + prog, readfiles[i], + ps->ErrorMsg()); + } } -#if 0 - if ( secondary_filter ) - { - // We use a second PktFileSrc for the - // secondary path. - PktFileSrc* ps = new PktFileSrc(readfiles[i], - secondary_filter, - TYPE_FILTER_SECONDARY); - - if ( ! ps->IsOpen() ) - reporter->FatalError("%s: problem with trace file %s - %s\n", - prog, readfiles[i], - ps->ErrorMsg()); - else - { - pkt_srcs.append(ps); - io_sources.Register(ps); - } - - ps->AddSecondaryTablePrograms(); - } - - for ( int i = 0; i < flowfiles.length(); ++i ) - { - FlowFileSrc* fs = new FlowFileSrc(flowfiles[i]); - - if ( ! fs->IsOpen() ) - reporter->FatalError("%s: problem with netflow file %s - %s\n", - prog, flowfiles[i], fs->ErrorMsg()); - else - { - io_sources.Register(fs); - } - } -#endif - - else if ((interfaces.length() > 0 || netflows.length() > 0)) + else if ( interfaces.length() > 0 ) { reading_live = 1; reading_traces = 0; for ( int i = 0; i < interfaces.length(); ++i ) - iosource_mgr->OpenPktSrc(interfaces[i], filter, true); - } -#if 0 - - if ( secondary_filter ) - { - iosource::PktSrc* ps; - ps = new PktInterfaceSrc(interfaces[i], - filter, TYPE_FILTER_SECONDARY); - - if ( ! ps->IsOpen() ) - reporter->Error("%s: problem with interface %s - %s\n", - prog, interfaces[i], - ps->ErrorMsg()); - else - { - pkt_srcs.append(ps); - io_sources.Register(ps); - } - - ps->AddSecondaryTablePrograms(); - } - } - - for ( int i = 0; i < netflows.length(); ++i ) { - FlowSocketSrc* fs = new FlowSocketSrc(netflows[i]); + iosource::PktSrc* ps = iosource_mgr->OpenPktSrc(interfaces[i], filter, true); + assert(ps); - if ( ! fs->IsOpen() ) - { - reporter->Error("%s: problem with netflow socket %s - %s\n", - prog, netflows[i], fs->ErrorMsg()); - delete fs; - } - - else - io_sources.Register(fs); + if ( ps->ErrorMsg() ) + reporter->FatalError("%s: problem with interface %s - %s\n", + prog, interfaces[i], + ps->ErrorMsg()); } -#endif + } else // have_pending_timers = 1, possibly. We don't set diff --git a/src/iosource/Manager.cc b/src/iosource/Manager.cc index 06608c7afd..6c01e5e57b 100644 --- a/src/iosource/Manager.cc +++ b/src/iosource/Manager.cc @@ -231,7 +231,7 @@ PktSrc* Manager::OpenPktSrc(const std::string& path, const std::string& filter, { pktsrc::SourceComponent* c = *i; - if ( c->Prefix() == prefix && + if ( c->HandlesPrefix(prefix) && (( is_live && c->DoesLive() ) || (! is_live && c->DoesTrace())) ) { @@ -242,7 +242,7 @@ PktSrc* Manager::OpenPktSrc(const std::string& path, const std::string& filter, if ( ! component ) - reporter->FatalError("type of packet source '%s' not recognized", prefix.c_str()); + reporter->FatalError("type of packet source '%s' not recognized, or mode not supported", prefix.c_str()); // Instantiate packet source. @@ -279,7 +279,7 @@ PktDumper* Manager::OpenPktDumper(const string& path, bool append) for ( std::list::const_iterator i = all_components.begin(); i != all_components.end(); i++ ) { - if ( (*i)->Prefix() == prefix ) + if ( (*i)->HandlesPrefix(prefix) ) { component = (*i); break; diff --git a/src/iosource/pktsrc/CMakeLists.txt b/src/iosource/pktsrc/CMakeLists.txt index 07303b46a3..9c8a458c54 100644 --- a/src/iosource/pktsrc/CMakeLists.txt +++ b/src/iosource/pktsrc/CMakeLists.txt @@ -1,2 +1,3 @@ add_subdirectory(pcap) +add_subdirectory(netmap) diff --git a/src/iosource/pktsrc/Component.cc b/src/iosource/pktsrc/Component.cc index 7597e1aaa5..6caf743ff9 100644 --- a/src/iosource/pktsrc/Component.cc +++ b/src/iosource/pktsrc/Component.cc @@ -3,13 +3,14 @@ #include "Component.h" #include "../Desc.h" +#include "../Reporter.h" using namespace iosource::pktsrc; SourceComponent::SourceComponent(const std::string& arg_name, const std::string& arg_prefix, InputType arg_type, factory_callback arg_factory) : iosource::Component(plugin::component::PKTSRC, arg_name) { - prefix = arg_prefix; + tokenize_string(arg_prefix, ":", &prefixes); type = arg_type; factory = arg_factory; } @@ -18,9 +19,21 @@ SourceComponent::~SourceComponent() { } -const std::string& SourceComponent::Prefix() const +const std::vector& SourceComponent::Prefixes() const { - return prefix; + return prefixes; + } + +bool SourceComponent::HandlesPrefix(const string& prefix) const + { + for ( std::vector::const_iterator i = prefixes.begin(); + i != prefixes.end(); i++ ) + { + if ( *i == prefix ) + return true; + } + + return false; } bool SourceComponent::DoesLive() const @@ -43,16 +56,50 @@ void SourceComponent::Describe(ODesc* d) const { iosource::Component::Describe(d); - d->Add(" (interface prefix: "); - d->Add(prefix); + string prefs; + + for ( std::vector::const_iterator i = prefixes.begin(); + i != prefixes.end(); i++ ) + { + if ( prefs.size() ) + prefs += ", "; + + prefs += *i; + } + + d->Add(" (interface prefix"); + if ( prefixes.size() > 1 ) + d->Add("es"); + + d->Add(": "); + d->Add(prefs); + d->Add("; "); + + switch ( type ) { + case LIVE: + d->Add("live input"); + break; + + case TRACE: + d->Add("trace input"); + break; + + case BOTH: + d->Add("live and trace input"); + break; + + default: + reporter->InternalError("unknown PkrSrc type"); + } + d->Add(")"); } DumperComponent::DumperComponent(const std::string& name, const std::string& arg_prefix, factory_callback arg_factory) : plugin::Component(plugin::component::PKTDUMPER, name) { + tokenize_string(arg_prefix, ":", &prefixes); factory = arg_factory; - prefix = arg_prefix; } DumperComponent::~DumperComponent() @@ -64,17 +111,45 @@ DumperComponent::factory_callback DumperComponent::Factory() const return factory; } -const std::string& DumperComponent::Prefix() const +const std::vector& DumperComponent::Prefixes() const { - return prefix; + return prefixes; + } + +bool DumperComponent::HandlesPrefix(const string& prefix) const + { + for ( std::vector::const_iterator i = prefixes.begin(); + i != prefixes.end(); i++ ) + { + if ( *i == prefix ) + return true; + } + + return false; } void DumperComponent::Describe(ODesc* d) const { plugin::Component::Describe(d); - d->Add(" (dumper prefix: "); - d->Add(prefix); + string prefs; + + for ( std::vector::const_iterator i = prefixes.begin(); + i != prefixes.end(); i++ ) + { + if ( prefs.size() ) + prefs += ", "; + + prefs += *i; + } + + d->Add(" (dumper prefix"); + + if ( prefixes.size() > 1 ) + d->Add("es"); + + d->Add(": "); + d->Add(prefs); d->Add(")"); } diff --git a/src/iosource/pktsrc/Component.h b/src/iosource/pktsrc/Component.h index 31ed8a7180..0e4755d7b8 100644 --- a/src/iosource/pktsrc/Component.h +++ b/src/iosource/pktsrc/Component.h @@ -3,6 +3,8 @@ #ifndef IOSOURCE_PKTSRC_PLUGIN_COMPONENT_H #define IOSOURCE_PKTSRC_PLUGIN_COMPONENT_H +#include + #include "../Component.h" namespace iosource { @@ -24,7 +26,7 @@ public: /** * XXX */ - SourceComponent(const std::string& name, const std::string& prefix, InputType type, factory_callback factory); + SourceComponent(const std::string& name, const std::string& prefixes, InputType type, factory_callback factory); /** * Destructor. @@ -32,9 +34,14 @@ public: virtual ~SourceComponent(); /** - * Returns the prefix passes to the constructor. + * Returns the prefix(es) passed to the constructor. */ - const std::string& Prefix() const; + const std::vector& Prefixes() const; + + /** + * Returns true if the given prefix is among the one specified for the component. + */ + bool HandlesPrefix(const std::string& prefix) const; /** * Returns true if packet source instantiated by the component handle @@ -60,7 +67,7 @@ public: virtual void Describe(ODesc* d) const; private: - std::string prefix; + std::vector prefixes; InputType type; factory_callback factory; }; @@ -78,7 +85,7 @@ public: /** * XXX */ - DumperComponent(const std::string& name, const std::string& prefix, factory_callback factory); + DumperComponent(const std::string& name, const std::string& prefixes, factory_callback factory); /** * Destructor. @@ -86,9 +93,14 @@ public: ~DumperComponent(); /** - * Returns the prefix passes to the constructor. + * Returns the prefix(es) passed to the constructor. */ - const std::string& Prefix() const; + const std::vector& Prefixes() const; + + /** + * Returns true if the given prefix is among the one specified for the component. + */ + bool HandlesPrefix(const std::string& prefix) const; /** * Returns the source's factory function. @@ -102,7 +114,7 @@ public: virtual void Describe(ODesc* d) const; private: - std::string prefix; + std::vector prefixes; factory_callback factory; }; diff --git a/src/iosource/pktsrc/PktSrc.cc b/src/iosource/pktsrc/PktSrc.cc index 703a2d634b..c608f5267f 100644 --- a/src/iosource/pktsrc/PktSrc.cc +++ b/src/iosource/pktsrc/PktSrc.cc @@ -409,3 +409,12 @@ int PktSrc::ExtractNextPacketInternal() return 0; } +int PktSrc::PrecompileFilter(int index, const std::string& filter) + { + return 1; + } + +int PktSrc::SetFilter(int index) + { + return 1; + } diff --git a/src/iosource/pktsrc/PktSrc.h b/src/iosource/pktsrc/PktSrc.h index 3c3436bb19..edeecfe6cf 100644 --- a/src/iosource/pktsrc/PktSrc.h +++ b/src/iosource/pktsrc/PktSrc.h @@ -3,9 +3,11 @@ #ifndef IOSOURCE_PKTSRC_PKTSRC_H #define IOSOURCE_PKTSRC_PKTSRC_H -#include "../IOSource.h" +extern "C" { +#include +} -struct pcap_pkthdr; +#include "../IOSource.h" namespace iosource { @@ -48,12 +50,12 @@ public: // Precompiles a filter and associates the given index with it. // Returns true on success, 0 if a problem occurred or filtering is // not supported. - virtual int PrecompileFilter(int index, const std::string& filter) = 0; + virtual int PrecompileFilter(int index, const std::string& filter); // Activates the filter with the given index. Returns true on // success, 0 if a problem occurred or the filtering is not // supported. - virtual int SetFilter(int index) = 0; + virtual int SetFilter(int index); static int GetLinkHeaderSize(int link_type); diff --git a/src/iosource/pktsrc/pcap/Plugin.cc b/src/iosource/pktsrc/pcap/Plugin.cc index 884b5b2bfd..a412e4f650 100644 --- a/src/iosource/pktsrc/pcap/Plugin.cc +++ b/src/iosource/pktsrc/pcap/Plugin.cc @@ -12,8 +12,8 @@ class Plugin : public plugin::Plugin { public: plugin::Configuration Configure() { - AddComponent(new ::iosource::pktsrc::SourceComponent("PcapReader", "pcap", ::iosource::pktsrc::SourceComponent::BOTH, ::iosource::pktsrc::PcapSource::Instantiate)); - AddComponent(new ::iosource::pktsrc::DumperComponent("PcapWriter", "pcap", ::iosource::pktsrc::PcapDumper::Instantiate)); + AddComponent(new ::iosource::pktsrc::SourceComponent("PcapReader", "pcap", ::iosource::pktsrc::SourceComponent::BOTH, ::iosource::pktsrc::PcapSource::Instantiate)); + AddComponent(new ::iosource::pktsrc::DumperComponent("PcapWriter", "pcap", ::iosource::pktsrc::PcapDumper::Instantiate)); plugin::Configuration config; config.name = "Bro::Pcap"; diff --git a/src/iosource/pktsrc/pcap/Source.cc b/src/iosource/pktsrc/pcap/Source.cc index 86c0273adf..8165724871 100644 --- a/src/iosource/pktsrc/pcap/Source.cc +++ b/src/iosource/pktsrc/pcap/Source.cc @@ -1,4 +1,6 @@ +#include + #include "config.h" #include "Source.h" @@ -92,9 +94,7 @@ void PcapSource::OpenLive() if ( ! pd ) { - safe_snprintf(errbuf, sizeof(errbuf), - "pcap_open_live: %s", tmp_errbuf); - Error(errbuf); + Error(tmp_errbuf); return; } @@ -106,10 +106,7 @@ void PcapSource::OpenLive() #ifdef HAVE_LINUX if ( pcap_setnonblock(pd, 1, tmp_errbuf) < 0 ) { - safe_snprintf(errbuf, sizeof(errbuf), - "pcap_setnonblock: %s", tmp_errbuf); - Error(errbuf); - pcap_close(pd); + PcapError(); return; } #endif @@ -136,7 +133,6 @@ void PcapSource::OpenLive() void PcapSource::OpenOffline() { char errbuf[PCAP_ERRBUF_SIZE]; - char tmp_errbuf[PCAP_ERRBUF_SIZE]; #if 0 filter_type = ft; @@ -146,8 +142,6 @@ void PcapSource::OpenOffline() if ( ! pd ) { - safe_snprintf(errbuf, sizeof(errbuf), - "pcap_open_offline: %s", tmp_errbuf); Error(errbuf); return; } @@ -178,6 +172,9 @@ void PcapSource::OpenOffline() int PcapSource::ExtractNextPacket(Packet* pkt) { + if ( ! pd ) + return 0; + const u_char* data = pcap_next(pd, ¤t_hdr); if ( ! data ) @@ -214,6 +211,9 @@ void PcapSource::DoneWithPacket(Packet* pkt) int PcapSource::PrecompileFilter(int index, const std::string& filter) { + if ( ! pd ) + return 1; // Prevent error message. + char errbuf[PCAP_ERRBUF_SIZE]; // Compile filter. @@ -240,6 +240,9 @@ int PcapSource::PrecompileFilter(int index, const std::string& filter) int PcapSource::SetFilter(int index) { + if ( ! pd ) + return 1; // Prevent error message + char errbuf[PCAP_ERRBUF_SIZE]; #if 0 @@ -279,7 +282,7 @@ void PcapSource::Statistics(Stats* s) { char errbuf[PCAP_ERRBUF_SIZE]; - if ( ! props.is_live ) + if ( ! (props.is_live && pd) ) s->received = s->dropped = s->link = 0; else @@ -316,13 +319,19 @@ bool PcapSource::GetCurrentPacket(const pcap_pkthdr** hdr, const u_char** pkt) void PcapSource::PcapError() { - assert(pd); - Error(fmt("pcap_error: %s", pcap_geterr(pd))); + if ( pd ) + Error(fmt("pcap_error: %s", pcap_geterr(pd))); + else + Error("pcap_error: not open"); + Close(); } void PcapSource::SetHdrSize() { + if ( ! pd ) + return; + char errbuf[PCAP_ERRBUF_SIZE]; props.link_type = pcap_datalink(pd); diff --git a/src/iosource/pktsrc/pcap/Source.h b/src/iosource/pktsrc/pcap/Source.h index 9f1d7c7eb8..03b75c1ca7 100644 --- a/src/iosource/pktsrc/pcap/Source.h +++ b/src/iosource/pktsrc/pcap/Source.h @@ -3,10 +3,6 @@ #ifndef IOSOURCE_PKTSRC_PCAP_SOURCE_H #define IOSOURCE_PKTSRC_PCAP_SOURCE_H -extern "C" { -#include -} - #include "../PktSrc.h" #include "BPF_Program.h" #include "Dict.h" From 462fd68931d523a4c712e4531e7386c6e3c2c96c Mon Sep 17 00:00:00 2001 From: Robin Sommer Date: Mon, 27 Jan 2014 15:20:27 -0800 Subject: [PATCH 04/25] Prototype of a netmap packet source. TODO: Add userland BPF filtering so that our filters work. --- src/iosource/pktsrc/PktSrc.cc | 2 +- src/iosource/pktsrc/netmap/CMakeLists.txt | 12 ++ src/iosource/pktsrc/netmap/FindNetmap.cmake | 33 +++++ src/iosource/pktsrc/netmap/Plugin.cc | 26 ++++ src/iosource/pktsrc/netmap/Source.cc | 127 ++++++++++++++++++++ src/iosource/pktsrc/netmap/Source.h | 48 ++++++++ 6 files changed, 247 insertions(+), 1 deletion(-) create mode 100644 src/iosource/pktsrc/netmap/CMakeLists.txt create mode 100644 src/iosource/pktsrc/netmap/FindNetmap.cmake create mode 100644 src/iosource/pktsrc/netmap/Plugin.cc create mode 100644 src/iosource/pktsrc/netmap/Source.cc create mode 100644 src/iosource/pktsrc/netmap/Source.h diff --git a/src/iosource/pktsrc/PktSrc.cc b/src/iosource/pktsrc/PktSrc.cc index c608f5267f..66d18638b8 100644 --- a/src/iosource/pktsrc/PktSrc.cc +++ b/src/iosource/pktsrc/PktSrc.cc @@ -35,7 +35,7 @@ const std::string& PktSrc::Path() const const char* PktSrc::ErrorMsg() const { - return errbuf.c_str(); + return errbuf.size() ? errbuf.c_str() : 0; } int PktSrc::LinkType() const diff --git a/src/iosource/pktsrc/netmap/CMakeLists.txt b/src/iosource/pktsrc/netmap/CMakeLists.txt new file mode 100644 index 0000000000..a8a8a78a16 --- /dev/null +++ b/src/iosource/pktsrc/netmap/CMakeLists.txt @@ -0,0 +1,12 @@ + +set(CMAKE_MODULE_PATH ${CMAKE_MODULE_PATH} ${CMAKE_CURRENT_SOURCE_DIR}) + +find_package(Netmap) + +if ( NETMAP_FOUND ) + include(BroPlugin) + include_directories(BEFORE ${CMAKE_CURRENT_SOURCE_DIR} ${CMAKE_CURRENT_BINARY_DIR} ${NETMAP_INCLUDE_DIR}/sys) + bro_plugin_begin(Bro Netmap) + bro_plugin_cc(Source.cc Plugin.cc) + bro_plugin_end() +endif () diff --git a/src/iosource/pktsrc/netmap/FindNetmap.cmake b/src/iosource/pktsrc/netmap/FindNetmap.cmake new file mode 100644 index 0000000000..a04da2a6a0 --- /dev/null +++ b/src/iosource/pktsrc/netmap/FindNetmap.cmake @@ -0,0 +1,33 @@ +# - Try to find netmap includes. +# +# +# Variables used by this module, they can change the default behaviour and need +# to be set before calling find_package: +# +# NETMAP_ROOT_DIR Set this variable to the root installation of +# netmap if the module has problems finding the +# proper installation path. +# +# Variables defined by this module: +# +# NETMAP_FOUND System has netmap API files. +# NETMAP_INCLUDE_DIR The netmap include directory. + +find_path(NETMAP_ROOT_DIR + NAMES sys/net/netmap_user.h +) + +find_path(NETMAP_INCLUDE_DIR + NAMES sys/net/netmap_user.h + HINTS ${NETMAP_ROOT_DIR} +) + +include(FindPackageHandleStandardArgs) +find_package_handle_standard_args(Netmap DEFAULT_MSG + NETMAP_INCLUDE_DIR +) + +mark_as_advanced( + NETMAP_ROOT_DIR + NETMAP_INCLUDE_DIR +) diff --git a/src/iosource/pktsrc/netmap/Plugin.cc b/src/iosource/pktsrc/netmap/Plugin.cc new file mode 100644 index 0000000000..05b4434e16 --- /dev/null +++ b/src/iosource/pktsrc/netmap/Plugin.cc @@ -0,0 +1,26 @@ +// See the file in the main distribution directory for copyright. + +#include "plugin/Plugin.h" + +#include "Source.h" + +namespace plugin { +namespace Bro_Netmap { + +class Plugin : public plugin::Plugin { +public: + plugin::Configuration Configure() + { + AddComponent(new ::iosource::pktsrc::SourceComponent("NetmapReader", "netmap", ::iosource::pktsrc::SourceComponent::LIVE, ::iosource::pktsrc::NetmapSource::InstantiateNetmap)); + AddComponent(new ::iosource::pktsrc::SourceComponent("NetmapReader", "vale", ::iosource::pktsrc::SourceComponent::LIVE, ::iosource::pktsrc::NetmapSource::InstantiateVale)); + + plugin::Configuration config; + config.name = "Bro::Netmap"; + config.description = "Packet aquisition via netmap"; + return config; + } +} plugin; + +} +} + diff --git a/src/iosource/pktsrc/netmap/Source.cc b/src/iosource/pktsrc/netmap/Source.cc new file mode 100644 index 0000000000..b0569a5a0c --- /dev/null +++ b/src/iosource/pktsrc/netmap/Source.cc @@ -0,0 +1,127 @@ + +#include + +#include "config.h" + +#include "Source.h" + +using namespace iosource::pktsrc; + +NetmapSource::~NetmapSource() + { + Close(); + } + +NetmapSource::NetmapSource(const std::string& path, const std::string& filter, bool is_live, const std::string& arg_kind) + { + if ( ! is_live ) + Error("netmap source does not support offline input"); + + kind = arg_kind; + props.path = path; + props.filter = filter; + last_data = 0; + } + +void NetmapSource::Close() + { + if ( ! nd ) + return; + + nm_close(nd); + nd = 0; + last_data = 0; + + Closed(); + } + +void NetmapSource::Open() + { + std::string iface = kind + ":" + props.path; + nd = nm_open(iface.c_str(), getenv("NETMAP_RING_ID"), 0, 0); + + if ( ! nd ) + { + Error(errno ? strerror(errno) : "invalid interface"); + return; + } + + props.selectable_fd = NETMAP_FD(nd); + props.is_live = true; + props.link_type = DLT_EN10MB; + props.hdr_size = GetLinkHeaderSize(props.link_type); + assert(props.hdr_size >= 0); + + Info(fmt("netmap listening on %s\n", props.path.c_str())); + + Opened(props); + } + +int NetmapSource::ExtractNextPacket(Packet* pkt) + { + nm_hdr_t hdr; + const u_char* data = nm_nextpkt(nd, &hdr); + + if ( ! data ) + // Source has gone dry. + return 0; + + current_hdr.ts = hdr.ts; + current_hdr.caplen = hdr.caplen; + current_hdr.len = hdr.len; + + pkt->ts = current_hdr.ts.tv_sec + double(current_hdr.ts.tv_usec) / 1e6; + pkt->hdr = ¤t_hdr; + pkt->data = last_data = data; + + if ( current_hdr.len == 0 || current_hdr.caplen == 0 ) + { + Weird("empty_netmap_header", pkt); + return 0; + } + + last_hdr = current_hdr; + last_data = data; + ++stats.received; + return 1; + } + +void NetmapSource::DoneWithPacket(Packet* pkt) + { + // Nothing to do. + } + +void NetmapSource::Statistics(Stats* s) + { + if ( ! nd ) + { + s->received = s->link = s->dropped = 0; + return; + } + + s->received = stats.received; + + // TODO: Seems these counter's aren't actually set? + s->link = nd->st.ps_recv; + s->dropped = nd->st.ps_drop + nd->st.ps_ifdrop; + } + +bool NetmapSource::GetCurrentPacket(const pcap_pkthdr** hdr, const u_char** pkt) + { + if ( ! last_data ) + return false; + + *hdr = &last_hdr; + *pkt = last_data; + return true; + } + +iosource::PktSrc* NetmapSource::InstantiateNetmap(const std::string& path, const std::string& filter, bool is_live) + { + return new NetmapSource(path, filter, is_live, "netmap"); + } + +iosource::PktSrc* NetmapSource::InstantiateVale(const std::string& path, const std::string& filter, bool is_live) + { + return new NetmapSource(path, filter, is_live, "value"); + } diff --git a/src/iosource/pktsrc/netmap/Source.h b/src/iosource/pktsrc/netmap/Source.h new file mode 100644 index 0000000000..ff17fe792c --- /dev/null +++ b/src/iosource/pktsrc/netmap/Source.h @@ -0,0 +1,48 @@ +// See the file "COPYING" in the main distribution directory for copyright. + +#ifndef IOSOURCE_PKTSRC_NETMAP_SOURCE_H +#define IOSOURCE_PKTSRC_NETMAP_SOURCE_H + +extern "C" { +#define NETMAP_WITH_LIBS +#include +} + +#include "../PktSrc.h" + +namespace iosource { +namespace pktsrc { + +class NetmapSource : public iosource::PktSrc { +public: + // XXX + NetmapSource(const std::string& path, const std::string& filter, bool is_live, const std::string& kind); + virtual ~NetmapSource(); + + static PktSrc* InstantiateNetmap(const std::string& path, const std::string& filter, bool is_live); + static PktSrc* InstantiateVale(const std::string& path, const std::string& filter, bool is_live); + +protected: + // PktSrc interface. + virtual void Open(); + virtual void Close(); + virtual int ExtractNextPacket(Packet* pkt); + virtual void DoneWithPacket(Packet* pkt); + virtual void Statistics(Stats* stats); + virtual bool GetCurrentPacket(const pcap_pkthdr** hdr, const u_char** pkt); + +private: + std::string kind; + Properties props; + Stats stats; + + nm_desc_t *nd; + pcap_pkthdr current_hdr; + pcap_pkthdr last_hdr; + const u_char* last_data; +}; + +} +} + +#endif From 93e6a4a9dbc13827e2269fb9faf0fa04a77d36de Mon Sep 17 00:00:00 2001 From: Robin Sommer Date: Fri, 22 Aug 2014 16:13:33 -0700 Subject: [PATCH 05/25] Removing netmap, remaining pieces of the 2ndary path, and left-over files of packet sorter. Netmap will move to a plugin. --- TODO.iosources | 1 + src/DebugLogger.cc | 2 +- src/Net.cc | 11 +- src/Net.h | 2 +- src/PacketSort.cc | 364 -------------------- src/PacketSort.h | 130 ------- src/PktSrc.cc | 95 ----- src/Sessions.cc | 49 --- src/Sessions.h | 8 +- src/iosource/pktsrc/CMakeLists.txt | 1 - src/iosource/pktsrc/PktSrc.h | 13 - src/iosource/pktsrc/netmap/CMakeLists.txt | 12 - src/iosource/pktsrc/netmap/FindNetmap.cmake | 33 -- src/iosource/pktsrc/netmap/Plugin.cc | 26 -- src/iosource/pktsrc/netmap/Source.cc | 127 ------- src/iosource/pktsrc/netmap/Source.h | 48 --- src/iosource/pktsrc/old-2ndary-code.h | 69 ---- src/iosource/pktsrc/pcap/Source.cc | 6 - src/main.cc | 14 +- src/plugin/Component.h | 4 +- 20 files changed, 16 insertions(+), 999 deletions(-) delete mode 100644 src/PacketSort.cc delete mode 100644 src/PacketSort.h delete mode 100644 src/iosource/pktsrc/netmap/CMakeLists.txt delete mode 100644 src/iosource/pktsrc/netmap/FindNetmap.cmake delete mode 100644 src/iosource/pktsrc/netmap/Plugin.cc delete mode 100644 src/iosource/pktsrc/netmap/Source.cc delete mode 100644 src/iosource/pktsrc/netmap/Source.h delete mode 100644 src/iosource/pktsrc/old-2ndary-code.h diff --git a/TODO.iosources b/TODO.iosources index 7380c89b92..7b84a60fb4 100644 --- a/TODO.iosources +++ b/TODO.iosources @@ -7,3 +7,4 @@ - PktDumper: Move Dump() to public and remove Record() - Wrap BPF_Program into namespace and clean up - Tests, in particular the packet dumping needs testing. +- Add an interface for derived pkt source to run a BPF filter. diff --git a/src/DebugLogger.cc b/src/DebugLogger.cc index 08b6a35203..6f025e3c2b 100644 --- a/src/DebugLogger.cc +++ b/src/DebugLogger.cc @@ -16,7 +16,7 @@ DebugLogger::Stream DebugLogger::streams[NUM_DBGS] = { { "compressor", 0, false }, {"string", 0, false }, { "notifiers", 0, false }, { "main-loop", 0, false }, { "dpd", 0, false }, { "tm", 0, false }, - { "logging", 0, false }, {"input", 0, false }, + { "logging", 0, false }, {"input", 0, false }, { "threading", 0, false }, { "file_analysis", 0, false }, { "plugins", 0, false }, { "broxygen", 0, false }, { "pktio", 0, false} diff --git a/src/Net.cc b/src/Net.cc index 787bcb56a2..8168f6807d 100644 --- a/src/Net.cc +++ b/src/Net.cc @@ -155,10 +155,10 @@ void net_update_time(double new_network_time) void net_init(name_list& interfaces, name_list& readfiles, name_list& netflows, name_list& flowfiles, - const char* writefile, const char* filter, - const char* secondary_filter, int do_watchdog) + const char* writefile, const char* filter, + int do_watchdog) { - if ( readfiles.length() > 0 || flowfiles.length() > 0 ) + if ( readfiles.length() > 0 ) { reading_live = pseudo_realtime > 0.0; reading_traces = 1; @@ -202,6 +202,11 @@ void net_init(name_list& interfaces, name_list& readfiles, if ( writefile ) { pkt_dumper = iosource_mgr->OpenPktDumper(writefile, false); + assert(pkt_dumper); + + if ( pkt_dumper->ErrorMsg().size() ) + reporter->FatalError("problem opening dump file %s - %s\n", + writefile, pkt_dumper->ErrorMsg().c_str()); ID* id = global_scope()->Lookup("trace_output_file"); if ( ! id ) diff --git a/src/Net.h b/src/Net.h index bf42222538..06b7da9691 100644 --- a/src/Net.h +++ b/src/Net.h @@ -16,7 +16,7 @@ extern void net_init(name_list& interfaces, name_list& readfiles, name_list& netflows, name_list& flowfiles, const char* writefile, const char* filter, - const char* secondary_filter, int do_watchdog); + int do_watchdog); extern void net_run(); extern void net_get_final_stats(); extern void net_finish(int drain_events); diff --git a/src/PacketSort.cc b/src/PacketSort.cc deleted file mode 100644 index 606d21b689..0000000000 --- a/src/PacketSort.cc +++ /dev/null @@ -1,364 +0,0 @@ -#include "IP.h" -#include "PacketSort.h" - -const bool DEBUG_packetsort = false; - -PacketSortElement::PacketSortElement(iosource::PktSrc* arg_src, - double arg_timestamp, const struct pcap_pkthdr* arg_hdr, - const u_char* arg_pkt, int arg_hdr_size) - { - src = arg_src; - timestamp = arg_timestamp; - hdr = *arg_hdr; - hdr_size = arg_hdr_size; - - pkt = new u_char[hdr.caplen]; - memcpy(pkt, arg_pkt, hdr.caplen); - - is_tcp = 0; - ip_hdr = 0; - tcp_flags = 0; - endp = 0; - payload_length = 0; - key = 0; - - // Now check if it is a "parsable" TCP packet. - uint32 caplen = hdr.caplen; - uint32 tcp_offset; - - if ( caplen >= sizeof(struct ip) + hdr_size ) - { - const struct ip* ip = (const struct ip*) (pkt + hdr_size); - if ( ip->ip_v == 4 ) - ip_hdr = new IP_Hdr(ip, false); - else if ( ip->ip_v == 6 && (caplen >= sizeof(struct ip6_hdr) + hdr_size) ) - ip_hdr = new IP_Hdr((const struct ip6_hdr*) ip, false, caplen - hdr_size); - else - // Weird will be generated later in NetSessions::NextPacket. - return; - - if ( ip_hdr->NextProto() == IPPROTO_TCP && - // Note: can't sort fragmented packets - ( ! ip_hdr->IsFragment() ) ) - { - tcp_offset = hdr_size + ip_hdr->HdrLen(); - if ( caplen >= tcp_offset + sizeof(struct tcphdr) ) - { - const struct tcphdr* tp = (const struct tcphdr*) - (pkt + tcp_offset); - - id.src_addr = ip_hdr->SrcAddr(); - id.dst_addr = ip_hdr->DstAddr(); - id.src_port = tp->th_sport; - id.dst_port = tp->th_dport; - id.is_one_way = 0; - - endp = addr_port_canon_lt(id.src_addr, - id.src_port, - id.dst_addr, - id.dst_port) ? 0 : 1; - - seq[endp] = ntohl(tp->th_seq); - - if ( tp->th_flags & TH_ACK ) - seq[1-endp] = ntohl(tp->th_ack); - else - seq[1-endp] = 0; - - tcp_flags = tp->th_flags; - - // DEBUG_MSG("%.6f: %u, %u\n", timestamp, seq[0], seq[1]); - - payload_length = ip_hdr->PayloadLen() - tp->th_off * 4; - - key = BuildConnIDHashKey(id); - - is_tcp = 1; - } - } - } - - if ( DEBUG_packetsort && ! is_tcp ) - DEBUG_MSG("%.6f non-TCP packet\n", timestamp); - } - -PacketSortElement::~PacketSortElement() - { - delete [] pkt; - delete ip_hdr; - delete key; - } - -int PacketSortPQ::Timestamp_Cmp(PacketSortElement* a, PacketSortElement* b) - { - double d = a->timestamp - b->timestamp; - - if ( d > 0 ) return 1; - else if ( d < 0 ) return -1; - else return 0; - } - -int PacketSortPQ::UpdatePQ(PacketSortElement* prev_e, PacketSortElement* new_e) - { - int index = prev_e->pq_index[pq_level]; - - new_e->pq_index[pq_level] = index; - pq[index] = new_e; - - if ( Cmp(prev_e, new_e) > 0 ) - return FixUp(new_e, index); - else - { - FixDown(new_e, index); - return index == 0; - } - } - -int PacketSortPQ::AddToPQ(PacketSortElement* new_e) - { - int index = pq.size(); - - new_e->pq_index[pq_level] = index; - pq.push_back(new_e); - - return FixUp(new_e, index); - } - -int PacketSortPQ::RemoveFromPQ(PacketSortElement* prev_e) - { - if ( pq.size() > 1 ) - { - PacketSortElement* new_e = pq[pq.size() - 1]; - pq.pop_back(); - return UpdatePQ(prev_e, new_e); - } - else - { - pq.pop_back(); - return 1; - } - } - -void PacketSortPQ::Assign(int k, PacketSortElement* e) - { - pq[k] = e; - e->pq_index[pq_level] = k; - } - -PacketSortConnPQ::~PacketSortConnPQ() - { - // Delete elements only in ConnPQ (not in GlobalPQ) to avoid - // double delete. - for ( int i = 0; i < (int) pq.size(); ++i ) - { - delete pq[i]; - pq[i] = 0; - } - } - -int PacketSortConnPQ::Cmp(PacketSortElement* a, PacketSortElement* b) - { - // Note: here we do not distinguish between packets without - // an ACK and packets with seq/ack of 0. The later will sorted - // only by their timestamps. - - if ( a->seq[0] && b->seq[0] && a->seq[0] != b->seq[0] ) - return (a->seq[0] > b->seq[0]) ? 1 : -1; - - else if ( a->seq[1] && b->seq[1] && a->seq[1] != b->seq[1] ) - return (a->seq[1] > b->seq[1]) ? 1 : -1; - - else - return Timestamp_Cmp(a, b); - } - -int PacketSortPQ::FixUp(PacketSortElement* e, int k) - { - if ( k == 0 ) - { - Assign(0, e); - return 1; - } - - int parent = (k-1) / 2; - if ( Cmp(pq[parent], e) > 0 ) - { - Assign(k, pq[parent]); - return FixUp(e, parent); - } - else - { - Assign(k, e); - return 0; - } - } - -void PacketSortPQ::FixDown(PacketSortElement* e, int k) - { - uint32 kid = k * 2 + 1; - - if ( kid >= pq.size() ) - { - Assign(k, e); - return; - } - - if ( kid + 1 < pq.size() && Cmp(pq[kid], pq[kid+1]) > 0 ) - ++kid; - - if ( Cmp(e, pq[kid]) > 0 ) - { - Assign(k, pq[kid]); - FixDown(e, kid); - } - else - Assign(k, e); - } - - -int PacketSortConnPQ::Add(PacketSortElement* e) - { -#if 0 - int endp = e->endp; - uint32 end_seq = e->seq[endp] + e->payload_length; - - int p = 1 - endp; - if ( (e->tcp_flags & TH_RST) && ! (e->tcp_flags & TH_ACK) ) - { - DEBUG_MSG("%.6f %c: %u -> %u\n", - e->TimeStamp(), (p == endp) ? 'S' : 'A', - e->seq[p], next_seq[p]); - e->seq[p] = next_seq[p]; - } - - if ( end_seq > next_seq[endp] ) - next_seq[endp] = end_seq; -#endif - - return AddToPQ(e); - } - -void PacketSortConnPQ::UpdateDeliveredSeq(int endp, int seq, int len, int ack) - { - if ( delivered_seq[endp] == 0 || delivered_seq[endp] == seq ) - delivered_seq[endp] = seq + len; - if ( ack > delivered_seq[1 - endp] ) - delivered_seq[endp] = ack; - } - -bool PacketSortConnPQ::IsContentGapSafe(PacketSortElement* e) - { - int ack = e->seq[1 - e->endp]; - return ack <= delivered_seq[1 - e->endp]; - } - -int PacketSortConnPQ::Remove(PacketSortElement* e) - { - int ret = RemoveFromPQ(e); - UpdateDeliveredSeq(e->endp, e->seq[e->endp], e->payload_length, - e->seq[1 - e->endp]); - return ret; - } - -static void DeleteConnPQ(void* p) - { - delete (PacketSortConnPQ*) p; - } - -PacketSortGlobalPQ::PacketSortGlobalPQ() - { - pq_level = GLOBAL_PQ; - conn_pq_table.SetDeleteFunc(DeleteConnPQ); - } - -PacketSortGlobalPQ::~PacketSortGlobalPQ() - { - // Destruction of PacketSortConnPQ will delete all conn_pq's. - } - -int PacketSortGlobalPQ::Add(PacketSortElement* e) - { - if ( e->is_tcp ) - { - // TCP packets are sorted by sequence numbers - PacketSortConnPQ* conn_pq = FindConnPQ(e); - PacketSortElement* prev_min = conn_pq->Min(); - - if ( conn_pq->Add(e) ) - { - ASSERT(conn_pq->Min() != prev_min); - - if ( prev_min ) - return UpdatePQ(prev_min, e); - else - return AddToPQ(e); - } - - else - { - ASSERT(conn_pq->Min() == prev_min); - return 0; - } - } - else - return AddToPQ(e); - } - -PacketSortElement* PacketSortGlobalPQ::RemoveMin(double timestamp) - { - PacketSortElement* e = Min(); - - if ( ! e ) - return 0; - - if ( e->is_tcp ) - { - PacketSortConnPQ* conn_pq = FindConnPQ(e); - -#if 0 - // Note: the content gap safety check does not work - // because we remove the state for a connection once - // it has no packet in the priority queue. - - // Do not deliver e if it arrives later than timestamp, - // and is not content-gap-safe. - if ( e->timestamp > timestamp && - ! conn_pq->IsContentGapSafe(e) ) - return 0; -#else - if ( e->timestamp > timestamp ) - return 0; -#endif - - conn_pq->Remove(e); - PacketSortElement* new_e = conn_pq->Min(); - - if ( new_e ) - UpdatePQ(e, new_e); - else - { - RemoveFromPQ(e); - conn_pq_table.Remove(e->key); - delete conn_pq; - } - } - else - RemoveFromPQ(e); - - return e; - } - -PacketSortConnPQ* PacketSortGlobalPQ::FindConnPQ(PacketSortElement* e) - { - if ( ! e->is_tcp ) - reporter->InternalError("cannot find a connection for an invalid id"); - - PacketSortConnPQ* pq = (PacketSortConnPQ*) conn_pq_table.Lookup(e->key); - if ( ! pq ) - { - pq = new PacketSortConnPQ(); - conn_pq_table.Insert(e->key, pq); - } - - return pq; - } diff --git a/src/PacketSort.h b/src/PacketSort.h deleted file mode 100644 index d61f66994e..0000000000 --- a/src/PacketSort.h +++ /dev/null @@ -1,130 +0,0 @@ -#ifndef packetsort_h -#define packetsort_h - -// Timestamps can be imprecise and even inconsistent among packets -// from different sources. This class tries to guess a "correct" -// order by looking at TCP sequence numbers. -// -// In particular, it tries to eliminate "false" content gaps. - -#include "Dict.h" -#include "Conn.h" - -enum { - CONN_PQ, - GLOBAL_PQ, - NUM_OF_PQ_LEVEL, -}; - -class PacketSortElement { -public: - PacketSortElement(iosource::PktSrc* src, double timestamp, - const struct pcap_pkthdr* hdr, - const u_char* pkt, int hdr_size); - ~PacketSortElement(); - - iosource::PktSrc* Src() const { return src; } - double TimeStamp() const { return timestamp; } - const struct pcap_pkthdr* Hdr() const { return &hdr; } - const u_char* Pkt() const { return pkt; } - int HdrSize() const { return hdr_size; } - const IP_Hdr* IPHdr() const { return ip_hdr; } - -protected: - iosource::PktSrc* src; - double timestamp; - struct pcap_pkthdr hdr; - u_char* pkt; - int hdr_size; - - IP_Hdr* ip_hdr; - int is_tcp; - ConnID id; - uint32 seq[2]; // indexed by endpoint - int tcp_flags; - int endp; // 0 or 1 - int payload_length; - - HashKey* key; - - int pq_index[NUM_OF_PQ_LEVEL]; - - friend class PacketSortPQ; - friend class PacketSortConnPQ; - friend class PacketSortGlobalPQ; -}; - -class PacketSortPQ { -public: - PacketSortPQ() - { pq_level = -1; } - virtual ~PacketSortPQ() {} - - PacketSortElement* Min() const { return (pq.size() > 0) ? pq[0] : 0; } - -protected: - virtual int Cmp(PacketSortElement* a, PacketSortElement* b) = 0; - int Timestamp_Cmp(PacketSortElement* a, PacketSortElement* b); - - int UpdatePQ(PacketSortElement* prev_e, PacketSortElement* new_e); - int AddToPQ(PacketSortElement* e); - int RemoveFromPQ(PacketSortElement* e); - - void Assign(int k, PacketSortElement* e); - int FixUp(PacketSortElement* e, int k); - void FixDown(PacketSortElement* e, int k); - - vector pq; - int pq_level; -}; - -// Sort by sequence numbers within a connection -class PacketSortConnPQ : public PacketSortPQ { -public: - PacketSortConnPQ() - { - pq_level = CONN_PQ; - delivered_seq[0] = delivered_seq[1] = 0; - } - ~PacketSortConnPQ(); - - int Add(PacketSortElement* e); - - int Remove(PacketSortElement* e); - - bool IsContentGapSafe(PacketSortElement* e); - -protected: - int Cmp(PacketSortElement* a, PacketSortElement* b); - void UpdateDeliveredSeq(int endp, int seq, int len, int ack); - - int delivered_seq[2]; -}; - -declare(PDict, PacketSortConnPQ); - -// Sort by timestamps. -class PacketSortGlobalPQ : public PacketSortPQ { -public: - PacketSortGlobalPQ(); - ~PacketSortGlobalPQ(); - - int Add(PacketSortElement* e); - - int Empty() const { return conn_pq_table.Length() == 0; } - - // Returns the next packet to dispatch if it arrives earlier than the - // given timestamp, otherwise returns 0. - // The packet, if to be returned, is also removed from the - // priority queue. - PacketSortElement* RemoveMin(double timestamp); - -protected: - int Cmp(PacketSortElement* a, PacketSortElement* b) - { return Timestamp_Cmp(a, b); } - PacketSortConnPQ* FindConnPQ(PacketSortElement* e); - - PDict(PacketSortConnPQ) conn_pq_table; -}; - -#endif diff --git a/src/PktSrc.cc b/src/PktSrc.cc index b5ac3a5d69..7a0ed4fa0b 100644 --- a/src/PktSrc.cc +++ b/src/PktSrc.cc @@ -364,10 +364,6 @@ int PktSrc::PrecompileFilter(int index, const char* filter) int PktSrc::SetFilter(int index) { - // We don't want load-level filters for the secondary path. - if ( filter_type == TYPE_FILTER_SECONDARY && index > 0 ) - return 1; - HashKey* hash = new HashKey(HashKey(bro_int_t(index))); BPF_Program* code = filters.Lookup(hash); delete hash; @@ -421,28 +417,6 @@ void PktSrc::Close() } } -void PktSrc::AddSecondaryTablePrograms() - { - BPF_Program* program; - - loop_over_list(secondary_path->EventTable(), i) - { - SecondaryEvent* se = secondary_path->EventTable()[i]; - program = new BPF_Program(); - - if ( ! program->Compile(snaplen, datalink, se->Filter(), - netmask, errbuf, sizeof(errbuf)) ) - { - delete program; - Close(); - return; - } - - SecondaryProgram* sp = new SecondaryProgram(program, se); - program_list.append(sp); - } - } - void PktSrc::Statistics(Stats* s) { if ( reading_traces ) @@ -581,75 +555,6 @@ PktFileSrc::PktFileSrc(const char* arg_readfile, const char* filter, closed = true; } - -SecondaryPath::SecondaryPath() - { - filter = 0; - - // Glue together the secondary filter, if exists. - Val* secondary_fv = internal_val("secondary_filters"); - if ( secondary_fv->AsTableVal()->Size() == 0 ) - return; - - int did_first = 0; - const TableEntryValPDict* v = secondary_fv->AsTable(); - IterCookie* c = v->InitForIteration(); - TableEntryVal* tv; - HashKey* h; - - while ( (tv = v->NextEntry(h, c)) ) - { - // Get the index values. - ListVal* index = - secondary_fv->AsTableVal()->RecoverIndex(h); - - const char* str = - index->Index(0)->Ref()->AsString()->CheckString(); - - if ( ++did_first == 1 ) - { - filter = copy_string(str); - } - else - { - if ( strlen(filter) > 0 ) - { - char* tmp_f = new char[strlen(str) + strlen(filter) + 32]; - if ( strlen(str) == 0 ) - sprintf(tmp_f, "%s", filter); - else - sprintf(tmp_f, "(%s) or (%s)", filter, str); - delete [] filter; - filter = tmp_f; - } - } - - // Build secondary_path event table item and link it. - SecondaryEvent* se = - new SecondaryEvent(index->Index(0)->Ref()->AsString()->CheckString(), - tv->Value()->AsFunc() ); - - event_list.append(se); - - delete h; - Unref(index); - } - } - -SecondaryPath::~SecondaryPath() - { - loop_over_list(event_list, i) - delete event_list[i]; - - delete [] filter; - } - - -SecondaryProgram::~SecondaryProgram() - { - delete program; - } - PktDumper::PktDumper(const char* arg_filename, bool arg_append) { filename[0] = '\0'; diff --git a/src/Sessions.cc b/src/Sessions.cc index 01f81afde6..3aeeaadf3b 100644 --- a/src/Sessions.cc +++ b/src/Sessions.cc @@ -259,55 +259,6 @@ void NetSessions::NextPacket(double t, const struct pcap_pkthdr* hdr, DumpPacket(hdr, pkt); } -void NetSessions::NextPacketSecondary(double /* t */, const struct pcap_pkthdr* hdr, - const u_char* const pkt, int hdr_size, - const iosource::PktSrc* src_ps) - { - SegmentProfiler(segment_logger, "processing-secondary-packet"); - - ++num_packets_processed; - - uint32 caplen = hdr->caplen - hdr_size; - if ( caplen < sizeof(struct ip) ) - { - Weird("truncated_IP", hdr, pkt); - return; - } - -#if 0 - const struct ip* ip = (const struct ip*) (pkt + hdr_size); - if ( ip->ip_v == 4 ) - { - const secondary_program_list& spt = src_ps->ProgramTable(); - - loop_over_list(spt, i) - { - SecondaryProgram* sp = spt[i]; - if ( ! net_packet_match(sp->Program(), pkt, - hdr->len, hdr->caplen) ) - continue; - - val_list* args = new val_list; - StringVal* cmd_val = - new StringVal(sp->Event()->Filter()); - args->append(cmd_val); - IP_Hdr ip_hdr(ip, false); - args->append(ip_hdr.BuildPktHdrVal()); - // ### Need to queue event here. - try - { - sp->Event()->Event()->Call(args); - } - - catch ( InterpreterException& e ) - { /* Already reported. */ } - - delete args; - } - } -#endif - } - int NetSessions::CheckConnectionTag(Connection* conn) { if ( current_iosrc->GetCurrentTag() ) diff --git a/src/Sessions.h b/src/Sessions.h index 619574cfc2..c46c092263 100644 --- a/src/Sessions.h +++ b/src/Sessions.h @@ -69,8 +69,8 @@ public: ~NetSessions(); // Main entry point for packet processing. Dispatches the packet - // either through NextPacket() or NextPacketSecondary(), optionally - // employing the packet sorter first. + // either through NextPacket(), optionally employing the packet + // sorter first. void DispatchPacket(double t, const struct pcap_pkthdr* hdr, const u_char* const pkt, int hdr_size, iosource::PktSrc* src_ps); @@ -221,10 +221,6 @@ protected: void NextPacket(double t, const struct pcap_pkthdr* hdr, const u_char* const pkt, int hdr_size); - void NextPacketSecondary(double t, const struct pcap_pkthdr* hdr, - const u_char* const pkt, int hdr_size, - const iosource::PktSrc* src_ps); - // Record the given packet (if a dumper is active). If len=0 // then the whole packet is recorded, otherwise just the first // len bytes. diff --git a/src/iosource/pktsrc/CMakeLists.txt b/src/iosource/pktsrc/CMakeLists.txt index 9c8a458c54..07303b46a3 100644 --- a/src/iosource/pktsrc/CMakeLists.txt +++ b/src/iosource/pktsrc/CMakeLists.txt @@ -1,3 +1,2 @@ add_subdirectory(pcap) -add_subdirectory(netmap) diff --git a/src/iosource/pktsrc/PktSrc.h b/src/iosource/pktsrc/PktSrc.h index edeecfe6cf..3eddaec6e8 100644 --- a/src/iosource/pktsrc/PktSrc.h +++ b/src/iosource/pktsrc/PktSrc.h @@ -59,14 +59,6 @@ public: static int GetLinkHeaderSize(int link_type); -#if 0 - PktSrc_Filter_Type FilterType() const { return filter_type; } - - void AddSecondaryTablePrograms(); - const secondary_program_list& ProgramTable() const - { return program_list; } -#endif - protected: // Methods to use by derived classes. @@ -129,11 +121,6 @@ private: double next_sync_point; // For trace synchronziation in pseudo-realtime std::string errbuf; - -#if 0 - PktSrc_Filter_Type filter_type; // normal path or secondary path - secondary_program_list program_list; -#endif }; } diff --git a/src/iosource/pktsrc/netmap/CMakeLists.txt b/src/iosource/pktsrc/netmap/CMakeLists.txt deleted file mode 100644 index a8a8a78a16..0000000000 --- a/src/iosource/pktsrc/netmap/CMakeLists.txt +++ /dev/null @@ -1,12 +0,0 @@ - -set(CMAKE_MODULE_PATH ${CMAKE_MODULE_PATH} ${CMAKE_CURRENT_SOURCE_DIR}) - -find_package(Netmap) - -if ( NETMAP_FOUND ) - include(BroPlugin) - include_directories(BEFORE ${CMAKE_CURRENT_SOURCE_DIR} ${CMAKE_CURRENT_BINARY_DIR} ${NETMAP_INCLUDE_DIR}/sys) - bro_plugin_begin(Bro Netmap) - bro_plugin_cc(Source.cc Plugin.cc) - bro_plugin_end() -endif () diff --git a/src/iosource/pktsrc/netmap/FindNetmap.cmake b/src/iosource/pktsrc/netmap/FindNetmap.cmake deleted file mode 100644 index a04da2a6a0..0000000000 --- a/src/iosource/pktsrc/netmap/FindNetmap.cmake +++ /dev/null @@ -1,33 +0,0 @@ -# - Try to find netmap includes. -# -# -# Variables used by this module, they can change the default behaviour and need -# to be set before calling find_package: -# -# NETMAP_ROOT_DIR Set this variable to the root installation of -# netmap if the module has problems finding the -# proper installation path. -# -# Variables defined by this module: -# -# NETMAP_FOUND System has netmap API files. -# NETMAP_INCLUDE_DIR The netmap include directory. - -find_path(NETMAP_ROOT_DIR - NAMES sys/net/netmap_user.h -) - -find_path(NETMAP_INCLUDE_DIR - NAMES sys/net/netmap_user.h - HINTS ${NETMAP_ROOT_DIR} -) - -include(FindPackageHandleStandardArgs) -find_package_handle_standard_args(Netmap DEFAULT_MSG - NETMAP_INCLUDE_DIR -) - -mark_as_advanced( - NETMAP_ROOT_DIR - NETMAP_INCLUDE_DIR -) diff --git a/src/iosource/pktsrc/netmap/Plugin.cc b/src/iosource/pktsrc/netmap/Plugin.cc deleted file mode 100644 index 05b4434e16..0000000000 --- a/src/iosource/pktsrc/netmap/Plugin.cc +++ /dev/null @@ -1,26 +0,0 @@ -// See the file in the main distribution directory for copyright. - -#include "plugin/Plugin.h" - -#include "Source.h" - -namespace plugin { -namespace Bro_Netmap { - -class Plugin : public plugin::Plugin { -public: - plugin::Configuration Configure() - { - AddComponent(new ::iosource::pktsrc::SourceComponent("NetmapReader", "netmap", ::iosource::pktsrc::SourceComponent::LIVE, ::iosource::pktsrc::NetmapSource::InstantiateNetmap)); - AddComponent(new ::iosource::pktsrc::SourceComponent("NetmapReader", "vale", ::iosource::pktsrc::SourceComponent::LIVE, ::iosource::pktsrc::NetmapSource::InstantiateVale)); - - plugin::Configuration config; - config.name = "Bro::Netmap"; - config.description = "Packet aquisition via netmap"; - return config; - } -} plugin; - -} -} - diff --git a/src/iosource/pktsrc/netmap/Source.cc b/src/iosource/pktsrc/netmap/Source.cc deleted file mode 100644 index b0569a5a0c..0000000000 --- a/src/iosource/pktsrc/netmap/Source.cc +++ /dev/null @@ -1,127 +0,0 @@ - -#include - -#include "config.h" - -#include "Source.h" - -using namespace iosource::pktsrc; - -NetmapSource::~NetmapSource() - { - Close(); - } - -NetmapSource::NetmapSource(const std::string& path, const std::string& filter, bool is_live, const std::string& arg_kind) - { - if ( ! is_live ) - Error("netmap source does not support offline input"); - - kind = arg_kind; - props.path = path; - props.filter = filter; - last_data = 0; - } - -void NetmapSource::Close() - { - if ( ! nd ) - return; - - nm_close(nd); - nd = 0; - last_data = 0; - - Closed(); - } - -void NetmapSource::Open() - { - std::string iface = kind + ":" + props.path; - nd = nm_open(iface.c_str(), getenv("NETMAP_RING_ID"), 0, 0); - - if ( ! nd ) - { - Error(errno ? strerror(errno) : "invalid interface"); - return; - } - - props.selectable_fd = NETMAP_FD(nd); - props.is_live = true; - props.link_type = DLT_EN10MB; - props.hdr_size = GetLinkHeaderSize(props.link_type); - assert(props.hdr_size >= 0); - - Info(fmt("netmap listening on %s\n", props.path.c_str())); - - Opened(props); - } - -int NetmapSource::ExtractNextPacket(Packet* pkt) - { - nm_hdr_t hdr; - const u_char* data = nm_nextpkt(nd, &hdr); - - if ( ! data ) - // Source has gone dry. - return 0; - - current_hdr.ts = hdr.ts; - current_hdr.caplen = hdr.caplen; - current_hdr.len = hdr.len; - - pkt->ts = current_hdr.ts.tv_sec + double(current_hdr.ts.tv_usec) / 1e6; - pkt->hdr = ¤t_hdr; - pkt->data = last_data = data; - - if ( current_hdr.len == 0 || current_hdr.caplen == 0 ) - { - Weird("empty_netmap_header", pkt); - return 0; - } - - last_hdr = current_hdr; - last_data = data; - ++stats.received; - return 1; - } - -void NetmapSource::DoneWithPacket(Packet* pkt) - { - // Nothing to do. - } - -void NetmapSource::Statistics(Stats* s) - { - if ( ! nd ) - { - s->received = s->link = s->dropped = 0; - return; - } - - s->received = stats.received; - - // TODO: Seems these counter's aren't actually set? - s->link = nd->st.ps_recv; - s->dropped = nd->st.ps_drop + nd->st.ps_ifdrop; - } - -bool NetmapSource::GetCurrentPacket(const pcap_pkthdr** hdr, const u_char** pkt) - { - if ( ! last_data ) - return false; - - *hdr = &last_hdr; - *pkt = last_data; - return true; - } - -iosource::PktSrc* NetmapSource::InstantiateNetmap(const std::string& path, const std::string& filter, bool is_live) - { - return new NetmapSource(path, filter, is_live, "netmap"); - } - -iosource::PktSrc* NetmapSource::InstantiateVale(const std::string& path, const std::string& filter, bool is_live) - { - return new NetmapSource(path, filter, is_live, "value"); - } diff --git a/src/iosource/pktsrc/netmap/Source.h b/src/iosource/pktsrc/netmap/Source.h deleted file mode 100644 index ff17fe792c..0000000000 --- a/src/iosource/pktsrc/netmap/Source.h +++ /dev/null @@ -1,48 +0,0 @@ -// See the file "COPYING" in the main distribution directory for copyright. - -#ifndef IOSOURCE_PKTSRC_NETMAP_SOURCE_H -#define IOSOURCE_PKTSRC_NETMAP_SOURCE_H - -extern "C" { -#define NETMAP_WITH_LIBS -#include -} - -#include "../PktSrc.h" - -namespace iosource { -namespace pktsrc { - -class NetmapSource : public iosource::PktSrc { -public: - // XXX - NetmapSource(const std::string& path, const std::string& filter, bool is_live, const std::string& kind); - virtual ~NetmapSource(); - - static PktSrc* InstantiateNetmap(const std::string& path, const std::string& filter, bool is_live); - static PktSrc* InstantiateVale(const std::string& path, const std::string& filter, bool is_live); - -protected: - // PktSrc interface. - virtual void Open(); - virtual void Close(); - virtual int ExtractNextPacket(Packet* pkt); - virtual void DoneWithPacket(Packet* pkt); - virtual void Statistics(Stats* stats); - virtual bool GetCurrentPacket(const pcap_pkthdr** hdr, const u_char** pkt); - -private: - std::string kind; - Properties props; - Stats stats; - - nm_desc_t *nd; - pcap_pkthdr current_hdr; - pcap_pkthdr last_hdr; - const u_char* last_data; -}; - -} -} - -#endif diff --git a/src/iosource/pktsrc/old-2ndary-code.h b/src/iosource/pktsrc/old-2ndary-code.h deleted file mode 100644 index 0b47cccdc5..0000000000 --- a/src/iosource/pktsrc/old-2ndary-code.h +++ /dev/null @@ -1,69 +0,0 @@ -// Whether a PktSrc object is used by the normal filter structure or the -// secondary-path structure. -typedef enum { - TYPE_FILTER_NORMAL, // the normal filter - TYPE_FILTER_SECONDARY, // the secondary-path filter -} PktSrc_Filter_Type; - -// {filter,event} tuples conforming the secondary path. -class SecondaryEvent { -public: - SecondaryEvent(const char* arg_filter, Func* arg_event) - { - filter = arg_filter; - event = arg_event; - } - - const char* Filter() { return filter; } - Func* Event() { return event; } - -private: - const char* filter; - Func* event; -}; - -declare(PList,SecondaryEvent); -typedef PList(SecondaryEvent) secondary_event_list; - -class SecondaryPath { -public: - SecondaryPath(); - ~SecondaryPath(); - - secondary_event_list& EventTable() { return event_list; } - const char* Filter() { return filter; } - -private: - secondary_event_list event_list; - // OR'ed union of all SecondaryEvent filters - char* filter; -}; - -// Main secondary-path object. -extern SecondaryPath* secondary_path; - -// {program, {filter,event}} tuple table. -class SecondaryProgram { -public: - SecondaryProgram(BPF_Program* arg_program, SecondaryEvent* arg_event) - { - program = arg_program; - event = arg_event; - } - - ~SecondaryProgram(); - - BPF_Program* Program() { return program; } - SecondaryEvent* Event() { return event; } - -private: - // Associated program. - BPF_Program *program; - - // Event that is run in case the program is matched. - SecondaryEvent* event; -}; - -declare(PList,SecondaryProgram); -typedef PList(SecondaryProgram) secondary_program_list; - diff --git a/src/iosource/pktsrc/pcap/Source.cc b/src/iosource/pktsrc/pcap/Source.cc index 8165724871..271c3efcd7 100644 --- a/src/iosource/pktsrc/pcap/Source.cc +++ b/src/iosource/pktsrc/pcap/Source.cc @@ -245,12 +245,6 @@ int PcapSource::SetFilter(int index) char errbuf[PCAP_ERRBUF_SIZE]; -#if 0 - // We don't want load-level filters for the secondary path. - if ( filter_type == TYPE_FILTER_SECONDARY && index > 0 ) - return 1; -#endif - HashKey* hash = new HashKey(HashKey(bro_int_t(index))); BPF_Program* code = filters.Lookup(hash); delete hash; diff --git a/src/main.cc b/src/main.cc index 15a14e4733..5d5bc38384 100644 --- a/src/main.cc +++ b/src/main.cc @@ -111,9 +111,6 @@ int optimize = 0; int do_notice_analysis = 0; int rule_bench = 0; int generate_documentation = 0; -#if 0 -SecondaryPath* secondary_path = 0; -#endif extern char version[]; char* command_line_policy = 0; vector params; @@ -386,9 +383,6 @@ void terminate_bro() delete event_serializer; delete state_serializer; delete event_registry; -#if 0 - delete secondary_path; -#endif delete remote_serializer; delete analyzer_mgr; delete file_mgr; @@ -1007,15 +1001,9 @@ int main(int argc, char** argv) snaplen = internal_val("snaplen")->AsCount(); -#if 0 - // Initialize the secondary path, if it's needed. - secondary_path = new SecondaryPath(); -#endif - if ( dns_type != DNS_PRIME ) net_init(interfaces, read_files, netflows, flow_files, - writefile, "", - "", do_watchdog); + writefile, "", do_watchdog); BroFile::SetDefaultRotation(log_rotate_interval, log_max_size); diff --git a/src/plugin/Component.h b/src/plugin/Component.h index 74cbbc6128..de24a7dbde 100644 --- a/src/plugin/Component.h +++ b/src/plugin/Component.h @@ -12,7 +12,7 @@ namespace plugin { namespace component { /** - * Component types. + * Component types. */ enum Type { READER, /// An input reader (not currently used). @@ -28,7 +28,7 @@ enum Type { /** * Base class for plugin components. A component is a specific piece of * functionality that a plugin provides, such as a protocol analyzer or a log - * writer. + * writer. */ class Component { From ecf1e32f607ba92c311f346bbef83e1d3246919f Mon Sep 17 00:00:00 2001 From: Robin Sommer Date: Fri, 22 Aug 2014 16:33:55 -0700 Subject: [PATCH 06/25] Removing FlowSrc. We could bring this back, now derived from PktSrc (though strickly speaking it's of course not *packets). But not sure if we want that, as the input framework seems the better place to host it. Then it would turns into a reader. --- src/CMakeLists.txt | 1 - src/FlowSrc.cc | 227 --------------------------------------- src/FlowSrc.h | 84 --------------- src/Net.cc | 1 - src/Net.h | 2 - src/analyzer/Analyzer.cc | 1 + src/main.cc | 23 +--- 7 files changed, 6 insertions(+), 333 deletions(-) delete mode 100644 src/FlowSrc.cc delete mode 100644 src/FlowSrc.h diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index b1b504e62a..d1f6657630 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -279,7 +279,6 @@ set(bro_SRCS EventRegistry.cc Expr.cc File.cc - FlowSrc.cc Frag.cc Frame.cc Func.cc diff --git a/src/FlowSrc.cc b/src/FlowSrc.cc deleted file mode 100644 index f96ba72026..0000000000 --- a/src/FlowSrc.cc +++ /dev/null @@ -1,227 +0,0 @@ -// See the file "COPYING" in the main distribution directory for copyright. -// -// Written by Bernhard Ager, TU Berlin (2006/2007). - -#include -#include -#include -#include - -#include "FlowSrc.h" -#include "Net.h" -#include "analyzer/protocol/netflow/netflow_pac.h" -#include - -FlowSrc::FlowSrc() - { // TODO: v9. - selectable_fd = -1; - data = 0; - pdu_len = -1; - exporter_ip = 0; - current_timestamp = next_timestamp = 0.0; - netflow_analyzer = new binpac::NetFlow::NetFlow_Analyzer(); - } - -FlowSrc::~FlowSrc() - { - delete netflow_analyzer; - } - -void FlowSrc::GetFds(int* read, int* write, int* except) - { - if ( selectable_fd >= 0 ) - *read = selectable_fd; - } - -double FlowSrc::NextTimestamp(double* network_time) - { - if ( ! data && ! ExtractNextPDU() ) - return -1.0; - else - return next_timestamp; - } - -void FlowSrc::Process() - { - if ( ! data && ! ExtractNextPDU() ) - return; - - // This is normally done by calling net_packet_dispatch(), - // but as we don't have a packet to dispatch ... - net_update_time(next_timestamp); - expire_timers(); - - netflow_analyzer->downflow()->set_exporter_ip(exporter_ip); - - // We handle exceptions in NewData (might have changed w/ new binpac). - netflow_analyzer->NewData(0, data, data + pdu_len); - data = 0; - } - -void FlowSrc::Close() - { - safe_close(selectable_fd); - } - - -FlowSocketSrc::~FlowSocketSrc() - { - } - -int FlowSocketSrc::ExtractNextPDU() - { - sockaddr_in from; - socklen_t fromlen = sizeof(from); - pdu_len = recvfrom(selectable_fd, buffer, NF_MAX_PKT_SIZE, 0, - (struct sockaddr*) &from, &fromlen); - if ( pdu_len < 0 ) - { - reporter->Error("problem reading NetFlow data from socket"); - data = 0; - next_timestamp = -1.0; - SetClosed(true); - return 0; - } - - if ( fromlen != sizeof(from) ) - { - reporter->Error("malformed NetFlow PDU"); - return 0; - } - - data = buffer; - exporter_ip = from.sin_addr.s_addr; - next_timestamp = current_time(); - - if ( next_timestamp < current_timestamp ) - next_timestamp = current_timestamp; - else - current_timestamp = next_timestamp; - - return 1; - } - -FlowSocketSrc::FlowSocketSrc(const char* listen_parms) - { - int n = strlen(listen_parms) + 1; - - char laddr[n], port[n], ident[n]; - laddr[0] = port[0] = ident[0] = '\0'; - - int ret = sscanf(listen_parms, "%[^:]:%[^=]=%s", laddr, port, ident); - if ( ret < 2 ) - { - snprintf(errbuf, BRO_FLOW_ERRBUF_SIZE, - "parsing your listen-spec went nuts: laddr='%s', port='%s'\n", - laddr[0] ? laddr : "", port[0] ? port : ""); - SetClosed(true); - return; - } - - const char* id = (ret == 3) ? ident : listen_parms; - netflow_analyzer->downflow()->set_identifier(id); - - struct addrinfo aiprefs = { - 0, PF_INET, SOCK_DGRAM, IPPROTO_UDP, 0, NULL, NULL, NULL - }; - struct addrinfo* ainfo = 0; - if ( (ret = getaddrinfo(laddr, port, &aiprefs, &ainfo)) != 0 ) - { - snprintf(errbuf, BRO_FLOW_ERRBUF_SIZE, - "getaddrinfo(%s, %s, ...): %s", - laddr, port, gai_strerror(ret)); - SetClosed(true); - return; - } - - if ( (selectable_fd = socket (PF_INET, SOCK_DGRAM, 0)) < 0 ) - { - snprintf(errbuf, BRO_FLOW_ERRBUF_SIZE, - "socket: %s", strerror(errno)); - SetClosed(true); - goto cleanup; - } - - if ( bind (selectable_fd, ainfo->ai_addr, ainfo->ai_addrlen) < 0 ) - { - snprintf(errbuf, BRO_FLOW_ERRBUF_SIZE, - "bind: %s", strerror(errno)); - SetClosed(true); - goto cleanup; - } - -cleanup: - freeaddrinfo(ainfo); - } - - -FlowFileSrc::~FlowFileSrc() - { - delete [] readfile; - } - -int FlowFileSrc::ExtractNextPDU() - { - FlowFileSrcPDUHeader pdu_header; - - if ( read(selectable_fd, &pdu_header, sizeof(pdu_header)) < - int(sizeof(pdu_header)) ) - return Error(errno, "read header"); - - if ( pdu_header.pdu_length > NF_MAX_PKT_SIZE ) - { - reporter->Error("NetFlow packet too long"); - - // Safely skip over the too-long PDU. - if ( lseek(selectable_fd, pdu_header.pdu_length, SEEK_CUR) < 0 ) - return Error(errno, "lseek"); - return 0; - } - - if ( read(selectable_fd, buffer, pdu_header.pdu_length) < - pdu_header.pdu_length ) - return Error(errno, "read data"); - - if ( next_timestamp < pdu_header.network_time ) - { - next_timestamp = pdu_header.network_time; - current_timestamp = pdu_header.network_time; - } - else - current_timestamp = next_timestamp; - - data = buffer; - pdu_len = pdu_header.pdu_length; - exporter_ip = pdu_header.ipaddr; - - return 1; - } - -FlowFileSrc::FlowFileSrc(const char* readfile) - { - int n = strlen(readfile) + 1; - char ident[n]; - this->readfile = new char[n]; - - int ret = sscanf(readfile, "%[^=]=%s", this->readfile, ident); - const char* id = (ret == 2) ? ident : this->readfile; - netflow_analyzer->downflow()->set_identifier(id); - - selectable_fd = open(this->readfile, O_RDONLY); - if ( selectable_fd < 0 ) - { - SetClosed(true); - snprintf(errbuf, BRO_FLOW_ERRBUF_SIZE, - "open: %s", strerror(errno)); - } - } - -int FlowFileSrc::Error(int errlvl, const char* errmsg) - { - snprintf(errbuf, BRO_FLOW_ERRBUF_SIZE, - "%s: %s", errmsg, strerror(errlvl)); - data = 0; - next_timestamp = -1.0; - SetClosed(true); - return 0; - } diff --git a/src/FlowSrc.h b/src/FlowSrc.h deleted file mode 100644 index 71c8b0cd11..0000000000 --- a/src/FlowSrc.h +++ /dev/null @@ -1,84 +0,0 @@ -// See the file "COPYING" in the main distribution directory for copyright. -// -// Written by Bernhard Ager, TU Berlin (2006/2007). - -#ifndef flowsrc_h -#define flowsrc_h - -#include "iosource/IOSource.h" -#include "NetVar.h" -#include "binpac.h" - -#define BRO_FLOW_ERRBUF_SIZE 512 - -// TODO: 1500 is enough for v5 - how about the others? -// 65536 would be enough for any UDP packet. -#define NF_MAX_PKT_SIZE 8192 - -struct FlowFileSrcPDUHeader { - double network_time; - int pdu_length; - uint32 ipaddr; -}; - -// Avoid including netflow_pac.h by explicitly declaring the NetFlow_Analyzer. -namespace binpac { - namespace NetFlow { - class NetFlow_Analyzer; - } -} - -class FlowSrc : public iosource::IOSource { -public: - virtual ~FlowSrc(); - - // IOSource interface: - bool IsReady(); - void GetFds(int* read, int* write, int* except); - double NextTimestamp(double* network_time); - void Process(); - - const char* Tag() { return "FlowSrc"; } - const char* ErrorMsg() const { return errbuf; } - -protected: - FlowSrc(); - - virtual int ExtractNextPDU() = 0; - virtual void Close(); - - int selectable_fd; - - double current_timestamp; - double next_timestamp; - binpac::NetFlow::NetFlow_Analyzer* netflow_analyzer; - - u_char buffer[NF_MAX_PKT_SIZE]; - u_char* data; - int pdu_len; - uint32 exporter_ip; // in network byte order - - char errbuf[BRO_FLOW_ERRBUF_SIZE]; -}; - -class FlowSocketSrc : public FlowSrc { -public: - FlowSocketSrc(const char* listen_parms); - virtual ~FlowSocketSrc(); - - int ExtractNextPDU(); -}; - -class FlowFileSrc : public FlowSrc { -public: - FlowFileSrc(const char* readfile); - ~FlowFileSrc(); - - int ExtractNextPDU(); - -protected: - int Error(int errlvl, const char* errmsg); - char* readfile; -}; - -#endif diff --git a/src/Net.cc b/src/Net.cc index 8168f6807d..af9e3bb57f 100644 --- a/src/Net.cc +++ b/src/Net.cc @@ -154,7 +154,6 @@ void net_update_time(double new_network_time) } void net_init(name_list& interfaces, name_list& readfiles, - name_list& netflows, name_list& flowfiles, const char* writefile, const char* filter, int do_watchdog) { diff --git a/src/Net.h b/src/Net.h index 06b7da9691..073ffcd527 100644 --- a/src/Net.h +++ b/src/Net.h @@ -6,7 +6,6 @@ #include "net_util.h" #include "util.h" #include "List.h" -#include "FlowSrc.h" #include "Func.h" #include "RemoteSerializer.h" #include "iosource/IOSource.h" @@ -14,7 +13,6 @@ #include "iosource/pktsrc/PktDumper.h" extern void net_init(name_list& interfaces, name_list& readfiles, - name_list& netflows, name_list& flowfiles, const char* writefile, const char* filter, int do_watchdog); extern void net_run(); diff --git a/src/analyzer/Analyzer.cc b/src/analyzer/Analyzer.cc index bd85f8263a..fb5602f96e 100644 --- a/src/analyzer/Analyzer.cc +++ b/src/analyzer/Analyzer.cc @@ -4,6 +4,7 @@ #include "Analyzer.h" #include "Manager.h" +#include "binpac.h" #include "analyzer/protocol/pia/PIA.h" #include "../Event.h" diff --git a/src/main.cc b/src/main.cc index 5d5bc38384..5066ef85ee 100644 --- a/src/main.cc +++ b/src/main.cc @@ -449,8 +449,6 @@ int main(int argc, char** argv) name_list interfaces; name_list read_files; - name_list netflows; - name_list flow_files; name_list rule_files; char* bst_file = 0; char* id_name = 0; @@ -552,7 +550,7 @@ int main(int argc, char** argv) opterr = 0; char opts[256]; - safe_strncpy(opts, "B:D:e:f:I:i:K:l:n:p:R:r:s:T:t:U:w:x:X:y:Y:z:CFGLNOPSWabdghvZQ", + safe_strncpy(opts, "B:D:e:f:I:i:K:l:n:p:R:r:s:T:t:U:w:x:X:z:CFGLNOPSWabdghvZQ", sizeof(opts)); #ifdef USE_PERFTOOLS_DEBUG @@ -612,10 +610,6 @@ int main(int argc, char** argv) writefile = optarg; break; - case 'y': - flow_files.append(optarg); - break; - case 'z': if ( streq(optarg, "notice") ) do_notice_analysis = 1; @@ -709,10 +703,6 @@ int main(int argc, char** argv) do_watchdog = 1; break; - case 'Y': - netflows.append(optarg); - break; - case 'h': usage(); break; @@ -800,8 +790,7 @@ int main(int argc, char** argv) // seed the PRNG. We should do this here (but at least Linux, FreeBSD // and Solaris provide /dev/urandom). - if ( (interfaces.length() > 0 || netflows.length() > 0) && - (read_files.length() > 0 || flow_files.length() > 0 )) + if ( interfaces.length() > 0 && read_files.length() > 0 ) usage(); #ifdef USE_IDMEF @@ -824,7 +813,7 @@ int main(int argc, char** argv) plugin_mgr->SearchDynamicPlugins(bro_plugin_path()); if ( optind == argc && - read_files.length() == 0 && flow_files.length() == 0 && + read_files.length() == 0 && interfaces.length() == 0 && ! (id_name || bst_file) && ! command_line_policy && ! print_plugins ) add_input_file("-"); @@ -983,8 +972,7 @@ int main(int argc, char** argv) // ### Add support for debug command file. dbg_init_debugger(0); - if ( (flow_files.length() == 0 || read_files.length() == 0) && - (netflows.length() == 0 || interfaces.length() == 0) ) + if ( read_files.length() == 0 && interfaces.length() == 0 ) { Val* interfaces_val = internal_val("interfaces"); if ( interfaces_val ) @@ -1002,8 +990,7 @@ int main(int argc, char** argv) snaplen = internal_val("snaplen")->AsCount(); if ( dns_type != DNS_PRIME ) - net_init(interfaces, read_files, netflows, flow_files, - writefile, "", do_watchdog); + net_init(interfaces, read_files, writefile, "", do_watchdog); BroFile::SetDefaultRotation(log_rotate_interval, log_max_size); From caa55ad3529f3793a474cbd55896bd0142a4c0d8 Mon Sep 17 00:00:00 2001 From: Robin Sommer Date: Fri, 22 Aug 2014 16:44:35 -0700 Subject: [PATCH 07/25] Moving Pkt{Src,Dumper} a directory level up. Also renaming PktSourceComponent to PktSrcComponent. --- TODO.iosources | 8 +- src/Net.cc | 4 +- src/Net.h | 4 +- src/iosource/CMakeLists.txt | 8 +- src/iosource/Component.cc | 147 +++++++++++++++++ src/iosource/Component.h | 109 ++++++++++++ src/iosource/Manager.cc | 19 +-- src/iosource/{pktsrc => }/PktDumper.cc | 0 src/iosource/{pktsrc => }/PktDumper.h | 2 +- src/iosource/{pktsrc => }/PktSrc.cc | 0 src/iosource/{pktsrc => }/PktSrc.h | 2 +- src/iosource/{pktsrc => }/pcap/BPF_Program.cc | 0 src/iosource/{pktsrc => }/pcap/BPF_Program.h | 0 src/iosource/{pktsrc => }/pcap/CMakeLists.txt | 0 src/iosource/{pktsrc => }/pcap/Dumper.cc | 0 src/iosource/{pktsrc => }/pcap/Dumper.h | 0 src/iosource/{pktsrc => }/pcap/Plugin.cc | 0 src/iosource/{pktsrc => }/pcap/Source.cc | 0 src/iosource/{pktsrc => }/pcap/Source.h | 0 src/iosource/pktsrc/CMakeLists.txt | 2 - src/iosource/pktsrc/Component.cc | 155 ------------------ src/iosource/pktsrc/Component.h | 124 -------------- src/plugin/Plugin.h | 1 - 23 files changed, 274 insertions(+), 311 deletions(-) rename src/iosource/{pktsrc => }/PktDumper.cc (100%) rename src/iosource/{pktsrc => }/PktDumper.h (97%) rename src/iosource/{pktsrc => }/PktSrc.cc (100%) rename src/iosource/{pktsrc => }/PktSrc.h (99%) rename src/iosource/{pktsrc => }/pcap/BPF_Program.cc (100%) rename src/iosource/{pktsrc => }/pcap/BPF_Program.h (100%) rename src/iosource/{pktsrc => }/pcap/CMakeLists.txt (100%) rename src/iosource/{pktsrc => }/pcap/Dumper.cc (100%) rename src/iosource/{pktsrc => }/pcap/Dumper.h (100%) rename src/iosource/{pktsrc => }/pcap/Plugin.cc (100%) rename src/iosource/{pktsrc => }/pcap/Source.cc (100%) rename src/iosource/{pktsrc => }/pcap/Source.h (100%) delete mode 100644 src/iosource/pktsrc/CMakeLists.txt delete mode 100644 src/iosource/pktsrc/Component.cc delete mode 100644 src/iosource/pktsrc/Component.h diff --git a/TODO.iosources b/TODO.iosources index 7b84a60fb4..e6afb3978a 100644 --- a/TODO.iosources +++ b/TODO.iosources @@ -1,10 +1,4 @@ -- Move the current_{iosrc,pkt_src,etc.} into manager -- Remove all 2ndary path code -- Remove all flow src code. -- Move pktsrc/*.{h,cc} up a level? Or create a subsublibrary there? -- Create a global Packet data structure and pass that around instead - of the pcap_* stuff? - PktDumper: Move Dump() to public and remove Record() - Wrap BPF_Program into namespace and clean up -- Tests, in particular the packet dumping needs testing. - Add an interface for derived pkt source to run a BPF filter. +- Tests, in particular the packet dumping needs testing. diff --git a/src/Net.cc b/src/Net.cc index af9e3bb57f..70d8b70169 100644 --- a/src/Net.cc +++ b/src/Net.cc @@ -30,8 +30,8 @@ #include "Serializer.h" #include "PacketDumper.h" #include "iosource/Manager.h" -#include "iosource/pktsrc/PktSrc.h" -#include "iosource/pktsrc/PktDumper.h" +#include "iosource/PktSrc.h" +#include "iosource/PktDumper.h" #include "plugin/Manager.h" extern "C" { diff --git a/src/Net.h b/src/Net.h index 073ffcd527..41cbd69abe 100644 --- a/src/Net.h +++ b/src/Net.h @@ -9,8 +9,8 @@ #include "Func.h" #include "RemoteSerializer.h" #include "iosource/IOSource.h" -#include "iosource/pktsrc/PktSrc.h" -#include "iosource/pktsrc/PktDumper.h" +#include "iosource/PktSrc.h" +#include "iosource/PktDumper.h" extern void net_init(name_list& interfaces, name_list& readfiles, const char* writefile, const char* filter, diff --git a/src/iosource/CMakeLists.txt b/src/iosource/CMakeLists.txt index c2b63e3d7e..6bcfa16289 100644 --- a/src/iosource/CMakeLists.txt +++ b/src/iosource/CMakeLists.txt @@ -6,15 +6,11 @@ include_directories(BEFORE ${CMAKE_CURRENT_BINARY_DIR} ) -add_subdirectory(pktsrc) - set(iosource_SRCS Component.cc Manager.cc - - pktsrc/Component.cc - pktsrc/PktDumper.cc - pktsrc/PktSrc.cc + PktDumper.cc + PktSrc.cc ) bro_add_subdir_library(iosource ${iosource_SRCS}) diff --git a/src/iosource/Component.cc b/src/iosource/Component.cc index fa57a7d5ad..f54c212352 100644 --- a/src/iosource/Component.cc +++ b/src/iosource/Component.cc @@ -2,6 +2,7 @@ #include "Component.h" #include "Desc.h" +#include "Reporter.h" using namespace iosource; @@ -18,3 +19,149 @@ Component::Component(plugin::component::Type type, const std::string& name) Component::~Component() { } + +PktSrcComponent::PktSrcComponent(const std::string& arg_name, const std::string& arg_prefix, InputType arg_type, factory_callback arg_factory) + : iosource::Component(plugin::component::PKTSRC, arg_name) + { + tokenize_string(arg_prefix, ":", &prefixes); + type = arg_type; + factory = arg_factory; + } + +PktSrcComponent::~PktSrcComponent() + { + } + +const std::vector& PktSrcComponent::Prefixes() const + { + return prefixes; + } + +bool PktSrcComponent::HandlesPrefix(const string& prefix) const + { + for ( std::vector::const_iterator i = prefixes.begin(); + i != prefixes.end(); i++ ) + { + if ( *i == prefix ) + return true; + } + + return false; + } + +bool PktSrcComponent::DoesLive() const + { + return type == LIVE || type == BOTH; + } + +bool PktSrcComponent::DoesTrace() const + { + return type == TRACE || type == BOTH; + } + +PktSrcComponent::factory_callback PktSrcComponent::Factory() const + { + return factory; + } + + +void PktSrcComponent::Describe(ODesc* d) const + { + iosource::Component::Describe(d); + + string prefs; + + for ( std::vector::const_iterator i = prefixes.begin(); + i != prefixes.end(); i++ ) + { + if ( prefs.size() ) + prefs += ", "; + + prefs += *i; + } + + d->Add(" (interface prefix"); + if ( prefixes.size() > 1 ) + d->Add("es"); + + d->Add(": "); + d->Add(prefs); + d->Add("; "); + + switch ( type ) { + case LIVE: + d->Add("live input"); + break; + + case TRACE: + d->Add("trace input"); + break; + + case BOTH: + d->Add("live and trace input"); + break; + + default: + reporter->InternalError("unknown PkrSrc type"); + } + + d->Add(")"); + } + +PktDumperComponent::PktDumperComponent(const std::string& name, const std::string& arg_prefix, factory_callback arg_factory) + : plugin::Component(plugin::component::PKTDUMPER, name) + { + tokenize_string(arg_prefix, ":", &prefixes); + factory = arg_factory; + } + +PktDumperComponent::~PktDumperComponent() + { + } + +PktDumperComponent::factory_callback PktDumperComponent::Factory() const + { + return factory; + } + +const std::vector& PktDumperComponent::Prefixes() const + { + return prefixes; + } + +bool PktDumperComponent::HandlesPrefix(const string& prefix) const + { + for ( std::vector::const_iterator i = prefixes.begin(); + i != prefixes.end(); i++ ) + { + if ( *i == prefix ) + return true; + } + + return false; + } + +void PktDumperComponent::Describe(ODesc* d) const + { + plugin::Component::Describe(d); + + string prefs; + + for ( std::vector::const_iterator i = prefixes.begin(); + i != prefixes.end(); i++ ) + { + if ( prefs.size() ) + prefs += ", "; + + prefs += *i; + } + + d->Add(" (dumper prefix"); + + if ( prefixes.size() > 1 ) + d->Add("es"); + + d->Add(": "); + d->Add(prefs); + d->Add(")"); + } diff --git a/src/iosource/Component.h b/src/iosource/Component.h index 065855caea..c93597fd67 100644 --- a/src/iosource/Component.h +++ b/src/iosource/Component.h @@ -3,11 +3,16 @@ #ifndef IOSOURCE_PLUGIN_COMPONENT_H #define IOSOURCE_PLUGIN_COMPONENT_H +#include +#include + #include "plugin/Component.h" namespace iosource { class IOSource; +class PktSrc; +class PktDumper; /** * Component description for plugins providing IOSources. @@ -38,6 +43,110 @@ protected: Component(plugin::component::Type type, const std::string& name); }; +/** + * Component description for plugins providing a PktSrc for packet input. + */ +class PktSrcComponent : public iosource::Component { +public: + enum InputType { LIVE, TRACE, BOTH }; + + typedef PktSrc* (*factory_callback)(const std::string& path, const std::string& filter, bool is_live); + + /** + * XXX + */ + PktSrcComponent(const std::string& name, const std::string& prefixes, InputType type, factory_callback factory); + + /** + * Destructor. + */ + virtual ~PktSrcComponent(); + + /** + * Returns the prefix(es) passed to the constructor. + */ + const std::vector& Prefixes() const; + + /** + * Returns true if the given prefix is among the one specified for the component. + */ + bool HandlesPrefix(const std::string& prefix) const; + + /** + * Returns true if packet source instantiated by the component handle + * live traffic. + */ + bool DoesLive() const; + + /** + * Returns true if packet source instantiated by the component handle + * offline traces. + */ + bool DoesTrace() const; + + /** + * Returns the source's factory function. + */ + factory_callback Factory() const; + + /** + * Generates a human-readable description of the component. This goes + * into the output of \c "bro -NN". + */ + virtual void Describe(ODesc* d) const; + +private: + std::vector prefixes; + InputType type; + factory_callback factory; +}; + +/** + * Component description for plugins providing a PktDumper for packet output. + * + * PktDumpers aren't IOSurces but we locate them here to keep them along with + * the PktSrc. + */ +class PktDumperComponent : public plugin::Component { +public: + typedef PktDumper* (*factory_callback)(const std::string& path, bool append); + + /** + * XXX + */ + PktDumperComponent(const std::string& name, const std::string& prefixes, factory_callback factory); + + /** + * Destructor. + */ + ~PktDumperComponent(); + + /** + * Returns the prefix(es) passed to the constructor. + */ + const std::vector& Prefixes() const; + + /** + * Returns true if the given prefix is among the one specified for the component. + */ + bool HandlesPrefix(const std::string& prefix) const; + + /** + * Returns the source's factory function. + */ + factory_callback Factory() const; + + /** + * Generates a human-readable description of the component. This goes + * into the output of \c "bro -NN". + */ + virtual void Describe(ODesc* d) const; + +private: + std::vector prefixes; + factory_callback factory; +}; + } #endif diff --git a/src/iosource/Manager.cc b/src/iosource/Manager.cc index 6c01e5e57b..e2be297a77 100644 --- a/src/iosource/Manager.cc +++ b/src/iosource/Manager.cc @@ -8,9 +8,8 @@ #include "Manager.h" #include "IOSource.h" -#include "pktsrc/PktSrc.h" -#include "pktsrc/PktDumper.h" -#include "pktsrc/Component.h" +#include "PktSrc.h" +#include "PktDumper.h" #include "plugin/Manager.h" #include "util.h" @@ -222,14 +221,14 @@ PktSrc* Manager::OpenPktSrc(const std::string& path, const std::string& filter, // Find the component providing packet sources of the requested prefix. - pktsrc::SourceComponent* component = 0; + PktSrcComponent* component = 0; - std::list all_components = plugin_mgr->Components(); + std::list all_components = plugin_mgr->Components(); - for ( std::list::const_iterator i = all_components.begin(); + for ( std::list::const_iterator i = all_components.begin(); i != all_components.end(); i++ ) { - pktsrc::SourceComponent* c = *i; + PktSrcComponent* c = *i; if ( c->HandlesPrefix(prefix) && (( is_live && c->DoesLive() ) || @@ -272,11 +271,11 @@ PktDumper* Manager::OpenPktDumper(const string& path, bool append) // Find the component providing packet dumpers of the requested prefix. - pktsrc::DumperComponent* component = 0; + PktDumperComponent* component = 0; - std::list all_components = plugin_mgr->Components(); + std::list all_components = plugin_mgr->Components(); - for ( std::list::const_iterator i = all_components.begin(); + for ( std::list::const_iterator i = all_components.begin(); i != all_components.end(); i++ ) { if ( (*i)->HandlesPrefix(prefix) ) diff --git a/src/iosource/pktsrc/PktDumper.cc b/src/iosource/PktDumper.cc similarity index 100% rename from src/iosource/pktsrc/PktDumper.cc rename to src/iosource/PktDumper.cc diff --git a/src/iosource/pktsrc/PktDumper.h b/src/iosource/PktDumper.h similarity index 97% rename from src/iosource/pktsrc/PktDumper.h rename to src/iosource/PktDumper.h index b8f3595e32..794992d870 100644 --- a/src/iosource/pktsrc/PktDumper.h +++ b/src/iosource/PktDumper.h @@ -3,7 +3,7 @@ #ifndef IOSOURCE_PKTSRC_PKTDUMPER_H #define IOSOURCE_PKTSRC_PKTDUMPER_H -#include "../IOSource.h" +#include "IOSource.h" namespace iosource { diff --git a/src/iosource/pktsrc/PktSrc.cc b/src/iosource/PktSrc.cc similarity index 100% rename from src/iosource/pktsrc/PktSrc.cc rename to src/iosource/PktSrc.cc diff --git a/src/iosource/pktsrc/PktSrc.h b/src/iosource/PktSrc.h similarity index 99% rename from src/iosource/pktsrc/PktSrc.h rename to src/iosource/PktSrc.h index 3eddaec6e8..d04ca24817 100644 --- a/src/iosource/pktsrc/PktSrc.h +++ b/src/iosource/PktSrc.h @@ -7,7 +7,7 @@ extern "C" { #include } -#include "../IOSource.h" +#include "IOSource.h" namespace iosource { diff --git a/src/iosource/pktsrc/pcap/BPF_Program.cc b/src/iosource/pcap/BPF_Program.cc similarity index 100% rename from src/iosource/pktsrc/pcap/BPF_Program.cc rename to src/iosource/pcap/BPF_Program.cc diff --git a/src/iosource/pktsrc/pcap/BPF_Program.h b/src/iosource/pcap/BPF_Program.h similarity index 100% rename from src/iosource/pktsrc/pcap/BPF_Program.h rename to src/iosource/pcap/BPF_Program.h diff --git a/src/iosource/pktsrc/pcap/CMakeLists.txt b/src/iosource/pcap/CMakeLists.txt similarity index 100% rename from src/iosource/pktsrc/pcap/CMakeLists.txt rename to src/iosource/pcap/CMakeLists.txt diff --git a/src/iosource/pktsrc/pcap/Dumper.cc b/src/iosource/pcap/Dumper.cc similarity index 100% rename from src/iosource/pktsrc/pcap/Dumper.cc rename to src/iosource/pcap/Dumper.cc diff --git a/src/iosource/pktsrc/pcap/Dumper.h b/src/iosource/pcap/Dumper.h similarity index 100% rename from src/iosource/pktsrc/pcap/Dumper.h rename to src/iosource/pcap/Dumper.h diff --git a/src/iosource/pktsrc/pcap/Plugin.cc b/src/iosource/pcap/Plugin.cc similarity index 100% rename from src/iosource/pktsrc/pcap/Plugin.cc rename to src/iosource/pcap/Plugin.cc diff --git a/src/iosource/pktsrc/pcap/Source.cc b/src/iosource/pcap/Source.cc similarity index 100% rename from src/iosource/pktsrc/pcap/Source.cc rename to src/iosource/pcap/Source.cc diff --git a/src/iosource/pktsrc/pcap/Source.h b/src/iosource/pcap/Source.h similarity index 100% rename from src/iosource/pktsrc/pcap/Source.h rename to src/iosource/pcap/Source.h diff --git a/src/iosource/pktsrc/CMakeLists.txt b/src/iosource/pktsrc/CMakeLists.txt deleted file mode 100644 index 07303b46a3..0000000000 --- a/src/iosource/pktsrc/CMakeLists.txt +++ /dev/null @@ -1,2 +0,0 @@ - -add_subdirectory(pcap) diff --git a/src/iosource/pktsrc/Component.cc b/src/iosource/pktsrc/Component.cc deleted file mode 100644 index 6caf743ff9..0000000000 --- a/src/iosource/pktsrc/Component.cc +++ /dev/null @@ -1,155 +0,0 @@ -// See the file "COPYING" in the main distribution directory for copyright. - -#include "Component.h" - -#include "../Desc.h" -#include "../Reporter.h" - -using namespace iosource::pktsrc; - -SourceComponent::SourceComponent(const std::string& arg_name, const std::string& arg_prefix, InputType arg_type, factory_callback arg_factory) - : iosource::Component(plugin::component::PKTSRC, arg_name) - { - tokenize_string(arg_prefix, ":", &prefixes); - type = arg_type; - factory = arg_factory; - } - -SourceComponent::~SourceComponent() - { - } - -const std::vector& SourceComponent::Prefixes() const - { - return prefixes; - } - -bool SourceComponent::HandlesPrefix(const string& prefix) const - { - for ( std::vector::const_iterator i = prefixes.begin(); - i != prefixes.end(); i++ ) - { - if ( *i == prefix ) - return true; - } - - return false; - } - -bool SourceComponent::DoesLive() const - { - return type == LIVE || type == BOTH; - } - -bool SourceComponent::DoesTrace() const - { - return type == TRACE || type == BOTH; - } - -SourceComponent::factory_callback SourceComponent::Factory() const - { - return factory; - } - - -void SourceComponent::Describe(ODesc* d) const - { - iosource::Component::Describe(d); - - string prefs; - - for ( std::vector::const_iterator i = prefixes.begin(); - i != prefixes.end(); i++ ) - { - if ( prefs.size() ) - prefs += ", "; - - prefs += *i; - } - - d->Add(" (interface prefix"); - if ( prefixes.size() > 1 ) - d->Add("es"); - - d->Add(": "); - d->Add(prefs); - d->Add("; "); - - switch ( type ) { - case LIVE: - d->Add("live input"); - break; - - case TRACE: - d->Add("trace input"); - break; - - case BOTH: - d->Add("live and trace input"); - break; - - default: - reporter->InternalError("unknown PkrSrc type"); - } - - d->Add(")"); - } - -DumperComponent::DumperComponent(const std::string& name, const std::string& arg_prefix, factory_callback arg_factory) - : plugin::Component(plugin::component::PKTDUMPER, name) - { - tokenize_string(arg_prefix, ":", &prefixes); - factory = arg_factory; - } - -DumperComponent::~DumperComponent() - { - } - -DumperComponent::factory_callback DumperComponent::Factory() const - { - return factory; - } - -const std::vector& DumperComponent::Prefixes() const - { - return prefixes; - } - -bool DumperComponent::HandlesPrefix(const string& prefix) const - { - for ( std::vector::const_iterator i = prefixes.begin(); - i != prefixes.end(); i++ ) - { - if ( *i == prefix ) - return true; - } - - return false; - } - -void DumperComponent::Describe(ODesc* d) const - { - plugin::Component::Describe(d); - - string prefs; - - for ( std::vector::const_iterator i = prefixes.begin(); - i != prefixes.end(); i++ ) - { - if ( prefs.size() ) - prefs += ", "; - - prefs += *i; - } - - d->Add(" (dumper prefix"); - - if ( prefixes.size() > 1 ) - d->Add("es"); - - d->Add(": "); - d->Add(prefs); - d->Add(")"); - } - diff --git a/src/iosource/pktsrc/Component.h b/src/iosource/pktsrc/Component.h deleted file mode 100644 index 0e4755d7b8..0000000000 --- a/src/iosource/pktsrc/Component.h +++ /dev/null @@ -1,124 +0,0 @@ -// See the file "COPYING" in the main distribution directory for copyright. - -#ifndef IOSOURCE_PKTSRC_PLUGIN_COMPONENT_H -#define IOSOURCE_PKTSRC_PLUGIN_COMPONENT_H - -#include - -#include "../Component.h" - -namespace iosource { - -class PktSrc; -class PktDumper; - -namespace pktsrc { - -/** - * Component description for plugins providing a PktSrc for packet input. - */ -class SourceComponent : public iosource::Component { -public: - enum InputType { LIVE, TRACE, BOTH }; - - typedef PktSrc* (*factory_callback)(const std::string& path, const std::string& filter, bool is_live); - - /** - * XXX - */ - SourceComponent(const std::string& name, const std::string& prefixes, InputType type, factory_callback factory); - - /** - * Destructor. - */ - virtual ~SourceComponent(); - - /** - * Returns the prefix(es) passed to the constructor. - */ - const std::vector& Prefixes() const; - - /** - * Returns true if the given prefix is among the one specified for the component. - */ - bool HandlesPrefix(const std::string& prefix) const; - - /** - * Returns true if packet source instantiated by the component handle - * live traffic. - */ - bool DoesLive() const; - - /** - * Returns true if packet source instantiated by the component handle - * offline traces. - */ - bool DoesTrace() const; - - /** - * Returns the source's factory function. - */ - factory_callback Factory() const; - - /** - * Generates a human-readable description of the component. This goes - * into the output of \c "bro -NN". - */ - virtual void Describe(ODesc* d) const; - -private: - std::vector prefixes; - InputType type; - factory_callback factory; -}; - -/** - * Component description for plugins providing a PktDumper for packet output. - * - * PktDumpers aren't IOSurces but we locate them here to keep them along with - * the PktSrc. - */ -class DumperComponent : public plugin::Component { -public: - typedef PktDumper* (*factory_callback)(const std::string& path, bool append); - - /** - * XXX - */ - DumperComponent(const std::string& name, const std::string& prefixes, factory_callback factory); - - /** - * Destructor. - */ - ~DumperComponent(); - - /** - * Returns the prefix(es) passed to the constructor. - */ - const std::vector& Prefixes() const; - - /** - * Returns true if the given prefix is among the one specified for the component. - */ - bool HandlesPrefix(const std::string& prefix) const; - - /** - * Returns the source's factory function. - */ - factory_callback Factory() const; - - /** - * Generates a human-readable description of the component. This goes - * into the output of \c "bro -NN". - */ - virtual void Describe(ODesc* d) const; - -private: - std::vector prefixes; - factory_callback factory; -}; - -} -} - -#endif diff --git a/src/plugin/Plugin.h b/src/plugin/Plugin.h index 773fe139f6..ccda20054c 100644 --- a/src/plugin/Plugin.h +++ b/src/plugin/Plugin.h @@ -10,7 +10,6 @@ #include "analyzer/Component.h" #include "file_analysis/Component.h" #include "iosource/Component.h" -#include "iosource/pktsrc/Component.h" // We allow to override this externally for testing purposes. #ifndef BRO_PLUGIN_API_VERSION From 0186061aa83e98526ba9c1a4211902279aea7a6b Mon Sep 17 00:00:00 2001 From: Robin Sommer Date: Fri, 22 Aug 2014 16:49:14 -0700 Subject: [PATCH 08/25] Small packet dumper API change. --- TODO.iosources | 1 - src/Net.cc | 2 +- src/Serializer.cc | 2 +- src/Sessions.cc | 4 ++-- src/bro.bif | 4 ++-- src/iosource/PktDumper.cc | 5 ----- src/iosource/PktDumper.h | 6 ++---- 7 files changed, 8 insertions(+), 16 deletions(-) diff --git a/TODO.iosources b/TODO.iosources index e6afb3978a..e61e858087 100644 --- a/TODO.iosources +++ b/TODO.iosources @@ -1,4 +1,3 @@ -- PktDumper: Move Dump() to public and remove Record() - Wrap BPF_Program into namespace and clean up - Add an interface for derived pkt source to run a BPF filter. - Tests, in particular the packet dumping needs testing. diff --git a/src/Net.cc b/src/Net.cc index 70d8b70169..38ec7c3b06 100644 --- a/src/Net.cc +++ b/src/Net.cc @@ -127,7 +127,7 @@ RETSIGTYPE watchdog(int /* signo */) iosource::PktDumper::Packet p; p.hdr = current_hdr; p.data = current_pkt; - pkt_dumper->Record(&p); + pkt_dumper->Dump(&p); } } diff --git a/src/Serializer.cc b/src/Serializer.cc index fd3b34da44..e63e7eef77 100644 --- a/src/Serializer.cc +++ b/src/Serializer.cc @@ -1221,7 +1221,7 @@ Packet* Packet::Unserialize(UnserialInfo* info) iosource::PktDumper::Packet dp; dp.hdr = p->hdr; dp.data = p->pkt; - dump->Record(&dp); + dump->Dump(&dp); } } #endif diff --git a/src/Sessions.cc b/src/Sessions.cc index 3aeeaadf3b..43e55dd95a 100644 --- a/src/Sessions.cc +++ b/src/Sessions.cc @@ -1394,7 +1394,7 @@ void NetSessions::DumpPacket(const struct pcap_pkthdr* hdr, iosource::PktDumper::Packet p; p.hdr = hdr; p.data = pkt; - pkt_dumper->Record(&p); + pkt_dumper->Dump(&p); } else @@ -1407,7 +1407,7 @@ void NetSessions::DumpPacket(const struct pcap_pkthdr* hdr, iosource::PktDumper::Packet p; p.hdr = &h; p.data = pkt; - pkt_dumper->Record(&p); + pkt_dumper->Dump(&p); } } diff --git a/src/bro.bif b/src/bro.bif index 25d53cc736..adf5e3d6f0 100644 --- a/src/bro.bif +++ b/src/bro.bif @@ -3235,7 +3235,7 @@ function dump_current_packet%(file_name: string%) : bool iosource::PktDumper::Packet p; p.hdr = hdr; p.data = pkt; - addl_pkt_dumper->Record(&p); + addl_pkt_dumper->Dump(&p); } return new Val(! addl_pkt_dumper->IsError(), TYPE_BOOL); @@ -3300,7 +3300,7 @@ function dump_packet%(pkt: pcap_packet, file_name: string%) : bool iosource::PktDumper::Packet p; p.hdr = &hdr; p.data = (*pkt_vl)[4]->AsString()->Bytes(); - addl_pkt_dumper->Record(&p); + addl_pkt_dumper->Dump(&p); } return new Val(addl_pkt_dumper->IsError(), TYPE_BOOL); diff --git a/src/iosource/PktDumper.cc b/src/iosource/PktDumper.cc index 21ad79b87d..3c8595e11c 100644 --- a/src/iosource/PktDumper.cc +++ b/src/iosource/PktDumper.cc @@ -50,11 +50,6 @@ int PktDumper::HdrSize() const return is_open ? props.hdr_size : -1; } -bool PktDumper::Record(const Packet* pkt) - { - return Dump(pkt); - } - void PktDumper::Opened(const Properties& arg_props) { is_open = true; diff --git a/src/iosource/PktDumper.h b/src/iosource/PktDumper.h index 794992d870..85c64ffd1d 100644 --- a/src/iosource/PktDumper.h +++ b/src/iosource/PktDumper.h @@ -25,8 +25,10 @@ public: int HdrSize() const; bool Record(const Packet* pkt); + // PktSrc interface for derived classes to implement. virtual void Close() = 0; virtual void Open() = 0; + virtual bool Dump(const Packet* pkt) = 0; protected: // Methods to use by derived classed. @@ -41,10 +43,6 @@ protected: void Closed(); void Error(const std::string& msg); - // PktSrc interface for derived classes to implement. - - virtual bool Dump(const Packet* pkt) = 0; - private: bool is_open; Properties props; From ce9f16490c28155788928b89cc7a81a5931ea5f8 Mon Sep 17 00:00:00 2001 From: Robin Sommer Date: Fri, 22 Aug 2014 17:27:20 -0700 Subject: [PATCH 09/25] Moving some of the BPF filtering code into base class. This will allow packet sources that don't support BPF natively to emulate the filtering via libpcap. --- src/iosource/{pcap => }/BPF_Program.cc | 0 src/iosource/{pcap => }/BPF_Program.h | 0 src/iosource/CMakeLists.txt | 1 + src/iosource/PktSrc.cc | 59 ++++++++++++++++- src/iosource/PktSrc.h | 38 ++++++++++- src/iosource/pcap/CMakeLists.txt | 2 +- src/iosource/pcap/Source.cc | 89 +++++--------------------- src/iosource/pcap/Source.h | 6 -- 8 files changed, 110 insertions(+), 85 deletions(-) rename src/iosource/{pcap => }/BPF_Program.cc (100%) rename src/iosource/{pcap => }/BPF_Program.h (100%) diff --git a/src/iosource/pcap/BPF_Program.cc b/src/iosource/BPF_Program.cc similarity index 100% rename from src/iosource/pcap/BPF_Program.cc rename to src/iosource/BPF_Program.cc diff --git a/src/iosource/pcap/BPF_Program.h b/src/iosource/BPF_Program.h similarity index 100% rename from src/iosource/pcap/BPF_Program.h rename to src/iosource/BPF_Program.h diff --git a/src/iosource/CMakeLists.txt b/src/iosource/CMakeLists.txt index 6bcfa16289..272ced7740 100644 --- a/src/iosource/CMakeLists.txt +++ b/src/iosource/CMakeLists.txt @@ -7,6 +7,7 @@ include_directories(BEFORE ) set(iosource_SRCS + BPF_Program.cc Component.cc Manager.cc PktDumper.cc diff --git a/src/iosource/PktSrc.cc b/src/iosource/PktSrc.cc index 0ec04b65d8..2e056a10ba 100644 --- a/src/iosource/PktSrc.cc +++ b/src/iosource/PktSrc.cc @@ -25,6 +25,10 @@ PktSrc::PktSrc() PktSrc::~PktSrc() { + BPF_Program* code; + IterCookie* cookie = filters.InitForIteration(); + while ( (code = filters.NextEntry(cookie)) ) + delete code; } const std::string& PktSrc::Path() const @@ -43,6 +47,11 @@ int PktSrc::LinkType() const return IsOpen() ? props.link_type : -1; } +uint32 PktSrc::Netmask() const + { + return IsOpen() ? props.netmask : PCAP_NETMASK_UNKNOWN; + } + int PktSrc::HdrSize() const { return IsOpen() ? props.hdr_size : -1; @@ -77,6 +86,12 @@ void PktSrc::Opened(const Properties& arg_props) props = arg_props; SetClosed(false); + if ( ! PrecompileFilter(0, props.filter) || ! SetFilter(0) ) + { + Close(); + return; + } + DBG_LOG(DBG_PKTIO, "Opened source %s", props.path.c_str()); } @@ -409,12 +424,50 @@ int PktSrc::ExtractNextPacketInternal() return 0; } -int PktSrc::PrecompileFilter(int index, const std::string& filter) +int PktSrc::PrecompileBPFFilter(int index, const std::string& filter) { + char errbuf[PCAP_ERRBUF_SIZE]; + + // Compile filter. + BPF_Program* code = new BPF_Program(); + + if ( ! code->Compile(SnapLen(), LinkType(), filter.c_str(), Netmask(), errbuf, sizeof(errbuf)) ) + { + Error(fmt("cannot compile BPF filter \"%s\": %s", filter.c_str(), errbuf)); + Close(); + delete code; + return 0; + } + + // Store it in hash. + HashKey* hash = new HashKey(HashKey(bro_int_t(index))); + BPF_Program* oldcode = filters.Lookup(hash); + if ( oldcode ) + delete oldcode; + + filters.Insert(hash, code); + delete hash; + return 1; } -int PktSrc::SetFilter(int index) +BPF_Program* PktSrc::GetBPFFilter(int index) { - return 1; + HashKey* hash = new HashKey(HashKey(bro_int_t(index))); + BPF_Program* code = filters.Lookup(hash); + delete hash; + return code; + } + +int PktSrc::ApplyBPFFilter(int index, const struct pcap_pkthdr *hdr, const u_char *pkt) + { + BPF_Program* code = GetBPFFilter(index); + + if ( ! code ) + { + Error(fmt("BPF filter %d not compiled", index)); + Close(); + } + + return pcap_offline_filter(code->GetProgram(), hdr, pkt); } diff --git a/src/iosource/PktSrc.h b/src/iosource/PktSrc.h index d04ca24817..5dd51ab5f2 100644 --- a/src/iosource/PktSrc.h +++ b/src/iosource/PktSrc.h @@ -8,6 +8,10 @@ extern "C" { } #include "IOSource.h" +#include "BPF_Program.h" +#include "Dict.h" + +declare(PDict,BPF_Program); namespace iosource { @@ -29,6 +33,7 @@ public: const std::string& Filter() const; bool IsLive() const; int LinkType() const; + uint32 Netmask() const; const char* ErrorMsg() const; int HdrSize() const; int SnapLen() const; @@ -41,7 +46,22 @@ public: // going to be continued. void ContinueAfterSuspend(); - virtual void Statistics(Stats* stats) = 0; + // Precompiles a BPF filter and associates the given index with it. + // Returns true on success, 0 if a problem occurred. The compiled + // filter will be then available via GetBPFFilter*(. + int PrecompileBPFFilter(int index, const std::string& filter); + + // Returns the BPF filter with the given index, as compiled by + // PrecompileBPFFilter(), or null if none has been (successfully) + // compiled. + BPF_Program* GetBPFFilter(int index); + + // Applies a precompiled BPF filter to a packet, returning true if it + // maches. This will close the source with an error message if no + // filter with that index has been compiled. + int ApplyBPFFilter(int index, const struct pcap_pkthdr *hdr, const u_char *pkt); + + // PacketSource interace for derived classes to override. // Returns the packet last processed; false if there is no // current packet available. @@ -50,12 +70,15 @@ public: // Precompiles a filter and associates the given index with it. // Returns true on success, 0 if a problem occurred or filtering is // not supported. - virtual int PrecompileFilter(int index, const std::string& filter); + virtual int PrecompileFilter(int index, const std::string& filter) = 0; // Activates the filter with the given index. Returns true on // success, 0 if a problem occurred or the filtering is not // supported. - virtual int SetFilter(int index); + virtual int SetFilter(int index) = 0; + + // Returns current statistics about the source. + virtual void Statistics(Stats* stats) = 0; static int GetLinkHeaderSize(int link_type); @@ -68,7 +91,13 @@ protected: int selectable_fd; int link_type; int hdr_size; + uint32 netmask; bool is_live; + + Properties() + { + netmask = PCAP_NETMASK_UNKNOWN; + } }; struct Packet { @@ -113,6 +142,9 @@ private: bool have_packet; Packet current_packet; + // For BPF filtering support. + PDict(BPF_Program) filters; + // Only set in pseudo-realtime mode. double first_timestamp; double first_wallclock; diff --git a/src/iosource/pcap/CMakeLists.txt b/src/iosource/pcap/CMakeLists.txt index b43d51b0ca..1c57bb6ac9 100644 --- a/src/iosource/pcap/CMakeLists.txt +++ b/src/iosource/pcap/CMakeLists.txt @@ -4,5 +4,5 @@ include(BroPlugin) include_directories(BEFORE ${CMAKE_CURRENT_SOURCE_DIR} ${CMAKE_CURRENT_BINARY_DIR}) bro_plugin_begin(Bro Pcap) -bro_plugin_cc(Source.cc Dumper.cc BPF_Program.cc Plugin.cc) +bro_plugin_cc(Source.cc Dumper.cc Plugin.cc) bro_plugin_end() diff --git a/src/iosource/pcap/Source.cc b/src/iosource/pcap/Source.cc index 271c3efcd7..67a73fbd8a 100644 --- a/src/iosource/pcap/Source.cc +++ b/src/iosource/pcap/Source.cc @@ -37,13 +37,6 @@ void PcapSource::Close() if ( ! pd ) return; - BPF_Program* code; - IterCookie* cookie = filters.InitForIteration(); - while ( (code = filters.NextEntry(cookie)) ) - delete code; - - filters.Clear(); - pcap_close(pd); pd = 0; last_data = 0; @@ -56,10 +49,6 @@ void PcapSource::OpenLive() char errbuf[PCAP_ERRBUF_SIZE]; char tmp_errbuf[PCAP_ERRBUF_SIZE]; -#if 0 - filter_type = ft; -#endif - // Determine interface if not specified. if ( props.path.empty() ) props.path = pcap_lookupdev(tmp_errbuf); @@ -74,7 +63,7 @@ void PcapSource::OpenLive() // Determine network and netmask. uint32 net; - if ( pcap_lookupnet(props.path.c_str(), &net, &netmask, tmp_errbuf) < 0 ) + if ( pcap_lookupnet(props.path.c_str(), &net, &props.netmask, tmp_errbuf) < 0 ) { // ### The lookup can fail if no address is assigned to // the interface; and libpcap doesn't have any useful notion @@ -82,7 +71,7 @@ void PcapSource::OpenLive() // just kludge around the error :-(. // sprintf(errbuf, "pcap_lookupnet %s", tmp_errbuf); // return; - netmask = 0xffffff00; + props.netmask = 0xffffff00; } // We use the smallest time-out possible to return almost immediately if @@ -113,31 +102,22 @@ void PcapSource::OpenLive() props.selectable_fd = pcap_fileno(pd); - if ( PrecompileFilter(0, props.filter) && SetFilter(0) ) - { - SetHdrSize(); + SetHdrSize(); - if ( ! pd ) - // Was closed, couldn't get header size. - return; - - Info(fmt("listening on %s, capture length %d bytes\n", props.path.c_str(), SnapLen())); - } - else - Close(); + if ( ! pd ) + // Was closed, couldn't get header size. + return; props.is_live = true; Opened(props); + + Info(fmt("listening on %s, capture length %d bytes\n", props.path.c_str(), SnapLen())); } void PcapSource::OpenOffline() { char errbuf[PCAP_ERRBUF_SIZE]; -#if 0 - filter_type = ft; -#endif - pd = pcap_open_offline(props.path.c_str(), errbuf); if ( ! pd ) @@ -146,25 +126,16 @@ void PcapSource::OpenOffline() return; } - if ( PrecompileFilter(0, props.filter) && SetFilter(0) ) - { - SetHdrSize(); + SetHdrSize(); - if ( ! pd ) - // Was closed, unknown link layer type. - return; + if ( ! pd ) + // Was closed, unknown link layer type. + return; - // We don't put file sources into non-blocking mode as - // otherwise we would not be able to identify the EOF. + props.selectable_fd = fileno(pcap_file(pd)); - props.selectable_fd = fileno(pcap_file(pd)); - - if ( props.selectable_fd < 0 ) - InternalError("OS does not support selectable pcap fd"); - } - - else - Close(); + if ( props.selectable_fd < 0 ) + InternalError("OS does not support selectable pcap fd"); props.is_live = false; Opened(props); @@ -211,31 +182,7 @@ void PcapSource::DoneWithPacket(Packet* pkt) int PcapSource::PrecompileFilter(int index, const std::string& filter) { - if ( ! pd ) - return 1; // Prevent error message. - - char errbuf[PCAP_ERRBUF_SIZE]; - - // Compile filter. - BPF_Program* code = new BPF_Program(); - - if ( ! code->Compile(pd, filter.c_str(), netmask, errbuf, sizeof(errbuf)) ) - { - PcapError(); - delete code; - return 0; - } - - // Store it in hash. - HashKey* hash = new HashKey(HashKey(bro_int_t(index))); - BPF_Program* oldcode = filters.Lookup(hash); - if ( oldcode ) - delete oldcode; - - filters.Insert(hash, code); - delete hash; - - return 1; + return PktSrc::PrecompileBPFFilter(index, filter). } int PcapSource::SetFilter(int index) @@ -245,9 +192,7 @@ int PcapSource::SetFilter(int index) char errbuf[PCAP_ERRBUF_SIZE]; - HashKey* hash = new HashKey(HashKey(bro_int_t(index))); - BPF_Program* code = filters.Lookup(hash); - delete hash; + BPF_Program* code = GetFilter(index); if ( ! code ) { diff --git a/src/iosource/pcap/Source.h b/src/iosource/pcap/Source.h index 03b75c1ca7..b9b61ac618 100644 --- a/src/iosource/pcap/Source.h +++ b/src/iosource/pcap/Source.h @@ -4,10 +4,6 @@ #define IOSOURCE_PKTSRC_PCAP_SOURCE_H #include "../PktSrc.h" -#include "BPF_Program.h" -#include "Dict.h" - -declare(PDict,BPF_Program); namespace iosource { namespace pktsrc { @@ -42,8 +38,6 @@ private: Stats stats; pcap_t *pd; - uint32 netmask; - PDict(BPF_Program) filters; struct pcap_pkthdr current_hdr; struct pcap_pkthdr last_hdr; From 5f817513d098d2c0141ba74d48807a44d0131afa Mon Sep 17 00:00:00 2001 From: Robin Sommer Date: Fri, 22 Aug 2014 17:56:16 -0700 Subject: [PATCH 10/25] A set of various fixes and smaller API tweaks, plus tests. Also moving PCAP-related bifs to iosource/pcap.bif. --- TODO.iosources | 7 +- src/Net.cc | 25 ++-- src/bro.bif | 108 ------------------ src/iosource/BPF_Program.cc | 12 +- src/iosource/BPF_Program.h | 3 +- src/iosource/CMakeLists.txt | 4 + src/iosource/IOSource.h | 3 +- src/iosource/Manager.cc | 37 +++--- src/iosource/Manager.h | 3 + src/iosource/PktDumper.cc | 16 ++- src/iosource/PktDumper.h | 9 +- src/iosource/PktSrc.cc | 13 ++- src/iosource/PktSrc.h | 2 + src/iosource/pcap.bif | 104 +++++++++++++++++ src/iosource/pcap/Dumper.cc | 4 +- src/iosource/pcap/Dumper.h | 2 +- src/iosource/pcap/Plugin.cc | 4 +- src/iosource/pcap/Source.cc | 6 +- src/iosource/pcap/Source.h | 2 +- src/main.cc | 1 + .../btest/Baseline/core.pcap.dumper/output | 1 + .../core.pcap.dynamic-filter/conn.log | 25 ++++ .../Baseline/core.pcap.dynamic-filter/output | 30 +++++ .../Baseline/core.pcap.filter-error/output | 3 + .../Baseline/core.pcap.input-error/output2 | 2 + .../Baseline/core.pcap.pseudo-realtime/output | 1 + .../core.pcap.read-trace-with-filter/conn.log | 10 ++ .../packet_filter.log | 10 ++ testing/btest/core/pcap/dumper.bro | 5 + testing/btest/core/pcap/dynamic-filter.bro | 32 ++++++ testing/btest/core/pcap/filter-error.bro | 16 +++ testing/btest/core/pcap/input-error.bro | 14 +++ testing/btest/core/pcap/pseudo-realtime.bro | 42 +++++++ .../core/pcap/read-trace-with-filter.bro | 3 + 34 files changed, 395 insertions(+), 164 deletions(-) create mode 100644 src/iosource/pcap.bif create mode 100644 testing/btest/Baseline/core.pcap.dumper/output create mode 100644 testing/btest/Baseline/core.pcap.dynamic-filter/conn.log create mode 100644 testing/btest/Baseline/core.pcap.dynamic-filter/output create mode 100644 testing/btest/Baseline/core.pcap.filter-error/output create mode 100644 testing/btest/Baseline/core.pcap.input-error/output2 create mode 100644 testing/btest/Baseline/core.pcap.pseudo-realtime/output create mode 100644 testing/btest/Baseline/core.pcap.read-trace-with-filter/conn.log create mode 100644 testing/btest/Baseline/core.pcap.read-trace-with-filter/packet_filter.log create mode 100644 testing/btest/core/pcap/dumper.bro create mode 100644 testing/btest/core/pcap/dynamic-filter.bro create mode 100644 testing/btest/core/pcap/filter-error.bro create mode 100644 testing/btest/core/pcap/input-error.bro create mode 100644 testing/btest/core/pcap/pseudo-realtime.bro create mode 100644 testing/btest/core/pcap/read-trace-with-filter.bro diff --git a/TODO.iosources b/TODO.iosources index e61e858087..ee19dea169 100644 --- a/TODO.iosources +++ b/TODO.iosources @@ -1,3 +1,4 @@ -- Wrap BPF_Program into namespace and clean up -- Add an interface for derived pkt source to run a BPF filter. -- Tests, in particular the packet dumping needs testing. +- Tests + - pktsrc plugin + - pktdump plugin + diff --git a/src/Net.cc b/src/Net.cc index 38ec7c3b06..554aa890ba 100644 --- a/src/Net.cc +++ b/src/Net.cc @@ -116,8 +116,7 @@ RETSIGTYPE watchdog(int /* signo */) pkt_dumper = iosource_mgr->OpenPktDumper("watchdog-pkt.pcap", false); if ( ! pkt_dumper || pkt_dumper->IsError() ) { - reporter->Error("watchdog: can't open watchdog-pkt.pcap for writing\n"); - delete pkt_dumper; + reporter->Error("watchdog: can't open watchdog-pkt.pcap for writing"); pkt_dumper = 0; } } @@ -167,9 +166,9 @@ void net_init(name_list& interfaces, name_list& readfiles, iosource::PktSrc* ps = iosource_mgr->OpenPktSrc(readfiles[i], filter, false); assert(ps); - if ( ps->ErrorMsg() ) - reporter->FatalError("%s: problem with trace file %s - %s\n", - prog, readfiles[i], + if ( ! ps->IsOpen() ) + reporter->FatalError("problem with trace file %s (%s)", + readfiles[i], ps->ErrorMsg()); } } @@ -184,9 +183,9 @@ void net_init(name_list& interfaces, name_list& readfiles, iosource::PktSrc* ps = iosource_mgr->OpenPktSrc(interfaces[i], filter, true); assert(ps); - if ( ps->ErrorMsg() ) - reporter->FatalError("%s: problem with interface %s - %s\n", - prog, interfaces[i], + if ( ! ps->IsOpen() ) + reporter->FatalError("problem with interface %s (%s)", + interfaces[i], ps->ErrorMsg()); } } @@ -203,9 +202,9 @@ void net_init(name_list& interfaces, name_list& readfiles, pkt_dumper = iosource_mgr->OpenPktDumper(writefile, false); assert(pkt_dumper); - if ( pkt_dumper->ErrorMsg().size() ) - reporter->FatalError("problem opening dump file %s - %s\n", - writefile, pkt_dumper->ErrorMsg().c_str()); + if ( ! pkt_dumper->IsOpen() ) + reporter->FatalError("problem opening dump file %s (%s)", + writefile, pkt_dumper->ErrorMsg()); ID* id = global_scope()->Lookup("trace_output_file"); if ( ! id ) @@ -409,7 +408,7 @@ void net_get_final_stats() { iosource::PktSrc::Stats s; ps->Statistics(&s); - reporter->Info("%d packets received on interface %s, %d dropped\n", + reporter->Info("%d packets received on interface %s, %d dropped", s.received, ps->Path().c_str(), s.dropped); } } @@ -430,8 +429,6 @@ void net_finish(int drain_events) sessions->Done(); } - delete pkt_dumper; - #ifdef DEBUG extern int reassem_seen_bytes, reassem_copied_bytes; // DEBUG_MSG("Reassembly (TCP and IP/Frag): %d bytes seen, %d bytes copied\n", diff --git a/src/bro.bif b/src/bro.bif index adf5e3d6f0..1255f05f50 100644 --- a/src/bro.bif +++ b/src/bro.bif @@ -4228,114 +4228,6 @@ function enable_raw_output%(f: file%): any # # =========================================================================== -## Precompiles a PCAP filter and binds it to a given identifier. -## -## id: The PCAP identifier to reference the filter *s* later on. -## -## s: The PCAP filter. See ``man tcpdump`` for valid expressions. -## -## Returns: True if *s* is valid and precompiles successfully. -## -## .. bro:see:: install_pcap_filter -## install_src_addr_filter -## install_src_net_filter -## uninstall_src_addr_filter -## uninstall_src_net_filter -## install_dst_addr_filter -## install_dst_net_filter -## uninstall_dst_addr_filter -## uninstall_dst_net_filter -## pcap_error -function precompile_pcap_filter%(id: PcapFilterID, s: string%): bool - %{ - bool success = true; - - const iosource::Manager::PktSrcList& pkt_srcs(iosource_mgr->GetPktSrcs()); - - for ( iosource::Manager::PktSrcList::const_iterator i = pkt_srcs.begin(); - i != pkt_srcs.end(); i++ ) - { - iosource::PktSrc* ps = *i; - - if ( ! ps->PrecompileFilter(id->ForceAsInt(), - s->CheckString()) ) - { - reporter->Error("precompile_pcap_filter: %s", - ps->ErrorMsg()); - success = false; - } - } - - return new Val(success, TYPE_BOOL); - %} - -## Installs a PCAP filter that has been precompiled with -## :bro:id:`precompile_pcap_filter`. -## -## id: The PCAP filter id of a precompiled filter. -## -## Returns: True if the filter associated with *id* has been installed -## successfully. -## -## .. bro:see:: precompile_pcap_filter -## install_src_addr_filter -## install_src_net_filter -## uninstall_src_addr_filter -## uninstall_src_net_filter -## install_dst_addr_filter -## install_dst_net_filter -## uninstall_dst_addr_filter -## uninstall_dst_net_filter -## pcap_error -function install_pcap_filter%(id: PcapFilterID%): bool - %{ - bool success = true; - - const iosource::Manager::PktSrcList& pkt_srcs(iosource_mgr->GetPktSrcs()); - - for ( iosource::Manager::PktSrcList::const_iterator i = pkt_srcs.begin(); - i != pkt_srcs.end(); i++ ) - { - iosource::PktSrc* ps = *i; - - if ( ! ps->SetFilter(id->ForceAsInt()) ) - success = false; - } - - return new Val(success, TYPE_BOOL); - %} - -## Returns a string representation of the last PCAP error. -## -## Returns: A descriptive error message of the PCAP function that failed. -## -## .. bro:see:: precompile_pcap_filter -## install_pcap_filter -## install_src_addr_filter -## install_src_net_filter -## uninstall_src_addr_filter -## uninstall_src_net_filter -## install_dst_addr_filter -## install_dst_net_filter -## uninstall_dst_addr_filter -## uninstall_dst_net_filter -function pcap_error%(%): string - %{ - const iosource::Manager::PktSrcList& pkt_srcs(iosource_mgr->GetPktSrcs()); - - for ( iosource::Manager::PktSrcList::const_iterator i = pkt_srcs.begin(); - i != pkt_srcs.end(); i++ ) - { - iosource::PktSrc* ps = *i; - - const char* err = ps->ErrorMsg(); - if ( *err ) - return new StringVal(err); - } - - return new StringVal("no error"); - %} - ## Installs a filter to drop packets from a given IP source address with ## a certain probability if none of a given set of TCP flags are set. ## Note that for IPv6 packets with a Destination options header that has diff --git a/src/iosource/BPF_Program.cc b/src/iosource/BPF_Program.cc index 5260429eb0..8df7729ad1 100644 --- a/src/iosource/BPF_Program.cc +++ b/src/iosource/BPF_Program.cc @@ -91,7 +91,8 @@ bool BPF_Program::Compile(pcap_t* pcap, const char* filter, uint32 netmask, } bool BPF_Program::Compile(int snaplen, int linktype, const char* filter, - uint32 netmask, char* errbuf, bool optimize) + uint32 netmask, char* errbuf, unsigned int errbuf_len, + bool optimize) { FreeCode(); @@ -99,13 +100,18 @@ bool BPF_Program::Compile(int snaplen, int linktype, const char* filter, char my_error[PCAP_ERRBUF_SIZE]; int err = pcap_compile_nopcap(snaplen, linktype, &m_program, - (char *) filter, optimize, netmask, error); + (char *) filter, optimize, netmask, my_error); if ( err < 0 && errbuf ) - safe_strncpy(errbuf, my_errbuf, PCAP_ERRBUF_SIZE); + safe_strncpy(errbuf, my_error, errbuf_len); + *errbuf = '\0'; #else int err = pcap_compile_nopcap(snaplen, linktype, &m_program, (char*) filter, optimize, netmask); + + if ( err < 0 && errbuf && errbuf_len ) + *errbuf = '\0'; #endif + if ( err == 0 ) m_compiled = true; diff --git a/src/iosource/BPF_Program.h b/src/iosource/BPF_Program.h index 88ed669da2..3efa212bbc 100644 --- a/src/iosource/BPF_Program.h +++ b/src/iosource/BPF_Program.h @@ -30,7 +30,8 @@ public: // similarly to pcap_compile_nopcap(). Parameters are // similar. Returns true on success. bool Compile(int snaplen, int linktype, const char* filter, - uint32 netmask, char* errbuf = 0, bool optimize = true); + uint32 netmask, char* errbuf = 0, unsigned int errbuf_len = 0, + bool optimize = true); // Returns true if this program currently contains compiled // code, false otherwise. diff --git a/src/iosource/CMakeLists.txt b/src/iosource/CMakeLists.txt index 272ced7740..a36667aee7 100644 --- a/src/iosource/CMakeLists.txt +++ b/src/iosource/CMakeLists.txt @@ -6,6 +6,8 @@ include_directories(BEFORE ${CMAKE_CURRENT_BINARY_DIR} ) +add_subdirectory(pcap) + set(iosource_SRCS BPF_Program.cc Component.cc @@ -14,6 +16,8 @@ set(iosource_SRCS PktSrc.cc ) +bif_target(pcap.bif) + bro_add_subdir_library(iosource ${iosource_SRCS}) add_dependencies(bro_iosource generate_outputs) diff --git a/src/iosource/IOSource.h b/src/iosource/IOSource.h index 3419152a9a..9083e8f4f1 100644 --- a/src/iosource/IOSource.h +++ b/src/iosource/IOSource.h @@ -14,7 +14,7 @@ namespace iosource { */ class IOSource { public: - IOSource() { idle = closed = false; } + IOSource() { idle = false; closed = false; } virtual ~IOSource() {} // Returns true if source has nothing ready to process. @@ -57,7 +57,6 @@ protected: // Derived classed are to set this to true if they have gone dry // temporarily. void SetIdle(bool is_idle) { idle = is_idle; } - // Derived classed are to set this to true if they have gone dry // temporarily. diff --git a/src/iosource/Manager.cc b/src/iosource/Manager.cc index e2be297a77..63b6888801 100644 --- a/src/iosource/Manager.cc +++ b/src/iosource/Manager.cc @@ -22,11 +22,19 @@ Manager::~Manager() { for ( SourceList::iterator i = sources.begin(); i != sources.end(); ++i ) { - (*i)->src->Done(); + // ??? (*i)->src->Done(); delete *i; } sources.clear(); + + for ( PktDumperList::iterator i = pkt_dumpers.begin(); i != pkt_dumpers.end(); ++i ) + { + (*i)->Done(); + delete *i; + } + + pkt_dumpers.clear(); } void Manager::RemoveAll() @@ -43,6 +51,7 @@ IOSource* Manager::FindSoonest(double* ts) i != sources.end(); ++i ) if ( ! (*i)->src->IsOpen() ) { + (*i)->src->Done(); delete *i; sources.erase(i); break; @@ -246,15 +255,11 @@ PktSrc* Manager::OpenPktSrc(const std::string& path, const std::string& filter, // Instantiate packet source. PktSrc* ps = (*component->Factory())(npath, filter, is_live); + assert(ps); - if ( ! (ps && ps->IsOpen()) ) - { - string type = (is_live ? "interface" : "trace file"); - string pserr = ps->ErrorMsg() ? (string(" - ") + ps->ErrorMsg()) : ""; - - reporter->FatalError("%s: problem with %s %s%s", - prog, npath.c_str(), type.c_str(), pserr.c_str()); - } + if ( ! ps->IsOpen() && ps->ErrorMsg() ) + // Set an error message if it didn't open successfully. + ps->Error("could not open"); DBG_LOG(DBG_PKTIO, "Created packet source of type %s for %s", component->Name().c_str(), npath.c_str()); @@ -291,16 +296,16 @@ PktDumper* Manager::OpenPktDumper(const string& path, bool append) // Instantiate packet dumper. PktDumper* pd = (*component->Factory())(npath, append); + assert(pd); - if ( ! (pd && pd->IsOpen()) ) - { - string pderr = pd->ErrorMsg().size() ? (string(" - ") + pd->ErrorMsg()) : ""; - - reporter->FatalError("%s: can't open write file \"%s\"%s", - prog, npath.c_str(), pderr.c_str()); - } + if ( ! pd->IsOpen() && pd->ErrorMsg() ) + // Set an error message if it didn't open successfully. + pd->Error("could not open"); DBG_LOG(DBG_PKTIO, "Created packer dumper of type %s for %s", component->Name().c_str(), npath.c_str()); + pd->Init(); + pkt_dumpers.push_back(pd); + return pd; } diff --git a/src/iosource/Manager.h b/src/iosource/Manager.h index 5a3a58d798..4198c73680 100644 --- a/src/iosource/Manager.h +++ b/src/iosource/Manager.h @@ -64,7 +64,10 @@ protected: typedef std::list SourceList; SourceList sources; + typedef std::list PktDumperList; + PktSrcList pkt_srcs; + PktDumperList pkt_dumpers; }; } diff --git a/src/iosource/PktDumper.cc b/src/iosource/PktDumper.cc index 3c8595e11c..a4bc3a82f8 100644 --- a/src/iosource/PktDumper.cc +++ b/src/iosource/PktDumper.cc @@ -20,6 +20,16 @@ PktDumper::~PktDumper() { } +void PktDumper::Init() + { + Open(); + } + +void PktDumper::Done() + { + Close(); + } + const std::string& PktDumper::Path() const { return props.path; @@ -40,9 +50,9 @@ bool PktDumper::IsError() const return errmsg.size(); } -const std::string& PktDumper::ErrorMsg() const +const char* PktDumper::ErrorMsg() const { - return errmsg; + return errmsg.size() ? errmsg.c_str() : 0; } int PktDumper::HdrSize() const @@ -60,8 +70,8 @@ void PktDumper::Opened(const Properties& arg_props) void PktDumper::Closed() { is_open = false; - props.path = ""; DBG_LOG(DBG_PKTIO, "Closed dumper %s", props.path.c_str()); + props.path = ""; } void PktDumper::Error(const std::string& msg) diff --git a/src/iosource/PktDumper.h b/src/iosource/PktDumper.h index 85c64ffd1d..d8201f977c 100644 --- a/src/iosource/PktDumper.h +++ b/src/iosource/PktDumper.h @@ -21,16 +21,18 @@ public: bool IsOpen() const; double OpenTime() const; bool IsError() const; - const std::string& ErrorMsg() const; + const char* ErrorMsg() const; int HdrSize() const; bool Record(const Packet* pkt); - // PktSrc interface for derived classes to implement. + // PktDumper interface for derived classes to implement. virtual void Close() = 0; virtual void Open() = 0; virtual bool Dump(const Packet* pkt) = 0; protected: + friend class Manager; + // Methods to use by derived classed. // struct Properties { @@ -39,6 +41,9 @@ protected: double open_time; }; + void Init(); + void Done(); + void Opened(const Properties& props); void Closed(); void Error(const std::string& msg); diff --git a/src/iosource/PktSrc.cc b/src/iosource/PktSrc.cc index 2e056a10ba..30f58a5b3f 100644 --- a/src/iosource/PktSrc.cc +++ b/src/iosource/PktSrc.cc @@ -17,6 +17,7 @@ PktSrc::PktSrc() { have_packet = false; errbuf = ""; + SetClosed(true); next_sync_point = 0; first_timestamp = 0.0; @@ -195,7 +196,8 @@ void PktSrc::Init() void PktSrc::Done() { - Close(); + if ( IsOpen() ) + Close(); } void PktSrc::GetFds(int* read, int* write, int* except) @@ -433,8 +435,13 @@ int PktSrc::PrecompileBPFFilter(int index, const std::string& filter) if ( ! code->Compile(SnapLen(), LinkType(), filter.c_str(), Netmask(), errbuf, sizeof(errbuf)) ) { - Error(fmt("cannot compile BPF filter \"%s\": %s", filter.c_str(), errbuf)); - Close(); + string msg = fmt("cannot compile BPF filter \"%s\"", filter.c_str()); + + if ( *errbuf ) + msg += ": " + string(errbuf); + + Error(msg); + delete code; return 0; } diff --git a/src/iosource/PktSrc.h b/src/iosource/PktSrc.h index 5dd51ab5f2..72e1a0da8c 100644 --- a/src/iosource/PktSrc.h +++ b/src/iosource/PktSrc.h @@ -83,6 +83,8 @@ public: static int GetLinkHeaderSize(int link_type); protected: + friend class Manager; + // Methods to use by derived classes. struct Properties { diff --git a/src/iosource/pcap.bif b/src/iosource/pcap.bif new file mode 100644 index 0000000000..ee4e1e6c06 --- /dev/null +++ b/src/iosource/pcap.bif @@ -0,0 +1,104 @@ + +## Precompiles a PCAP filter and binds it to a given identifier. +## +## id: The PCAP identifier to reference the filter *s* later on. +## +## s: The PCAP filter. See ``man tcpdump`` for valid expressions. +## +## Returns: True if *s* is valid and precompiles successfully. +## +## .. bro:see:: install_pcap_filter +## install_src_addr_filter +## install_src_net_filter +## uninstall_src_addr_filter +## uninstall_src_net_filter +## install_dst_addr_filter +## install_dst_net_filter +## uninstall_dst_addr_filter +## uninstall_dst_net_filter +## pcap_error +function precompile_pcap_filter%(id: PcapFilterID, s: string%): bool + %{ + bool success = true; + + const iosource::Manager::PktSrcList& pkt_srcs(iosource_mgr->GetPktSrcs()); + + for ( iosource::Manager::PktSrcList::const_iterator i = pkt_srcs.begin(); + i != pkt_srcs.end(); i++ ) + { + iosource::PktSrc* ps = *i; + + if ( ! ps->PrecompileFilter(id->ForceAsInt(), + s->CheckString()) ) + success = false; + } + + return new Val(success, TYPE_BOOL); + %} + +## Installs a PCAP filter that has been precompiled with +## :bro:id:`precompile_pcap_filter`. +## +## id: The PCAP filter id of a precompiled filter. +## +## Returns: True if the filter associated with *id* has been installed +## successfully. +## +## .. bro:see:: precompile_pcap_filter +## install_src_addr_filter +## install_src_net_filter +## uninstall_src_addr_filter +## uninstall_src_net_filter +## install_dst_addr_filter +## install_dst_net_filter +## uninstall_dst_addr_filter +## uninstall_dst_net_filter +## pcap_error +function install_pcap_filter%(id: PcapFilterID%): bool + %{ + bool success = true; + + const iosource::Manager::PktSrcList& pkt_srcs(iosource_mgr->GetPktSrcs()); + + for ( iosource::Manager::PktSrcList::const_iterator i = pkt_srcs.begin(); + i != pkt_srcs.end(); i++ ) + { + iosource::PktSrc* ps = *i; + + if ( ! ps->SetFilter(id->ForceAsInt()) ) + success = false; + } + + return new Val(success, TYPE_BOOL); + %} + +## Returns a string representation of the last PCAP error. +## +## Returns: A descriptive error message of the PCAP function that failed. +## +## .. bro:see:: precompile_pcap_filter +## install_pcap_filter +## install_src_addr_filter +## install_src_net_filter +## uninstall_src_addr_filter +## uninstall_src_net_filter +## install_dst_addr_filter +## install_dst_net_filter +## uninstall_dst_addr_filter +## uninstall_dst_net_filter +function pcap_error%(%): string + %{ + const iosource::Manager::PktSrcList& pkt_srcs(iosource_mgr->GetPktSrcs()); + + for ( iosource::Manager::PktSrcList::const_iterator i = pkt_srcs.begin(); + i != pkt_srcs.end(); i++ ) + { + iosource::PktSrc* ps = *i; + + const char* err = ps->ErrorMsg(); + if ( *err ) + return new StringVal(err); + } + + return new StringVal("no error"); + %} diff --git a/src/iosource/pcap/Dumper.cc b/src/iosource/pcap/Dumper.cc index 2210e08e64..5d0b5e599b 100644 --- a/src/iosource/pcap/Dumper.cc +++ b/src/iosource/pcap/Dumper.cc @@ -5,9 +5,9 @@ #include "Dumper.h" #include "../PktSrc.h" -#include "../../../Net.h" +#include "../../Net.h" -using namespace iosource::pktsrc; +using namespace iosource::pcap; PcapDumper::PcapDumper(const std::string& path, bool arg_append) { diff --git a/src/iosource/pcap/Dumper.h b/src/iosource/pcap/Dumper.h index c2762a2b04..8013afcb8e 100644 --- a/src/iosource/pcap/Dumper.h +++ b/src/iosource/pcap/Dumper.h @@ -9,7 +9,7 @@ extern "C" { #include "../PktDumper.h" namespace iosource { -namespace pktsrc { +namespace pcap { class PcapDumper : public PktDumper { public: diff --git a/src/iosource/pcap/Plugin.cc b/src/iosource/pcap/Plugin.cc index a412e4f650..f0490e6e3d 100644 --- a/src/iosource/pcap/Plugin.cc +++ b/src/iosource/pcap/Plugin.cc @@ -12,8 +12,8 @@ class Plugin : public plugin::Plugin { public: plugin::Configuration Configure() { - AddComponent(new ::iosource::pktsrc::SourceComponent("PcapReader", "pcap", ::iosource::pktsrc::SourceComponent::BOTH, ::iosource::pktsrc::PcapSource::Instantiate)); - AddComponent(new ::iosource::pktsrc::DumperComponent("PcapWriter", "pcap", ::iosource::pktsrc::PcapDumper::Instantiate)); + AddComponent(new ::iosource::PktSrcComponent("PcapReader", "pcap", ::iosource::PktSrcComponent::BOTH, ::iosource::pcap::PcapSource::Instantiate)); + AddComponent(new ::iosource::PktDumperComponent("PcapWriter", "pcap", ::iosource::pcap::PcapDumper::Instantiate)); plugin::Configuration config; config.name = "Bro::Pcap"; diff --git a/src/iosource/pcap/Source.cc b/src/iosource/pcap/Source.cc index 67a73fbd8a..79ded790bd 100644 --- a/src/iosource/pcap/Source.cc +++ b/src/iosource/pcap/Source.cc @@ -9,7 +9,7 @@ #include #endif -using namespace iosource::pktsrc; +using namespace iosource::pcap; PcapSource::~PcapSource() { @@ -182,7 +182,7 @@ void PcapSource::DoneWithPacket(Packet* pkt) int PcapSource::PrecompileFilter(int index, const std::string& filter) { - return PktSrc::PrecompileBPFFilter(index, filter). + return PktSrc::PrecompileBPFFilter(index, filter); } int PcapSource::SetFilter(int index) @@ -192,7 +192,7 @@ int PcapSource::SetFilter(int index) char errbuf[PCAP_ERRBUF_SIZE]; - BPF_Program* code = GetFilter(index); + BPF_Program* code = GetBPFFilter(index); if ( ! code ) { diff --git a/src/iosource/pcap/Source.h b/src/iosource/pcap/Source.h index b9b61ac618..039bdec81a 100644 --- a/src/iosource/pcap/Source.h +++ b/src/iosource/pcap/Source.h @@ -6,7 +6,7 @@ #include "../PktSrc.h" namespace iosource { -namespace pktsrc { +namespace pcap { class PcapSource : public iosource::PktSrc { public: diff --git a/src/main.cc b/src/main.cc index 5066ef85ee..295fa79ee7 100644 --- a/src/main.cc +++ b/src/main.cc @@ -390,6 +390,7 @@ void terminate_bro() delete plugin_mgr; delete thread_mgr; delete reporter; + delete iosource_mgr; reporter = 0; } diff --git a/testing/btest/Baseline/core.pcap.dumper/output b/testing/btest/Baseline/core.pcap.dumper/output new file mode 100644 index 0000000000..1055e73ebe --- /dev/null +++ b/testing/btest/Baseline/core.pcap.dumper/output @@ -0,0 +1 @@ +00000010 ff ff 00 00 01 00 00 00 1d a2 b2 4e 73 00 07 00 | | 00000010 00 20 00 00 01 00 00 00 1d a2 b2 4e 73 00 07 00 | diff --git a/testing/btest/Baseline/core.pcap.dynamic-filter/conn.log b/testing/btest/Baseline/core.pcap.dynamic-filter/conn.log new file mode 100644 index 0000000000..f42999c4fa --- /dev/null +++ b/testing/btest/Baseline/core.pcap.dynamic-filter/conn.log @@ -0,0 +1,25 @@ +#separator \x09 +#set_separator , +#empty_field (empty) +#unset_field - +#path conn +#open 2014-08-24-15-51-55 +#fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p proto service duration orig_bytes resp_bytes conn_state local_orig missed_bytes history orig_pkts orig_ip_bytes resp_pkts resp_ip_bytes tunnel_parents +#types time string addr port addr port enum string interval count count string bool count string count count count count set[string] +1300475167.096535 CXWv6p3arKYeMETxOg 141.142.220.202 5353 224.0.0.251 5353 udp dns - - - S0 - 0 D 1 73 0 0 (empty) +1300475168.853899 CCvvfg3TEfuqmmG4bh 141.142.220.118 43927 141.142.2.2 53 udp dns 0.000435 38 89 SF - 0 Dd 1 66 1 117 (empty) +1300475168.854378 CsRx2w45OKnoww6xl4 141.142.220.118 37676 141.142.2.2 53 udp dns 0.000420 52 99 SF - 0 Dd 1 80 1 127 (empty) +1300475168.854837 CRJuHdVW0XPVINV8a 141.142.220.118 40526 141.142.2.2 53 udp dns 0.000392 38 183 SF - 0 Dd 1 66 1 211 (empty) +1300475168.857956 CPbrpk1qSsw6ESzHV4 141.142.220.118 32902 141.142.2.2 53 udp dns 0.000317 38 89 SF - 0 Dd 1 66 1 117 (empty) +1300475168.858306 C6pKV8GSxOnSLghOa 141.142.220.118 59816 141.142.2.2 53 udp dns 0.000343 52 99 SF - 0 Dd 1 80 1 127 (empty) +1300475168.858713 CIPOse170MGiRM1Qf4 141.142.220.118 59714 141.142.2.2 53 udp dns 0.000375 38 183 SF - 0 Dd 1 66 1 211 (empty) +1300475168.891644 C7XEbhP654jzLoe3a 141.142.220.118 58206 141.142.2.2 53 udp dns 0.000339 38 89 SF - 0 Dd 1 66 1 117 (empty) +1300475168.892037 CJ3xTn1c4Zw9TmAE05 141.142.220.118 38911 141.142.2.2 53 udp dns 0.000335 52 99 SF - 0 Dd 1 80 1 127 (empty) +1300475168.892414 CMXxB5GvmoxJFXdTa 141.142.220.118 59746 141.142.2.2 53 udp dns 0.000421 38 183 SF - 0 Dd 1 66 1 211 (empty) +1300475168.893988 Caby8b1slFea8xwSmb 141.142.220.118 45000 141.142.2.2 53 udp dns 0.000384 38 89 SF - 0 Dd 1 66 1 117 (empty) +1300475168.894422 Che1bq3i2rO3KD1Syg 141.142.220.118 48479 141.142.2.2 53 udp dns 0.000317 52 99 SF - 0 Dd 1 80 1 127 (empty) +1300475168.894787 C3SfNE4BWaU4aSuwkc 141.142.220.118 48128 141.142.2.2 53 udp dns 0.000423 38 183 SF - 0 Dd 1 66 1 211 (empty) +1300475168.901749 CEle3f3zno26fFZkrh 141.142.220.118 56056 141.142.2.2 53 udp dns 0.000402 36 131 SF - 0 Dd 1 64 1 159 (empty) +1300475168.902195 CwSkQu4eWZCH7OONC1 141.142.220.118 55092 141.142.2.2 53 udp dns 0.000374 36 198 SF - 0 Dd 1 64 1 226 (empty) +1300475168.652003 CjhGID4nQcgTWjvg4c 141.142.220.118 35634 208.80.152.2 80 tcp - - - - OTH - 0 D 1 515 0 0 (empty) +#close 2014-08-24-15-51-55 diff --git a/testing/btest/Baseline/core.pcap.dynamic-filter/output b/testing/btest/Baseline/core.pcap.dynamic-filter/output new file mode 100644 index 0000000000..e8be0b0195 --- /dev/null +++ b/testing/btest/Baseline/core.pcap.dynamic-filter/output @@ -0,0 +1,30 @@ +1, [orig_h=141.142.220.202, orig_p=5353/udp, resp_h=224.0.0.251, resp_p=5353/udp] +2, [orig_h=141.142.220.118, orig_p=35634/tcp, resp_h=208.80.152.2, resp_p=80/tcp] +3, [orig_h=141.142.220.118, orig_p=43927/udp, resp_h=141.142.2.2, resp_p=53/udp] +4, [orig_h=141.142.220.118, orig_p=43927/udp, resp_h=141.142.2.2, resp_p=53/udp] +5, [orig_h=141.142.220.118, orig_p=37676/udp, resp_h=141.142.2.2, resp_p=53/udp] +6, [orig_h=141.142.220.118, orig_p=37676/udp, resp_h=141.142.2.2, resp_p=53/udp] +7, [orig_h=141.142.220.118, orig_p=40526/udp, resp_h=141.142.2.2, resp_p=53/udp] +8, [orig_h=141.142.220.118, orig_p=40526/udp, resp_h=141.142.2.2, resp_p=53/udp] +9, [orig_h=141.142.220.118, orig_p=32902/udp, resp_h=141.142.2.2, resp_p=53/udp] +10, [orig_h=141.142.220.118, orig_p=32902/udp, resp_h=141.142.2.2, resp_p=53/udp] +11, [orig_h=141.142.220.118, orig_p=59816/udp, resp_h=141.142.2.2, resp_p=53/udp] +12, [orig_h=141.142.220.118, orig_p=59816/udp, resp_h=141.142.2.2, resp_p=53/udp] +13, [orig_h=141.142.220.118, orig_p=59714/udp, resp_h=141.142.2.2, resp_p=53/udp] +14, [orig_h=141.142.220.118, orig_p=59714/udp, resp_h=141.142.2.2, resp_p=53/udp] +15, [orig_h=141.142.220.118, orig_p=58206/udp, resp_h=141.142.2.2, resp_p=53/udp] +16, [orig_h=141.142.220.118, orig_p=58206/udp, resp_h=141.142.2.2, resp_p=53/udp] +17, [orig_h=141.142.220.118, orig_p=38911/udp, resp_h=141.142.2.2, resp_p=53/udp] +18, [orig_h=141.142.220.118, orig_p=38911/udp, resp_h=141.142.2.2, resp_p=53/udp] +19, [orig_h=141.142.220.118, orig_p=59746/udp, resp_h=141.142.2.2, resp_p=53/udp] +20, [orig_h=141.142.220.118, orig_p=59746/udp, resp_h=141.142.2.2, resp_p=53/udp] +21, [orig_h=141.142.220.118, orig_p=45000/udp, resp_h=141.142.2.2, resp_p=53/udp] +22, [orig_h=141.142.220.118, orig_p=45000/udp, resp_h=141.142.2.2, resp_p=53/udp] +23, [orig_h=141.142.220.118, orig_p=48479/udp, resp_h=141.142.2.2, resp_p=53/udp] +24, [orig_h=141.142.220.118, orig_p=48479/udp, resp_h=141.142.2.2, resp_p=53/udp] +25, [orig_h=141.142.220.118, orig_p=48128/udp, resp_h=141.142.2.2, resp_p=53/udp] +26, [orig_h=141.142.220.118, orig_p=48128/udp, resp_h=141.142.2.2, resp_p=53/udp] +27, [orig_h=141.142.220.118, orig_p=56056/udp, resp_h=141.142.2.2, resp_p=53/udp] +28, [orig_h=141.142.220.118, orig_p=56056/udp, resp_h=141.142.2.2, resp_p=53/udp] +29, [orig_h=141.142.220.118, orig_p=55092/udp, resp_h=141.142.2.2, resp_p=53/udp] +30, [orig_h=141.142.220.118, orig_p=55092/udp, resp_h=141.142.2.2, resp_p=53/udp] diff --git a/testing/btest/Baseline/core.pcap.filter-error/output b/testing/btest/Baseline/core.pcap.filter-error/output new file mode 100644 index 0000000000..82804bb483 --- /dev/null +++ b/testing/btest/Baseline/core.pcap.filter-error/output @@ -0,0 +1,3 @@ +fatal error in /home/robin/bro/master/scripts/base/frameworks/packet-filter/./main.bro, line 282: Bad pcap filter 'kaputt' +---- +error, cannot compile BPF filter "kaputt, too" diff --git a/testing/btest/Baseline/core.pcap.input-error/output2 b/testing/btest/Baseline/core.pcap.input-error/output2 new file mode 100644 index 0000000000..74666797b9 --- /dev/null +++ b/testing/btest/Baseline/core.pcap.input-error/output2 @@ -0,0 +1,2 @@ +fatal error: problem with interface NO_SUCH_INTERFACE +fatal error: problem with trace file NO_SUCH_TRACE (NO_SUCH_TRACE: No such file or directory) diff --git a/testing/btest/Baseline/core.pcap.pseudo-realtime/output b/testing/btest/Baseline/core.pcap.pseudo-realtime/output new file mode 100644 index 0000000000..d708959ce9 --- /dev/null +++ b/testing/btest/Baseline/core.pcap.pseudo-realtime/output @@ -0,0 +1 @@ +real time matches trace time diff --git a/testing/btest/Baseline/core.pcap.read-trace-with-filter/conn.log b/testing/btest/Baseline/core.pcap.read-trace-with-filter/conn.log new file mode 100644 index 0000000000..8522d69ae6 --- /dev/null +++ b/testing/btest/Baseline/core.pcap.read-trace-with-filter/conn.log @@ -0,0 +1,10 @@ +#separator \x09 +#set_separator , +#empty_field (empty) +#unset_field - +#path conn +#open 2014-08-23-18-29-48 +#fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p proto service duration orig_bytes resp_bytes conn_state local_orig missed_bytes history orig_pkts orig_ip_bytes resp_pkts resp_ip_bytes tunnel_parents +#types time string addr port addr port enum string interval count count string bool count string count count count count set[string] +1300475168.892936 CXWv6p3arKYeMETxOg 141.142.220.118 50000 208.80.152.3 80 tcp http 0.229603 1148 734 S1 - 0 ShADad 6 1468 4 950 (empty) +#close 2014-08-23-18-29-48 diff --git a/testing/btest/Baseline/core.pcap.read-trace-with-filter/packet_filter.log b/testing/btest/Baseline/core.pcap.read-trace-with-filter/packet_filter.log new file mode 100644 index 0000000000..75b09c608a --- /dev/null +++ b/testing/btest/Baseline/core.pcap.read-trace-with-filter/packet_filter.log @@ -0,0 +1,10 @@ +#separator \x09 +#set_separator , +#empty_field (empty) +#unset_field - +#path packet_filter +#open 2014-08-23-18-29-48 +#fields ts node filter init success +#types time string string bool bool +1408818588.510297 bro port 50000 T T +#close 2014-08-23-18-29-48 diff --git a/testing/btest/core/pcap/dumper.bro b/testing/btest/core/pcap/dumper.bro new file mode 100644 index 0000000000..9cf5cbe71c --- /dev/null +++ b/testing/btest/core/pcap/dumper.bro @@ -0,0 +1,5 @@ +# @TEST-EXEC: bro -r $TRACES/workshop_2011_browse.trace -w dump +# @TEST-EXEC: hexdump -C $TRACES/workshop_2011_browse.trace >1 +# @TEST-EXEC: hexdump -C dump >2 +# @TEST-EXEC: sdiff -s 1 2 >output || true +# @TEST-EXEC: btest-diff output diff --git a/testing/btest/core/pcap/dynamic-filter.bro b/testing/btest/core/pcap/dynamic-filter.bro new file mode 100644 index 0000000000..012858fa65 --- /dev/null +++ b/testing/btest/core/pcap/dynamic-filter.bro @@ -0,0 +1,32 @@ +# @TEST-EXEC: bro -C -r $TRACES/wikipedia.trace %INPUT >output +# @TEST-EXEC: btest-diff output +# @TEST-EXEC: btest-diff conn.log + +redef enum PcapFilterID += { A, B }; + +global cnt = 0; + +event new_packet(c: connection, p: pkt_hdr) + { + ++cnt; + + print cnt, c$id; + + if ( cnt == 1 ) + if ( ! install_pcap_filter(A) ) + print "error 3"; + + if ( cnt == 2 ) + if ( ! install_pcap_filter(B) ) + print "error 4"; + } + +event bro_init() + { + if ( ! precompile_pcap_filter(A, "port 80") ) + print "error 1"; + + if ( ! precompile_pcap_filter(B, "port 53") ) + print "error 2"; + } + diff --git a/testing/btest/core/pcap/filter-error.bro b/testing/btest/core/pcap/filter-error.bro new file mode 100644 index 0000000000..1f8ad7a464 --- /dev/null +++ b/testing/btest/core/pcap/filter-error.bro @@ -0,0 +1,16 @@ +# @TEST-EXEC-FAIL: bro -r $TRACES/workshop_2011_browse.trace -f "kaputt" >>output 2>&1 +# @TEST-EXEC-FAIL: test -e conn.log +# @TEST-EXEC: echo ---- >>output +# @TEST-EXEC: bro -r $TRACES/workshop_2011_browse.trace %INPUT >>output 2>&1 +# @TEST-EXEC: test -e conn.log +# @TEST-EXEC: btest-diff output + +redef enum PcapFilterID += { A }; + +event bro_init() + { + if ( ! precompile_pcap_filter(A, "kaputt, too") ) + print "error", pcap_error(); + } + + diff --git a/testing/btest/core/pcap/input-error.bro b/testing/btest/core/pcap/input-error.bro new file mode 100644 index 0000000000..2a0787c832 --- /dev/null +++ b/testing/btest/core/pcap/input-error.bro @@ -0,0 +1,14 @@ +# @TEST-EXEC-FAIL: bro -i NO_SUCH_INTERFACE 2>&1 >>output 2>&1 +# @TEST-EXEC: cat output | sed 's/(.*)//g' >output2 +# @TEST-EXEC-FAIL: bro -r NO_SUCH_TRACE 2>&1 >>output2 2>&1 +# @TEST-EXEC: btest-diff output2 + +redef enum PcapFilterID += { A }; + +event bro_init() + { + if ( ! precompile_pcap_filter(A, "kaputt, too") ) + print "error", pcap_error(); + } + + diff --git a/testing/btest/core/pcap/pseudo-realtime.bro b/testing/btest/core/pcap/pseudo-realtime.bro new file mode 100644 index 0000000000..625706f321 --- /dev/null +++ b/testing/btest/core/pcap/pseudo-realtime.bro @@ -0,0 +1,42 @@ +# @TEST-EXEC: bro -C -r $TRACES/wikipedia.trace %INPUT --pseudo-realtime >output +# @TEST-EXEC: btest-diff output + +global init = F; +global last_network = network_time(); +global last_current = current_time(); +global cnt = 0; +global an = 0secs; +global ac = 0secs; + +event new_packet(c: connection, p: pkt_hdr) + { + local tn = network_time(); + local tc = current_time(); + local dn = tn - last_network; + local dc = tc - last_current; + + last_network = tn; + last_current = tc; + ++cnt; + + if ( ! init ) + { + init = T; + return; + } + + an += dn; + ac += dc; + + # print fmt("num=%d agg_delta_network=%.1f agg_delta_real=%.1f", cnt, an, ac); + } + +event bro_done() + { + local d = (an - ac); + if ( d < 0 secs) + d = -d; + + print fmt("real time %s trace time", d < 1.0secs ? "matches" : "does NOT match"); + } + diff --git a/testing/btest/core/pcap/read-trace-with-filter.bro b/testing/btest/core/pcap/read-trace-with-filter.bro new file mode 100644 index 0000000000..5878bada64 --- /dev/null +++ b/testing/btest/core/pcap/read-trace-with-filter.bro @@ -0,0 +1,3 @@ +# @TEST-EXEC: bro -C -r $TRACES/wikipedia.trace -f "port 50000" +# @TEST-EXEC: btest-diff conn.log +# @TEST-EXEC: btest-diff packet_filter.log From 3e669daa05733d8f320752ea310a379ea85ea01c Mon Sep 17 00:00:00 2001 From: Robin Sommer Date: Wed, 27 Aug 2014 23:44:39 -0400 Subject: [PATCH 11/25] Interface tweaks for PktSrc, plus docs for PktSrc and PktDumper. --- aux/plugins | 1 - src/Net.cc | 7 +- src/Net.h | 3 +- src/iosource/BPF_Program.cc | 1 + src/iosource/BPF_Program.h | 5 + src/iosource/Component.h | 2 +- src/iosource/Manager.cc | 8 +- src/iosource/Manager.h | 2 +- src/iosource/PktDumper.h | 114 +++++++- src/iosource/PktSrc.cc | 28 +- src/iosource/PktSrc.h | 336 ++++++++++++++++++++--- src/iosource/pcap/Source.cc | 41 +-- src/iosource/pcap/Source.h | 13 +- src/main.cc | 2 +- testing/btest/core/pcap/filter-error.bro | 2 +- 15 files changed, 464 insertions(+), 101 deletions(-) delete mode 160000 aux/plugins diff --git a/aux/plugins b/aux/plugins deleted file mode 160000 index 6de518922e..0000000000 --- a/aux/plugins +++ /dev/null @@ -1 +0,0 @@ -Subproject commit 6de518922e5f89d52d831ea6fb6adb7fff94437e diff --git a/src/Net.cc b/src/Net.cc index 554aa890ba..adac9c02fd 100644 --- a/src/Net.cc +++ b/src/Net.cc @@ -153,8 +153,7 @@ void net_update_time(double new_network_time) } void net_init(name_list& interfaces, name_list& readfiles, - const char* writefile, const char* filter, - int do_watchdog) + const char* writefile, int do_watchdog) { if ( readfiles.length() > 0 ) { @@ -163,7 +162,7 @@ void net_init(name_list& interfaces, name_list& readfiles, for ( int i = 0; i < readfiles.length(); ++i ) { - iosource::PktSrc* ps = iosource_mgr->OpenPktSrc(readfiles[i], filter, false); + iosource::PktSrc* ps = iosource_mgr->OpenPktSrc(readfiles[i], false); assert(ps); if ( ! ps->IsOpen() ) @@ -180,7 +179,7 @@ void net_init(name_list& interfaces, name_list& readfiles, for ( int i = 0; i < interfaces.length(); ++i ) { - iosource::PktSrc* ps = iosource_mgr->OpenPktSrc(interfaces[i], filter, true); + iosource::PktSrc* ps = iosource_mgr->OpenPktSrc(interfaces[i], true); assert(ps); if ( ! ps->IsOpen() ) diff --git a/src/Net.h b/src/Net.h index 41cbd69abe..2e466f8c7f 100644 --- a/src/Net.h +++ b/src/Net.h @@ -13,8 +13,7 @@ #include "iosource/PktDumper.h" extern void net_init(name_list& interfaces, name_list& readfiles, - const char* writefile, const char* filter, - int do_watchdog); + const char* writefile, int do_watchdog); extern void net_run(); extern void net_get_final_stats(); extern void net_finish(int drain_events); diff --git a/src/iosource/BPF_Program.cc b/src/iosource/BPF_Program.cc index 8df7729ad1..064e9a743e 100644 --- a/src/iosource/BPF_Program.cc +++ b/src/iosource/BPF_Program.cc @@ -86,6 +86,7 @@ bool BPF_Program::Compile(pcap_t* pcap, const char* filter, uint32 netmask, } m_compiled = true; + m_matches_anything = (strlen(filter) == 0 || strcmp(filter, "ip or not ip") == 0); return true; } diff --git a/src/iosource/BPF_Program.h b/src/iosource/BPF_Program.h index 3efa212bbc..88a4512d4e 100644 --- a/src/iosource/BPF_Program.h +++ b/src/iosource/BPF_Program.h @@ -37,6 +37,10 @@ public: // code, false otherwise. bool IsCompiled() { return m_compiled; } + // Returns true if this program matches any packets. This is not + // comprehensive, but can identify a few cases where it does. + bool MatchesAnything() { return m_matches_anything; } + // Accessor to the compiled program. Returns nil when // no program is currently compiled. bpf_program* GetProgram(); @@ -47,6 +51,7 @@ protected: // (I like to prefix member variables with m_, makes it clear // in the implementation whether it's a global or not. --ck) bool m_compiled; + bool m_matches_anything; struct bpf_program m_program; }; diff --git a/src/iosource/Component.h b/src/iosource/Component.h index c93597fd67..cef500e52f 100644 --- a/src/iosource/Component.h +++ b/src/iosource/Component.h @@ -50,7 +50,7 @@ class PktSrcComponent : public iosource::Component { public: enum InputType { LIVE, TRACE, BOTH }; - typedef PktSrc* (*factory_callback)(const std::string& path, const std::string& filter, bool is_live); + typedef PktSrc* (*factory_callback)(const std::string& path, bool is_live); /** * XXX diff --git a/src/iosource/Manager.cc b/src/iosource/Manager.cc index 63b6888801..ebd92e9527 100644 --- a/src/iosource/Manager.cc +++ b/src/iosource/Manager.cc @@ -222,7 +222,7 @@ static std::pair split_prefix(std::string path) return std::make_pair(prefix, path); } -PktSrc* Manager::OpenPktSrc(const std::string& path, const std::string& filter, bool is_live) +PktSrc* Manager::OpenPktSrc(const std::string& path, bool is_live) { std::pair t = split_prefix(path); std::string prefix = t.first; @@ -254,10 +254,10 @@ PktSrc* Manager::OpenPktSrc(const std::string& path, const std::string& filter, // Instantiate packet source. - PktSrc* ps = (*component->Factory())(npath, filter, is_live); + PktSrc* ps = (*component->Factory())(npath, is_live); assert(ps); - if ( ! ps->IsOpen() && ps->ErrorMsg() ) + if ( ! ps->IsOpen() && ps->IsError() ) // Set an error message if it didn't open successfully. ps->Error("could not open"); @@ -298,7 +298,7 @@ PktDumper* Manager::OpenPktDumper(const string& path, bool append) PktDumper* pd = (*component->Factory())(npath, append); assert(pd); - if ( ! pd->IsOpen() && pd->ErrorMsg() ) + if ( ! pd->IsOpen() && pd->IsError() ) // Set an error message if it didn't open successfully. pd->Error("could not open"); diff --git a/src/iosource/Manager.h b/src/iosource/Manager.h index 4198c73680..bebed61de7 100644 --- a/src/iosource/Manager.h +++ b/src/iosource/Manager.h @@ -35,7 +35,7 @@ public: // sources (and therefore returning a Size() of zero). void Terminate() { RemoveAll(); } - PktSrc* OpenPktSrc(const std::string& path, const std::string& filter, bool is_live); + PktSrc* OpenPktSrc(const std::string& path, bool is_live); PktDumper* OpenPktDumper(const std::string& path, bool append); protected: diff --git a/src/iosource/PktDumper.h b/src/iosource/PktDumper.h index d8201f977c..5e35bf1ca7 100644 --- a/src/iosource/PktDumper.h +++ b/src/iosource/PktDumper.h @@ -9,45 +9,145 @@ namespace iosource { class PktDumper { public: + /** + * Structure describing a packet. + */ struct Packet { + /** + * The pcap header associated with the packet. + */ const struct pcap_pkthdr* hdr; - const u_char* data; + + /** + * The full content of the packet. + */ + const unsigned char* data; }; + /** + * Constructor. + */ PktDumper(); + + /** + * Destructor. + */ virtual ~PktDumper(); + /** + * Returns the path associated with the dumper. + */ const std::string& Path() const; + + /** + * Returns true if the dumper is open for writing. + */ bool IsOpen() const; + + /** + * Returns the time when the dumper was opened for writing. + */ double OpenTime() const; + + /** + * Returns returns true if the dumper has encountered an error. + */ bool IsError() const; + + /** + * Returns if the dumper has encountered an error, returns a + * corresponding error message. Returns an emoty string otherwise. + */ const char* ErrorMsg() const; + + /** + * Returns the size of the link-layer headers with this dumper. + */ int HdrSize() const; + + /** + * Writes a packet to the dumper. + * + * @param pkt The packet to record. + */ bool Record(const Packet* pkt); // PktDumper interface for derived classes to implement. - virtual void Close() = 0; + + /** + * Called by the manager system to open the source. + * + * Derived classes must implement this method. If successful, the + * implementation must call \a Opened(); if not, it must call Error() + * with a corresponding message. + */ virtual void Open() = 0; + + /** + * Called by the manager system to close the dumper. + * + * Derived classes must implement this method. If successful, the + * implementation must call \a Closed(); if not, it must call Error() + * with a corresponding message. + */ + virtual void Close() = 0; + + /** + * Called to write a packet to the dumper. + * + * Derived classes must implement this method. + * + * @param pkt The packet to record. + * + * @return True if succesful, false otherwise (in which case \a + * Error() must have been called.) + */ virtual bool Dump(const Packet* pkt) = 0; protected: friend class Manager; - // Methods to use by derived classed. - // + /** + * Structure to pass back information about the packet dumper to the + * base class. Derived class pass an instance of this to \a Opened(). + */ struct Properties { std::string path; int hdr_size; double open_time; }; - void Init(); - void Done(); - + /** + * Called from the implementations of \a Open() to signal that the + * source has been successully opened. + * + * @param props A properties instance describing the now open source. + */ void Opened(const Properties& props); + + /** + * Called from the implementations of \a Close() to signal that the + * source has been closed. + */ void Closed(); + + /** + * Called from derived classes to signal an error. + * + * @param msg A corresponding error message. + */ void Error(const std::string& msg); + /** + * Called by the manager to initialize the dumper. + */ + void Init(); + + /** + * Called by the manager to shutdown the dumper. + */ + void Done(); + private: bool is_open; Properties props; diff --git a/src/iosource/PktSrc.cc b/src/iosource/PktSrc.cc index 30f58a5b3f..acde8d5ff6 100644 --- a/src/iosource/PktSrc.cc +++ b/src/iosource/PktSrc.cc @@ -53,6 +53,11 @@ uint32 PktSrc::Netmask() const return IsOpen() ? props.netmask : PCAP_NETMASK_UNKNOWN; } +bool PktSrc::IsError() const + { + return ErrorMsg(); + } + int PktSrc::HdrSize() const { return IsOpen() ? props.hdr_size : -1; @@ -87,7 +92,7 @@ void PktSrc::Opened(const Properties& arg_props) props = arg_props; SetClosed(false); - if ( ! PrecompileFilter(0, props.filter) || ! SetFilter(0) ) + if ( ! PrecompileFilter(0, "") || ! SetFilter(0) ) { Close(); return; @@ -378,7 +383,7 @@ void PktSrc::Process() net_packet_dispatch(current_packet.ts, current_packet.hdr, current_packet.data, pkt_hdr_size, this); have_packet = 0; - DoneWithPacket(¤t_packet); + DoneWithPacket(); } const char* PktSrc::Tag() @@ -386,7 +391,7 @@ const char* PktSrc::Tag() return "PktSrc"; } -int PktSrc::ExtractNextPacketInternal() +bool PktSrc::ExtractNextPacketInternal() { if ( have_packet ) return true; @@ -426,7 +431,7 @@ int PktSrc::ExtractNextPacketInternal() return 0; } -int PktSrc::PrecompileBPFFilter(int index, const std::string& filter) +bool PktSrc::PrecompileBPFFilter(int index, const std::string& filter) { char errbuf[PCAP_ERRBUF_SIZE]; @@ -466,7 +471,7 @@ BPF_Program* PktSrc::GetBPFFilter(int index) return code; } -int PktSrc::ApplyBPFFilter(int index, const struct pcap_pkthdr *hdr, const u_char *pkt) +bool PktSrc::ApplyBPFFilter(int index, const struct pcap_pkthdr *hdr, const u_char *pkt) { BPF_Program* code = GetBPFFilter(index); @@ -476,5 +481,18 @@ int PktSrc::ApplyBPFFilter(int index, const struct pcap_pkthdr *hdr, const u_cha Close(); } + if ( code->MatchesAnything() ) + return true; + return pcap_offline_filter(code->GetProgram(), hdr, pkt); } + +bool PktSrc::GetCurrentPacket(const pcap_pkthdr** hdr, const u_char** pkt) + { + if ( ! have_packet ) + return false; + + *hdr = current_packet.hdr; + *pkt = current_packet.data; + return true; + } diff --git a/src/iosource/PktSrc.h b/src/iosource/PktSrc.h index 72e1a0da8c..c616a2d5b4 100644 --- a/src/iosource/PktSrc.h +++ b/src/iosource/PktSrc.h @@ -17,69 +17,206 @@ namespace iosource { class PktSrc : public IOSource { public: + /** + * Struct for returning statistics on a packet source. + */ struct Stats { - unsigned int received; // pkts received (w/o drops) + /** + * Packets received by source after filtering (w/o drops). + */ + unsigned int received; + + /** + * Packets dropped by source. + */ unsigned int dropped; // pkts dropped - unsigned int link; // total packets on link - // (not always not available) - // + + /** + * Total number of packets on link before filtering. + * Optional, can be left unset if not available. + */ + unsigned int link; + Stats() { received = dropped = link = 0; } }; + /** + * Constructor. + */ PktSrc(); + + /** + * Destructor. + */ virtual ~PktSrc(); + /** + * Returns the path associated with the source. This is the interface + * name for live source, and a filename for offline sources. + */ const std::string& Path() const; - const std::string& Filter() const; + + /** + * Returns true if this is a live source. + */ bool IsLive() const; + + /** + * Returns the link type of the source. + */ int LinkType() const; + + /** + * Returns the netmask associated with the source, or \c + * PCAP_NETMASK_UNKNOWN if unknown. + */ uint32 Netmask() const; + + /** + * Returns true if the source has flagged an error. + */ + bool IsError() const; + + /** + * If the source encountered an error, returns a corresponding error + * message. Returns an empty string otherwise. + */ const char* ErrorMsg() const; + + /** + * Returns the size of the link-layer header for this source. + */ int HdrSize() const; + + /** + * Returns the snap length for this source. + */ int SnapLen() const; - // Only valid in pseudo-realtime mode. + /** + * In pseudo-realtime mode, returns the logical timestamp of the + * current packet. Undefined if not running pseudo-realtime mode. + */ double CurrentPacketTimestamp(); + + /** + * In pseudo-realtime mode, returns the wall clock time associated + * with current packet. Undefined if not running pseudo-realtime + * mode. + */ double CurrentPacketWallClock(); - // Signal packet source that processing was suspended and is now - // going to be continued. + /** + * Signals packet source that processing is going to be continued + * after previous suspension. + */ void ContinueAfterSuspend(); - // Precompiles a BPF filter and associates the given index with it. - // Returns true on success, 0 if a problem occurred. The compiled - // filter will be then available via GetBPFFilter*(. - int PrecompileBPFFilter(int index, const std::string& filter); + /** + * Precompiles a BPF filter and associates the given index with it. + * The compiled filter will be then available via \a GetBPFFilter(). + * + * This is primarily a helper for packet source implementation that + * want to apply BPF filtering to their packets. + * + * @param index The index to associate with the filter. + * + * @param BPF filter The filter string to precompile. + * + * @return True on success, false if a problem occurred. + */ + bool PrecompileBPFFilter(int index, const std::string& filter); - // Returns the BPF filter with the given index, as compiled by - // PrecompileBPFFilter(), or null if none has been (successfully) - // compiled. + /** + * Returns the precompiled BPF filter associated with a given index, + * if any, as compiled by \a PrecompileBPFFilter(). + * + * This is primarily a helper for packet source implementation that + * want to apply BPF filtering to their packets. + * + * @return The BPF filter associated, or null if none has been + * (successfully) compiled. + */ BPF_Program* GetBPFFilter(int index); - // Applies a precompiled BPF filter to a packet, returning true if it - // maches. This will close the source with an error message if no - // filter with that index has been compiled. - int ApplyBPFFilter(int index, const struct pcap_pkthdr *hdr, const u_char *pkt); + /** + * Applies a precompiled BPF filter to a packet. This will close the + * source with an error message if no filter with that index has been + * compiled. + * + * This is primarily a helper for packet source implementation that + * want to apply BPF filtering to their packets. + * + * @param index The index of the filter to apply. + * + * @param hdr The header of the packet to filter. + * + * @param pkt The content of the packet to filter. + * + * @return True if it maches. */ + bool ApplyBPFFilter(int index, const struct pcap_pkthdr *hdr, const u_char *pkt); + + /** + * Returns the packet currently being processed, if available. + * + * @param hdr A pointer to pass the header of the current packet back. + * + * @param pkt A pointer to pass the content of the current packet + * back. + * + * @return True if the current packet is available, or false if not. + */ + bool GetCurrentPacket(const pcap_pkthdr** hdr, const u_char** pkt); // PacketSource interace for derived classes to override. - // Returns the packet last processed; false if there is no - // current packet available. - virtual bool GetCurrentPacket(const pcap_pkthdr** hdr, const u_char** pkt) = 0; + /** + * Precompiles a filter and associates a given index with it. The + * filter syntax is defined by the packet source's implenentation. + * + * Derived classes must implement this to implement their filtering. + * If they want to use BPF but don't support it natively, they can + * call the corresponding helper method provided by \a PktSrc. + * + * @param index The index to associate with the filter + * + * @param filter The filter string to precompile. + * + * @return True on success, false if a problem occurred or filtering + * is not supported. + */ + virtual bool PrecompileFilter(int index, const std::string& filter) = 0; - // Precompiles a filter and associates the given index with it. - // Returns true on success, 0 if a problem occurred or filtering is - // not supported. - virtual int PrecompileFilter(int index, const std::string& filter) = 0; + /** + * Activates a precompiled filter with the given index. + * + * Derived classes must implement this to implement their filtering. + * If they want to use BPF but don't support it natively, they can + * call the corresponding helper method provided by \a PktSrc. + * + * @param index The index of the filter to activate. + * + * @return True on success, false if a problem occurred or the + * filtering is not supported. + */ + virtual bool SetFilter(int index) = 0; - // Activates the filter with the given index. Returns true on - // success, 0 if a problem occurred or the filtering is not - // supported. - virtual int SetFilter(int index) = 0; - - // Returns current statistics about the source. + /** + * Returns current statistics about the source. + * + * Derived classes must implement this method. + * + * @param stats A statistics structure that the method fill out. + */ virtual void Statistics(Stats* stats) = 0; + /** + * Helper method to return the header size for a given link tyoe. + * + * @param link_type The link tyoe. + * + * @return The header size in bytes. + */ static int GetLinkHeaderSize(int link_type); protected: @@ -87,13 +224,45 @@ protected: // Methods to use by derived classes. + /** + * Structure to pass back information about the packet source to the + * base class. Derived class pass an instance of this to \a Opened(). + */ struct Properties { + /** + * The path associated with the source. This is the interface + * name for live source, and a filename for offline sources. + */ std::string path; - std::string filter; // Maybe different than what's passed in if not (directly) supported. + + /** + * A file descriptor suitable to use with \a select() for + * determining if there's input available from this source. + */ int selectable_fd; + + /** + * The link type for packets from this source. + */ int link_type; + + /** + * The size of the link-layer header for packets from this + * source. \a GetLinkHeaderSize() may be used to derive this + * value. + */ int hdr_size; + + /** + * The netmask associated with the source, or \c + * PCAP_NETMASK_UNKNOWN if unknown. + */ uint32 netmask; + + /** + * True if the source is reading live inout, false for + * working offline. + */ bool is_live; Properties() @@ -102,34 +271,120 @@ protected: } }; + /** + * Structure describing a packet. + */ struct Packet { + /** + * Time associated with the packet. + */ double ts; + + /** + * The pcap header associated with the packet. + */ const struct ::pcap_pkthdr* hdr; + + /** + * The full content of the packet. + */ const u_char* data; }; + /** + * Called from the implementations of \a Open() to signal that the + * source has been successully opened. + * + * @param props A properties instance describing the now open source. + */ void Opened(const Properties& props); + + /** + * Called from the implementations of \a Close() to signal that the + * source has been closed. + */ void Closed(); + + /** + * Can be called from derived classes to send an informational + * message to the user. + * + * @param msg The message to pass on. + */ void Info(const std::string& msg); + + /** + * Can be called from derived classes to flag send an error. + * + * @param msg The message going with the error. + */ void Error(const std::string& msg); + + /** + * Can be called from derived classes to flah a "weird" situation. + * + * @param msg The message to pass on. + * + * @param pkt The packet associated with the weird, or null if none. + */ void Weird(const std::string& msg, const Packet* pkt); + + /** + * Can be called from derived classes to flag an internal error, + * which will abort execution. + * + * @param msg The message to pass on. + */ void InternalError(const std::string& msg); // PktSrc interface for derived classes to implement. + /** + * Called by the manager system to open the source. + * + * Derived classes must implement this method. If successful, the + * implementation must call \a Opened(); if not, it must call Error() + * with a corresponding message. + */ virtual void Open() = 0; + + /** + * Called by the manager system to close the source. + * + * Derived classes must implement this method. If successful, the + * implementation must call \a Closed(); if not, it must call Error() + * with a corresponding message. + */ virtual void Close() = 0; - // Returns 1 on success, 0 on time-out/gone dry. - virtual int ExtractNextPacket(Packet* pkt) = 0; - virtual void DoneWithPacket(Packet* pkt) = 0; + + /** + * Provides the next packet from the source. + * + * @param pkt The packet structure to fill in with the packet's + * information. The callee keep ownership of the data but must + * guaranetee that it stays available at least until \a + * DoneWithPacket() is called. It is guaranteed that no two calls to + * this method will hapen with \a DoneWithPacket() in between. + * + * @return True if a packet is available and *pkt* filled in. False + * if not packet is available or an error occured (which must be + * flageed via Error()). + */ + virtual bool ExtractNextPacket(Packet* pkt) = 0; + + /** + * Signals that the data of previously extracted packet will no + * longer be needed. + */ + virtual void DoneWithPacket() = 0; private: - // Checks if the current packet has a pseudo-time <= current_time. - // If yes, returns pseudo-time, otherwise 0. + // Checks if the current packet has a pseudo-time <= current_time. If + // yes, returns pseudo-time, otherwise 0. double CheckPseudoTime(); - // XXX - int ExtractNextPacketInternal(); + // Internal helper for ExtractNextPacket(). + bool ExtractNextPacketInternal(); // IOSource interface implementation. virtual void Init(); @@ -159,5 +414,4 @@ private: } - #endif diff --git a/src/iosource/pcap/Source.cc b/src/iosource/pcap/Source.cc index 79ded790bd..7cca94122b 100644 --- a/src/iosource/pcap/Source.cc +++ b/src/iosource/pcap/Source.cc @@ -16,10 +16,9 @@ PcapSource::~PcapSource() Close(); } -PcapSource::PcapSource(const std::string& path, const std::string& filter, bool is_live) +PcapSource::PcapSource(const std::string& path, bool is_live) { props.path = path; - props.filter = filter; props.is_live = is_live; last_data = 0; } @@ -141,10 +140,10 @@ void PcapSource::OpenOffline() Opened(props); } -int PcapSource::ExtractNextPacket(Packet* pkt) +bool PcapSource::ExtractNextPacket(Packet* pkt) { if ( ! pd ) - return 0; + return false; const u_char* data = pcap_next(pd, ¤t_hdr); @@ -156,7 +155,7 @@ int PcapSource::ExtractNextPacket(Packet* pkt) if ( ! props.is_live ) Close(); - return 0; + return false; } pkt->ts = current_hdr.ts.tv_sec + double(current_hdr.ts.tv_usec) / 1e6; @@ -166,29 +165,29 @@ int PcapSource::ExtractNextPacket(Packet* pkt) if ( current_hdr.len == 0 || current_hdr.caplen == 0 ) { Weird("empty_pcap_header", pkt); - return 0; + return false; } last_hdr = current_hdr; last_data = data; ++stats.received; - return 1; + return true; } -void PcapSource::DoneWithPacket(Packet* pkt) +void PcapSource::DoneWithPacket() { // Nothing to do. } -int PcapSource::PrecompileFilter(int index, const std::string& filter) +bool PcapSource::PrecompileFilter(int index, const std::string& filter) { return PktSrc::PrecompileBPFFilter(index, filter); } -int PcapSource::SetFilter(int index) +bool PcapSource::SetFilter(int index) { if ( ! pd ) - return 1; // Prevent error message + return true; // Prevent error message char errbuf[PCAP_ERRBUF_SIZE]; @@ -200,13 +199,13 @@ int PcapSource::SetFilter(int index) "No precompiled pcap filter for index %d", index); Error(errbuf); - return 0; + return false; } if ( pcap_setfilter(pd, code->GetProgram()) < 0 ) { PcapError(); - return 0; + return false; } #ifndef HAVE_LINUX @@ -214,7 +213,7 @@ int PcapSource::SetFilter(int index) stats.received = stats.dropped = stats.link = 0; #endif - return 1; + return true; } void PcapSource::Statistics(Stats* s) @@ -246,16 +245,6 @@ void PcapSource::Statistics(Stats* s) s->dropped = 0; } -bool PcapSource::GetCurrentPacket(const pcap_pkthdr** hdr, const u_char** pkt) - { - if ( ! last_data ) - return false; - - *hdr = &last_hdr; - *pkt = last_data; - return true; - } - void PcapSource::PcapError() { if ( pd ) @@ -285,7 +274,7 @@ void PcapSource::SetHdrSize() } } -iosource::PktSrc* PcapSource::Instantiate(const std::string& path, const std::string& filter, bool is_live) +iosource::PktSrc* PcapSource::Instantiate(const std::string& path, bool is_live) { - return new PcapSource(path, filter, is_live); + return new PcapSource(path, is_live); } diff --git a/src/iosource/pcap/Source.h b/src/iosource/pcap/Source.h index 039bdec81a..b914dc6b63 100644 --- a/src/iosource/pcap/Source.h +++ b/src/iosource/pcap/Source.h @@ -11,22 +11,21 @@ namespace pcap { class PcapSource : public iosource::PktSrc { public: // XXX - PcapSource(const std::string& path, const std::string& filter, bool is_live); + PcapSource(const std::string& path, bool is_live); virtual ~PcapSource(); - static PktSrc* Instantiate(const std::string& path, const std::string& filter, bool is_live); + static PktSrc* Instantiate(const std::string& path, bool is_live); protected: // PktSrc interface. virtual void Open(); virtual void Close(); - virtual int ExtractNextPacket(Packet* pkt); - virtual void DoneWithPacket(Packet* pkt); - virtual int PrecompileFilter(int index, const std::string& filter); - virtual int SetFilter(int index); + virtual bool ExtractNextPacket(Packet* pkt); + virtual void DoneWithPacket(); + virtual bool PrecompileFilter(int index, const std::string& filter); + virtual bool SetFilter(int index); virtual void Statistics(Stats* stats); - virtual bool GetCurrentPacket(const pcap_pkthdr** hdr, const u_char** pkt); private: void OpenLive(); diff --git a/src/main.cc b/src/main.cc index 295fa79ee7..bdd3d7072b 100644 --- a/src/main.cc +++ b/src/main.cc @@ -991,7 +991,7 @@ int main(int argc, char** argv) snaplen = internal_val("snaplen")->AsCount(); if ( dns_type != DNS_PRIME ) - net_init(interfaces, read_files, writefile, "", do_watchdog); + net_init(interfaces, read_files, writefile, do_watchdog); BroFile::SetDefaultRotation(log_rotate_interval, log_max_size); diff --git a/testing/btest/core/pcap/filter-error.bro b/testing/btest/core/pcap/filter-error.bro index 1f8ad7a464..1d7b6516db 100644 --- a/testing/btest/core/pcap/filter-error.bro +++ b/testing/btest/core/pcap/filter-error.bro @@ -3,7 +3,7 @@ # @TEST-EXEC: echo ---- >>output # @TEST-EXEC: bro -r $TRACES/workshop_2011_browse.trace %INPUT >>output 2>&1 # @TEST-EXEC: test -e conn.log -# @TEST-EXEC: btest-diff output +# @TEST-EXEC: TEST_DIFF_CANONIFIER=$SCRIPTS/diff-remove-abspath btest-diff output redef enum PcapFilterID += { A }; From 5e4f498083271c7bede1ac79ba4aed3084eefe10 Mon Sep 17 00:00:00 2001 From: Robin Sommer Date: Thu, 28 Aug 2014 00:53:15 -0400 Subject: [PATCH 12/25] Adding test creating a dynamic pktsrc plugin. --- .../btest/Baseline/plugins.pktsrc/conn.log | 10 +++ .../btest/plugins/pktsrc-plugin/.btest-ignore | 0 .../plugins/pktsrc-plugin/CMakeLists.txt | 17 ++++ .../btest/plugins/pktsrc-plugin/src/Foo.cc | 77 +++++++++++++++++++ testing/btest/plugins/pktsrc-plugin/src/Foo.h | 35 +++++++++ .../btest/plugins/pktsrc-plugin/src/Plugin.cc | 20 +++++ testing/btest/plugins/pktsrc.bro | 8 ++ 7 files changed, 167 insertions(+) create mode 100644 testing/btest/Baseline/plugins.pktsrc/conn.log create mode 100644 testing/btest/plugins/pktsrc-plugin/.btest-ignore create mode 100644 testing/btest/plugins/pktsrc-plugin/CMakeLists.txt create mode 100644 testing/btest/plugins/pktsrc-plugin/src/Foo.cc create mode 100644 testing/btest/plugins/pktsrc-plugin/src/Foo.h create mode 100644 testing/btest/plugins/pktsrc-plugin/src/Plugin.cc create mode 100644 testing/btest/plugins/pktsrc.bro diff --git a/testing/btest/Baseline/plugins.pktsrc/conn.log b/testing/btest/Baseline/plugins.pktsrc/conn.log new file mode 100644 index 0000000000..550f520352 --- /dev/null +++ b/testing/btest/Baseline/plugins.pktsrc/conn.log @@ -0,0 +1,10 @@ +#separator \x09 +#set_separator , +#empty_field (empty) +#unset_field - +#path conn +#open 2014-08-28-04-53-05 +#fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p proto service duration orig_bytes resp_bytes conn_state local_orig missed_bytes history orig_pkts orig_ip_bytes resp_pkts resp_ip_bytes tunnel_parents +#types time string addr port addr port enum string interval count count string bool count string count count count count set[string] +1409193037.000000 CXWv6p3arKYeMETxOg 1.2.0.2 2527 1.2.0.3 6649 tcp - - - - S0 - 0 S 1 64 0 0 (empty) +#close 2014-08-28-04-53-05 diff --git a/testing/btest/plugins/pktsrc-plugin/.btest-ignore b/testing/btest/plugins/pktsrc-plugin/.btest-ignore new file mode 100644 index 0000000000..e69de29bb2 diff --git a/testing/btest/plugins/pktsrc-plugin/CMakeLists.txt b/testing/btest/plugins/pktsrc-plugin/CMakeLists.txt new file mode 100644 index 0000000000..2234907ad2 --- /dev/null +++ b/testing/btest/plugins/pktsrc-plugin/CMakeLists.txt @@ -0,0 +1,17 @@ + +project(Bro-Plugin-Demo-Foo) + +cmake_minimum_required(VERSION 2.6.3) + +if ( NOT BRO_DIST ) + message(FATAL_ERROR "BRO_DIST not set") +endif () + +set(CMAKE_MODULE_PATH ${BRO_DIST}/cmake) + +include(BroPlugin) + +bro_plugin_begin(Demo Foo) +bro_plugin_cc(src/Plugin.cc) +bro_plugin_cc(src/Foo.cc) +bro_plugin_end() diff --git a/testing/btest/plugins/pktsrc-plugin/src/Foo.cc b/testing/btest/plugins/pktsrc-plugin/src/Foo.cc new file mode 100644 index 0000000000..b08bc51d72 --- /dev/null +++ b/testing/btest/plugins/pktsrc-plugin/src/Foo.cc @@ -0,0 +1,77 @@ + +#include +#include + +#include "Foo.h" + +using namespace plugin::Demo_Foo; + +Foo::Foo(const std::string& path, bool is_live) + { + packet = + string("\x45\x00\x00\x40\x15\x55\x40\x00\x3e\x06\x25\x5b\x01\x02\x00\x02" + "\x01\x02\x00\x03\x09\xdf\x19\xf9\x5d\x8a\x36\x7c\x00\x00\x00\x00" + "\xb0\x02\x40\x00\x3c\x72\x00\x00\x02\x04\x05\x5c\x01\x03\x03\x00" + "\x01\x01\x08\x0a\x00\x00\x00\x00\x00\x00\x00\x00\x01\x01\x04\x02", 64); + + props.path = path; + props.selectable_fd = open("/bin/sh", O_RDONLY); // any fd is fine. + props.link_type = DLT_RAW; + props.hdr_size = 0; + props.netmask = 0; + props.is_live = 0; + } + +iosource::PktSrc* Foo::Instantiate(const std::string& path, bool is_live) + { + return new Foo(path, is_live); + } + +void Foo::Open() + { + Opened(props); + } + +void Foo::Close() + { + Closed(); + } + +bool Foo::ExtractNextPacket(Packet* pkt) + { + if ( packet.empty() ) + { + Close(); + return false; + } + + hdr.ts.tv_sec = 1409193037; + hdr.ts.tv_usec = 0; + hdr.caplen = hdr.len = packet.size(); + pkt->ts = hdr.ts.tv_sec; + pkt->hdr = &hdr; + pkt->data = (const u_char *)packet.c_str(); + return true; + } + +void Foo::DoneWithPacket() + { + packet.clear(); + } + +bool Foo::PrecompileFilter(int index, const std::string& filter) + { + // skip for the testing. + return true; + } + +bool Foo::SetFilter(int index) + { + // skip for the testing. + return true; + } + +void Foo::Statistics(Stats* stats) + { + // skip for the testing. + } diff --git a/testing/btest/plugins/pktsrc-plugin/src/Foo.h b/testing/btest/plugins/pktsrc-plugin/src/Foo.h new file mode 100644 index 0000000000..902ac0e37a --- /dev/null +++ b/testing/btest/plugins/pktsrc-plugin/src/Foo.h @@ -0,0 +1,35 @@ + +#ifndef BRO_PLUGIN_DEMO_FOO_H +#define BRO_PLUGIN_DEMO_FOO_H + +#include +#include + +namespace plugin { +namespace Demo_Foo { + +class Foo : public iosource::PktSrc { +public: + Foo(const std::string& path, bool is_live); + + static PktSrc* Instantiate(const std::string& path, bool is_live); + +protected: + virtual void Open(); + virtual void Close(); + virtual bool ExtractNextPacket(Packet* pkt); + virtual void DoneWithPacket(); + virtual bool PrecompileFilter(int index, const std::string& filter); + virtual bool SetFilter(int index); + virtual void Statistics(Stats* stats); + +private: + Properties props; + string packet; + struct pcap_pkthdr hdr; +}; + +} +} + +#endif diff --git a/testing/btest/plugins/pktsrc-plugin/src/Plugin.cc b/testing/btest/plugins/pktsrc-plugin/src/Plugin.cc new file mode 100644 index 0000000000..ecc94866a6 --- /dev/null +++ b/testing/btest/plugins/pktsrc-plugin/src/Plugin.cc @@ -0,0 +1,20 @@ + +#include "Plugin.h" + +#include "Foo.h" + +namespace plugin { namespace Demo_Foo { Plugin plugin; } } + +using namespace plugin::Demo_Foo; + +plugin::Configuration Plugin::Configure() + { + AddComponent(new ::iosource::PktSrcComponent("FooPktSrc", "foo", ::iosource::PktSrcComponent::BOTH, ::plugin::Demo_Foo::Foo::Instantiate)); + + plugin::Configuration config; + config.name = "Demo::Foo"; + config.description = "A Foo packet source"; + config.version.major = 1; + config.version.minor = 0; + return config; + } diff --git a/testing/btest/plugins/pktsrc.bro b/testing/btest/plugins/pktsrc.bro new file mode 100644 index 0000000000..2bd9be7bb7 --- /dev/null +++ b/testing/btest/plugins/pktsrc.bro @@ -0,0 +1,8 @@ +# @TEST-EXEC: ${DIST}/aux/bro-aux/plugin-support/init-plugin Demo Foo +# @TEST-EXEC: cp -r %DIR/pktsrc-plugin/* . +# @TEST-EXEC: ./configure --bro-dist=${DIST} && make +# @TEST-EXEC: BRO_PLUGIN_PATH=`pwd` bro -NN Demo::Foo >>output +# @TEST-EXEC: echo === >>output +# @TEST-EXEC: BRO_PLUGIN_PATH=`pwd` bro -r foo:XXX %INPUT FilteredTraceDetection::enable=F >>output +# @TEST-EXEC: TEST_DIFF_CANONIFIER= btest-diff conn.log + From 569853444fd20a958958b40663c21af3ad373bce Mon Sep 17 00:00:00 2001 From: Robin Sommer Date: Wed, 3 Sep 2014 12:45:38 -0700 Subject: [PATCH 13/25] A set of smaller API tweaks, and polishing. --- src/iosource/Component.cc | 21 +++++++++------------ src/iosource/Component.h | 4 ++-- src/iosource/PktSrc.cc | 19 +++++++++++++++++++ src/iosource/pcap/Source.cc | 11 +---------- 4 files changed, 31 insertions(+), 24 deletions(-) diff --git a/src/iosource/Component.cc b/src/iosource/Component.cc index f54c212352..20232161cd 100644 --- a/src/iosource/Component.cc +++ b/src/iosource/Component.cc @@ -64,10 +64,9 @@ PktSrcComponent::factory_callback PktSrcComponent::Factory() const return factory; } - -void PktSrcComponent::Describe(ODesc* d) const +void PktSrcComponent::DoDescribe(ODesc* d) const { - iosource::Component::Describe(d); + iosource::Component::DoDescribe(d); string prefs; @@ -77,16 +76,16 @@ void PktSrcComponent::Describe(ODesc* d) const if ( prefs.size() ) prefs += ", "; - prefs += *i; + prefs += '"' + *i + '"'; } - d->Add(" (interface prefix"); + d->Add("interface prefix"); if ( prefixes.size() > 1 ) d->Add("es"); - d->Add(": "); + d->Add(" "); d->Add(prefs); - d->Add("; "); + d->Add("; supports "); switch ( type ) { case LIVE: @@ -105,7 +104,6 @@ void PktSrcComponent::Describe(ODesc* d) const reporter->InternalError("unknown PkrSrc type"); } - d->Add(")"); } PktDumperComponent::PktDumperComponent(const std::string& name, const std::string& arg_prefix, factory_callback arg_factory) @@ -141,9 +139,9 @@ bool PktDumperComponent::HandlesPrefix(const string& prefix) const return false; } -void PktDumperComponent::Describe(ODesc* d) const +void PktDumperComponent::DoDescribe(ODesc* d) const { - plugin::Component::Describe(d); + plugin::Component::DoDescribe(d); string prefs; @@ -156,12 +154,11 @@ void PktDumperComponent::Describe(ODesc* d) const prefs += *i; } - d->Add(" (dumper prefix"); + d->Add("dumper prefix"); if ( prefixes.size() > 1 ) d->Add("es"); d->Add(": "); d->Add(prefs); - d->Add(")"); } diff --git a/src/iosource/Component.h b/src/iosource/Component.h index cef500e52f..35e8f612e6 100644 --- a/src/iosource/Component.h +++ b/src/iosource/Component.h @@ -93,7 +93,7 @@ public: * Generates a human-readable description of the component. This goes * into the output of \c "bro -NN". */ - virtual void Describe(ODesc* d) const; + virtual void DoDescribe(ODesc* d) const; private: std::vector prefixes; @@ -140,7 +140,7 @@ public: * Generates a human-readable description of the component. This goes * into the output of \c "bro -NN". */ - virtual void Describe(ODesc* d) const; + virtual void DoDescribe(ODesc* d) const; private: std::vector prefixes; diff --git a/src/iosource/PktSrc.cc b/src/iosource/PktSrc.cc index acde8d5ff6..902aaa04be 100644 --- a/src/iosource/PktSrc.cc +++ b/src/iosource/PktSrc.cc @@ -89,6 +89,16 @@ double PktSrc::CurrentPacketWallClock() void PktSrc::Opened(const Properties& arg_props) { + if ( arg_props.hdr_size < 0 ) + { + char buf[512]; + safe_snprintf(buf, sizeof(buf), + "unknown data link type 0x%x", props.link_type); + Error(buf); + Close(); + return; + } + props = arg_props; SetClosed(false); @@ -98,6 +108,9 @@ void PktSrc::Opened(const Properties& arg_props) return; } + if ( props.is_live ) + Info(fmt("listening on %s, capture length %d bytes\n", props.path.c_str(), SnapLen())); + DBG_LOG(DBG_PKTIO, "Opened source %s", props.path.c_str()); } @@ -433,6 +446,9 @@ bool PktSrc::ExtractNextPacketInternal() bool PktSrc::PrecompileBPFFilter(int index, const std::string& filter) { + if ( index < 0 ) + return false; + char errbuf[PCAP_ERRBUF_SIZE]; // Compile filter. @@ -465,6 +481,9 @@ bool PktSrc::PrecompileBPFFilter(int index, const std::string& filter) BPF_Program* PktSrc::GetBPFFilter(int index) { + if ( index < 0 ) + return 0; + HashKey* hash = new HashKey(HashKey(bro_int_t(index))); BPF_Program* code = filters.Lookup(hash); delete hash; diff --git a/src/iosource/pcap/Source.cc b/src/iosource/pcap/Source.cc index 7cca94122b..96e0bb48e5 100644 --- a/src/iosource/pcap/Source.cc +++ b/src/iosource/pcap/Source.cc @@ -108,9 +108,8 @@ void PcapSource::OpenLive() return; props.is_live = true; - Opened(props); - Info(fmt("listening on %s, capture length %d bytes\n", props.path.c_str(), SnapLen())); + Opened(props); } void PcapSource::OpenOffline() @@ -264,14 +263,6 @@ void PcapSource::SetHdrSize() props.link_type = pcap_datalink(pd); props.hdr_size = GetLinkHeaderSize(props.link_type); - - if ( props.hdr_size < 0 ) - { - safe_snprintf(errbuf, sizeof(errbuf), - "unknown data link type 0x%x", props.link_type); - Error(errbuf); - Close(); - } } iosource::PktSrc* PcapSource::Instantiate(const std::string& path, bool is_live) From 43e63daa452f6933622d561c1d496fe4f36f2d3b Mon Sep 17 00:00:00 2001 From: Robin Sommer Date: Wed, 3 Sep 2014 17:37:35 -0700 Subject: [PATCH 14/25] Fixing Bro-level BPF filtering. --- src/iosource/BPF_Program.cc | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/src/iosource/BPF_Program.cc b/src/iosource/BPF_Program.cc index 064e9a743e..70469c97e7 100644 --- a/src/iosource/BPF_Program.cc +++ b/src/iosource/BPF_Program.cc @@ -58,7 +58,14 @@ int pcap_compile_nopcap(int snaplen_arg, int linktype_arg, } #endif -BPF_Program::BPF_Program() : m_compiled(), m_program() +// Simple heuristic to identify filters that always match, so that we can +// skip the filtering in that case. "ip or not ip" is Bro's default filter. +static bool filter_matches_anything(const char *filter) + { + return (! filter) || strlen(filter) == 0 || strcmp(filter, "ip or not ip") == 0; + } + +BPF_Program::BPF_Program() : m_compiled(), m_matches_anything(false), m_program() { } @@ -86,7 +93,7 @@ bool BPF_Program::Compile(pcap_t* pcap, const char* filter, uint32 netmask, } m_compiled = true; - m_matches_anything = (strlen(filter) == 0 || strcmp(filter, "ip or not ip") == 0); + m_matches_anything = filter_matches_anything(filter); return true; } @@ -114,7 +121,10 @@ bool BPF_Program::Compile(int snaplen, int linktype, const char* filter, #endif if ( err == 0 ) + { m_compiled = true; + m_matches_anything = filter_matches_anything(filter); + } return err == 0; } From 2d8368fee90bf53398ba572227c74a697033f2e4 Mon Sep 17 00:00:00 2001 From: Johanna Amann Date: Wed, 3 Sep 2014 22:07:21 -0700 Subject: [PATCH 15/25] fix null pointer dereference in ocsp verification code in case no certificate is sent as part as the ocsp reply. Addresses BIT-1212 There is an additional issue here that prevents the correct verification of proofs in quite a few cases; this will be addressed in a separate commit. --- src/file_analysis/analyzer/x509/functions.bif | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/src/file_analysis/analyzer/x509/functions.bif b/src/file_analysis/analyzer/x509/functions.bif index 9a8a8e78b7..d7903b4921 100644 --- a/src/file_analysis/analyzer/x509/functions.bif +++ b/src/file_analysis/analyzer/x509/functions.bif @@ -250,6 +250,17 @@ function x509_ocsp_verify%(certs: x509_opaque_vector, ocsp_reply: string, root_c // inject the certificates in the certificate list of the OCSP reply, they actually are used during // the lookup. // Yay. + + if ( basic->certs == 0 ) + { + basic->certs = sk_X509_new_null(); + if ( !basic->certs ) + { + rval = x509_result_record(-1, "Could not allocate basic x509 stack"); + goto x509_ocsp_cleanup; + } + } + issuer_certificate = 0; for ( int i = 0; i < sk_X509_num(untrusted_certs); i++) { From 8f1cbb8b0a9a8099e14aa806ae5120f6e8790d8f Mon Sep 17 00:00:00 2001 From: Johanna Amann Date: Thu, 4 Sep 2014 11:15:16 -0700 Subject: [PATCH 16/25] Fix ocsp reply validation - there were a few things that definitely were wrong. Now the right signer certificate for the reply is looked up (and no longer assumed that it is the first one) and a few compares are fixed. Plus - there are more test cases that partially send certificates in the ocsp message and partially do not - and it seems to work fine in all cases. Addresses BIT-1212 --- src/file_analysis/analyzer/x509/functions.bif | 57 ++++++++++++++++-- .../ssl-digicert.log | 10 +++ .../ssl-twimg.log | 10 +++ .../Traces/tls/ocsp-stapling-digicert.trace | Bin 0 -> 6395 bytes .../Traces/tls/ocsp-stapling-twimg.trace | Bin 0 -> 6513 bytes .../policy/protocols/ssl/validate-ocsp.bro | 6 ++ 6 files changed, 79 insertions(+), 4 deletions(-) create mode 100644 testing/btest/Baseline/scripts.policy.protocols.ssl.validate-ocsp/ssl-digicert.log create mode 100644 testing/btest/Baseline/scripts.policy.protocols.ssl.validate-ocsp/ssl-twimg.log create mode 100644 testing/btest/Traces/tls/ocsp-stapling-digicert.trace create mode 100644 testing/btest/Traces/tls/ocsp-stapling-twimg.trace diff --git a/src/file_analysis/analyzer/x509/functions.bif b/src/file_analysis/analyzer/x509/functions.bif index d7903b4921..a3d8258b33 100644 --- a/src/file_analysis/analyzer/x509/functions.bif +++ b/src/file_analysis/analyzer/x509/functions.bif @@ -104,6 +104,39 @@ STACK_OF(X509)* x509_get_untrusted_stack(VectorVal* certs_vec) return untrusted_certs; } +// we need this function to be able to identify the signer certificate of an OCSP request out +// of a list of possible certificates. +X509* x509_get_ocsp_signer(STACK_OF(X509) *certs, OCSP_RESPID *rid) + { + // we support two lookup types - either by response id or by key. + if ( rid->type == V_OCSP_RESPID_NAME ) + return X509_find_by_subject(certs, rid->value.byName); + + // there only should be name and type - but let's be sure... + if ( rid->type != V_OCSP_RESPID_KEY ) + return 0; + + // Just like OpenSSL, we just support SHA-1 lookups and bail out otherwhise. + if ( rid->value.byKey->length != SHA_DIGEST_LENGTH ) + return 0; + + unsigned char* key_hash = rid->value.byKey->data; + for ( int i = 0; i < sk_X509_num(certs); ++i ) + { + unsigned char digest[SHA_DIGEST_LENGTH]; + X509* cert = sk_X509_value(certs, i); + if ( !X509_pubkey_digest(cert, EVP_sha1(), digest, NULL) ) + // digest failed for this certificate, try with next + continue; + + if ( memcmp(digest, key_hash, SHA_DIGEST_LENGTH) == 0 ) + // keys match, return certificate + return cert; + } + + return 0; + } + %%} ## Parses a certificate into an X509::Certificate structure. @@ -221,6 +254,7 @@ function x509_ocsp_verify%(certs: x509_opaque_vector, ocsp_reply: string, root_c int out = -1; int result = -1; X509* issuer_certificate = 0; + X509* signer = 0; OCSP_RESPONSE *resp = d2i_OCSP_RESPONSE(NULL, &start, ocsp_reply->Len()); if ( ! resp ) { @@ -266,14 +300,30 @@ function x509_ocsp_verify%(certs: x509_opaque_vector, ocsp_reply: string, root_c { sk_X509_push(basic->certs, X509_dup(sk_X509_value(untrusted_certs, i))); - if ( X509_NAME_cmp(X509_get_issuer_name(cert), X509_get_subject_name(sk_X509_value(untrusted_certs, i))) ) + if ( X509_NAME_cmp(X509_get_issuer_name(cert), X509_get_subject_name(sk_X509_value(untrusted_certs, i))) == 0 ) issuer_certificate = sk_X509_value(untrusted_certs, i); } // Because we actually want to be able to give nice error messages that show why we were // not able to verify the OCSP response - do our own verification logic first. + signer = x509_get_ocsp_signer(basic->certs, basic->tbsResponseData->responderId); + /* + Do this perhaps - OpenSSL also cannot do it, so I do not really feel bad about it. + Needs a different lookup because the root store is no stack of X509 certs + + if ( !signer ) + // if we did not find it in the certificates that were sent, search in the root store + signer = x509_get_ocsp_signer(basic->certs, basic->tbsResponseData->responderId); + */ + + if ( !signer ) + { + rval = x509_result_record(-1, "Could not find OCSP responder certificate"); + goto x509_ocsp_cleanup; + } + csc = X509_STORE_CTX_new(); - X509_STORE_CTX_init(csc, ctx, sk_X509_value(basic->certs, 0), basic->certs); + X509_STORE_CTX_init(csc, ctx, signer, basic->certs); X509_STORE_CTX_set_time(csc, 0, (time_t) verify_time); X509_STORE_CTX_set_purpose(csc, X509_PURPOSE_OCSP_HELPER); @@ -292,7 +342,6 @@ function x509_ocsp_verify%(certs: x509_opaque_vector, ocsp_reply: string, root_c goto x509_ocsp_cleanup; } - // ok, now we verified the OCSP response. This means that we have a valid chain tying it // to a root that we trust and that the signature also hopefully is valid. This does not yet // mean that the ocsp response actually matches the certificate the server send us or that @@ -333,7 +382,7 @@ function x509_ocsp_verify%(certs: x509_opaque_vector, ocsp_reply: string, root_c goto x509_ocsp_cleanup; } - if ( ! OCSP_id_cmp(certid, single->certId) ) + if ( OCSP_id_cmp(certid, single->certId) != 0 ) return x509_result_record(-1, "OCSP reply is not for host certificate"); // next - check freshness of proof... diff --git a/testing/btest/Baseline/scripts.policy.protocols.ssl.validate-ocsp/ssl-digicert.log b/testing/btest/Baseline/scripts.policy.protocols.ssl.validate-ocsp/ssl-digicert.log new file mode 100644 index 0000000000..bb0a25ac0c --- /dev/null +++ b/testing/btest/Baseline/scripts.policy.protocols.ssl.validate-ocsp/ssl-digicert.log @@ -0,0 +1,10 @@ +#separator \x09 +#set_separator , +#empty_field (empty) +#unset_field - +#path ssl +#open 2014-09-04-19-17-18 +#fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p version cipher curve server_name session_id last_alert established cert_chain_fuids client_cert_chain_fuids subject issuer client_subject client_issuer ocsp_status +#types time string addr port addr port string string string string string string bool vector[string] vector[string] string string string string string +1404148886.994021 CXWv6p3arKYeMETxOg 192.168.4.149 51293 72.21.91.29 443 TLSv10 TLS_ECDHE_RSA_WITH_RC4_128_SHA secp256r1 - - - T FhwjYM0FkbvVCvMf2,Fajs2d2lipsadwoK1h (empty) CN=www.digicert.com,O=DigiCert\, Inc.,L=Lehi,ST=Utah,C=US,postalCode=84043,street=2600 West Executive Parkway,street=Suite 500,serialNumber=5299537-0142,1.3.6.1.4.1.311.60.2.1.2=#130455746168,1.3.6.1.4.1.311.60.2.1.3=#13025553,businessCategory=Private Organization CN=DigiCert SHA2 Extended Validation Server CA,OU=www.digicert.com,O=DigiCert Inc,C=US - - good +#close 2014-09-04-19-17-18 diff --git a/testing/btest/Baseline/scripts.policy.protocols.ssl.validate-ocsp/ssl-twimg.log b/testing/btest/Baseline/scripts.policy.protocols.ssl.validate-ocsp/ssl-twimg.log new file mode 100644 index 0000000000..4806744a5c --- /dev/null +++ b/testing/btest/Baseline/scripts.policy.protocols.ssl.validate-ocsp/ssl-twimg.log @@ -0,0 +1,10 @@ +#separator \x09 +#set_separator , +#empty_field (empty) +#unset_field - +#path ssl +#open 2014-09-04-19-17-14 +#fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p version cipher curve server_name session_id last_alert established cert_chain_fuids client_cert_chain_fuids subject issuer client_subject client_issuer ocsp_status +#types time string addr port addr port string string string string string string bool vector[string] vector[string] string string string string string +1409786981.016881 CXWv6p3arKYeMETxOg 192.168.4.149 53106 93.184.216.146 443 TLSv10 TLS_ECDHE_RSA_WITH_RC4_128_SHA secp256r1 - - - T FtaZVlJfywdNmVFr1,FoILekwkdtTuZtlVa (empty) CN=si0.twimg.com,O=Twitter\, Inc.,L=San Francisco,ST=California,C=US CN=DigiCert High Assurance CA-3,OU=www.digicert.com,O=DigiCert Inc,C=US - - good +#close 2014-09-04-19-17-14 diff --git a/testing/btest/Traces/tls/ocsp-stapling-digicert.trace b/testing/btest/Traces/tls/ocsp-stapling-digicert.trace new file mode 100644 index 0000000000000000000000000000000000000000..982249c0f5ac877bb97e32850978e3136b0f5692 GIT binary patch literal 6395 zcmc&&c|4Tc|3A-cwy}({j3R?B3VCL-riI8FiAY*yj|gK|G8mGgQc>Ac)K!$F8`bnJ zMK|1#O3BTwZlNquTI72knr@cd-}kTIyk6(^nsc7>yg%pj{(R2+bDq<3<3cVD5CHEt zItl;+J4a5G+Di|M02}xpYd|^i?|m_abPw}8tNFnS01Rb^41pmD0B#}Su({-Vx!R3L z)nny=F{l#U=%`m705}4%48h|G2*NKYnq$|50TFQH=(Y1i!E)Hg+BojGjWKjBXaMMt zo~*%;F$+jl?Z)oH5Cv#~xLaJG-y!lFIzqrf3nXm4V)I3T4n)J+l=Z5fcbj&-M2M|^t)l0qHby2cO0Y!yL0`ocMoDaeXB18meFhd9xAmJSdBr49qo*{`+v5p{>Y~tgY78)A5zzsffg^w1v`fq1wp-k)%o~ri!4SRDVCVfR1%iYh- z-Hq<(;_L0^!t(a_quaX&hPVgPjg8Pb(!%gfEmR*J`#Hm|4ywgqGMHH38D%3v3^{Z* zhAc{!unqJMabdaBR|R^y_<2W+fib8kg(Rd#LI?<3cQM>w-`D@cAdX6KV7Ykl+#*uR z+Dv_Y?S;A;C_{_M5J$x@3j$OD`(ST~uZ^ONxv1j!*leZ_iqhA*2eF_p?ykWs?+|yo ztxKR!s7p9Q1SMm$C{&^zbYtN}5`|GA>>>X*o4Dv3Q6q>AteQ^z=7 zz#|zym1IbV;*Wtthyc$C2eg7Uy+ur@W?LvsOhn(-Ba*hmxF$i&m$4T0vV^Aee_4jrNIlnM=qHW4_irY z;e7RfR5s9S;V1ug&z!VIN|%8hL-vbf^Y3Lo$8p(q3I`fYExnb`M^;p8>X@(bS|u`& z`r_98x5vtxZ@stO=CfhMW7)mNT~ax0Jqi%5Tkqz?TG>B>-&C|$qBNz~gOyoExM%Cg zFCv(ybJcQlOGh#debs+jJ*VS^7wOJFk0bAVN&C{JR%@}i1@x!Di68Z4R_kTDv7@ICn@I!r3tsG_!%30vWV(nO@sp%T%JJFhEke3Tn!#zYJ zXlyfTcrq5iIaA-7xSpT9wHz}`Cm6tetV!o}W{yu0UnWIKxTx__Mq3 ztKC*Sp?Zr5Jd?Hi{Q7$>)mwlhU~M|{iN z?3}$19cnr={&)g@@tGfMsk-U$IU(|MoR>-BQFM-W+mc_;|H?@0?|Sw5vF;I%l^Qw? z@GI|_8`dv{(wx$pY1M<->Dr>+vw+7yun4rKUUabr3E^+ab?X)xRL z%Q1mDfN&XQ6Dm+PKI>bRB8L~5lq6FQ(+|3A?(~t1oy0xrJ5gZ&_bR1^&V#86tCS)y z9q8uXo?djLprGJD7e7~b`U*$7oxeYeSDY3yP$sV$NeeUee(3&x)y#h@0uSs@5>^jq zoRTuP@>Dr<$l?$EH{_BMkkId`%85_j@7h*fYN*%QJk(P_q+MFFv?fEE*+WbI)K5O; zrV#%-pO{vOU80)2*YT#(6!DUaO@AlK%lGPquCd>|JAB!KS!W98Sg8r0t#tiOyK<*Q z^*XDL)s{`Uo#f6`cEYxET@Tz^(;Z%ooS$XuyQZ2lFk(57SrsiZc>n5tsXgKj54iza zZ{I)B=((Bj3uoU$h36&KF+(&wM%_bMm_&nnh9&e2Xj5n3Kp{4h=9`8#sv+v_%4cS&;tvecjUe-u-dS@^98 zAQrGJu0X{xX&T$h{3r=-X?V1pD5?nO-$Yf5Coxc%R8i&sqcB;GOR)*M95u{YnzNX* zh^J10zLP|imy1{`e$T`ej#wU!DUvU`WL#k3*$5u92vBRm|D>(|6uN2a;E#o}bm7C)}n)lKcjq-_Y#906`jDveqlhdaq%euZV`qC>?c+&(aeqcJ*3( z*)GN@JYSvM@pOAPW8iX5lyAR!%d;z?ak}^hW_j(gm|2zwZ46A0#ea>>`~oE^PRD(*P5I40mxX5k8h!58oIR=3}d-1^F&%<8_> zHhe8XINbI?>T#OE0ZjQNq4G!L$J^vHKR%%TCZ_yC_7jyqCTUFhX<%qn1AuiZ{o1P3 zv&y^fe~5sGcBDfJAdlg>= z6J^4)EUL2*MfJ3`u(NXz8h~2BLp@PV1dp2qzq2GP2CpR1u7!}&Yn~cvco_t2d;XF$ zJ=)X{hTUL_g&fMaAzf4#R(^Pfcb+a&?q!P|N%9Vwy#gnZXc*tye#l0m*7L#k5cRhO zH_J}(d8I_~bzCZw+5UMyb=lIoFHf&m>G5~Ze?u8zKb*&HIN&^6pDZ7;*sDQ5y6JWv zEq+d)=ab{ZkB_PD8Wgp=dnB%p+@t^Q-RU-wuxo{1JI_iTiP=(4VK2VuIWElSLJr+9u{LOGC<7mK-!3)lA@X=D8wjxmA?e# zlR3!eDx;`puotB6i7Pj-CDnXL%M<$C(|V$pLr<+*J9GgJKey%hA=%1P$;#Ze&k;#F zP`*})1-dG)zZ1zS*;%?aD5^42$~s-^eww)TjprkMtA<=JIbW_3h;NAgGwodKnG?|g zLl54VT(k3|vx*Put+~RV@FsBH?@IAzzBhAseEz-DBE9<{%_Dl)J2 zy56Dl~Q*XM@2{pFLYCZJn7 z)@f7jx&qTri(WdX7TC|S)}oDQ)w{R&`kRXzr}R8_Tc$=}D&^Oso`=(zAIg zP#>O+U0D!BzI}pQ!R`3-dz87ero90f=q`*&vK3oqgWaQGn|jrZDEYBJLy2pKC^9@0 zam|_5@(bULC|x|1_qqU-K<}_N`A!GqWm0EA!R~b6ofbxrsTjOeFd{sElO=Ja;$Wm> z9nml%!|S!5U)>-nz4_?dnuOMsk20eh4~kiT3|gmp#s2Q1jk5fiWhn-q1q$!sM+0g% z=)E%eRRa5}Xxey&16BO7hUM?HHGf z0sxSJcNAEgERYDPz6KLW;VTmbGOqcta-gp*6v!)O#c0KF%O!_1tPkPbhO;Az7ZW~xi9y<=PhZPVSd=?@`X!&4 z+ilP#nYC!cwSc3J^wi`tccpe`E^WN`pD$}OAN6*+M~?Pp%+G7eN_%%dLgKnyD+{K=Gy?4Dd`Y}R!jsAFiEgBB~#28tQk;Xrg#+rAPoaME1a1eem&=JTZhUm zjo@hWP{7mT96oz}-$pCUNKiWwR6~5MO%5kGSAPo&r-;PFaK@LyaPGyzdEIB+NDOyS z`P@z`gOgm7gq_KKl zYzbLQNvF_!>tCjyb|xYNz8xXVz!K#!0jI<)rR6c&j+co;O z@^%+dA)4-h6Gil-Q+0CK-Vd+J8+FqiH(ta~cb)HVEi@Jqq_Dtk*$%s@CafL$zNZHfWa`r0uA>5~lDjTxYT zq3S~T+Sm`n>VQ~3&yfrBr;YC3?m~_(%#jbg9C~^e<%u5eDwA}!stOy$TIc3f& z(;7GhDYLhwj9;o_C{m)N8G@)HhXt|Rcv2AUvAEen(GuD)9>Vc0DmydQoH4BB|Bm%v DA!xCz literal 0 HcmV?d00001 diff --git a/testing/btest/Traces/tls/ocsp-stapling-twimg.trace b/testing/btest/Traces/tls/ocsp-stapling-twimg.trace new file mode 100644 index 0000000000000000000000000000000000000000..f53762f6e0cf1a6c48338a95b48bffafd8e46997 GIT binary patch literal 6513 zcmcgwdpwle*Wb_FTn6KQOPX>yQg~cL?juQ&k}gis7-NjwVvwRVN0c8*bt)=Jlsc7@ z&_!vwoSag+DJ3069i@;&mx&4FDhbj0P}V-=4j;lpgANb&fQc27t??$u7W!0)U{D^!t*N_cz~& z#i~oJVpb#JKM9{d3r_<;AdxFDB9Vk)(y~TQ#ZVdm5@85E-vt0SNTV@?JETK|ZUq$p z8_Em^gp4#`)i+{ulLuU&hBc|mcG8z_52Irw0@Of3_F}F8*g!NizOmQnjn8-OT?0{F zeHC4RuxQAx9*Cj=U4X`D6ilWm?IU>(p>DBp&brw4n|L2NCoomfc#q^wHfD2<=}iuLLrfW5eSNM+Wl4oJ8x~*Uz{?p zVqj_}E>Dp$PWMcICy!xdGQdO4skjV<=ue`k5d8hj)o>L=$g0S>b2(fFJC;YE5ydjs z!Zi?^s-l>fm}nXTr&w^xlocImZh-5dDWZzbmld8|P8i)aE-pTn8O37L8LlRlxUPo0 zxfO0F z9tfQq$BGv7WmV+oC31N@cI;$iw5ga+RFRM4;-*#&wY@88s==GCpJs#|FAnf+?Oo}RY)R_o+P-gXSzN1I zIp$X69^!TGN&|YM!>5lHv^03%DU0xUzk1fO zye)Za??y5P2(TVt`}iaaoKG^v$3dTu)gx)*>Zz)}9dR2TP82F`EO>^|8~#bk^Ze}{ zJ{Acmkkm=4g}WPWyJt)ZE-%Y>kkJY6z(Vg`!}){_NI;3iC!|6mnJ*a)z8ok1YiuB8 zF@lm$FlB{AnI>?f*f4T@S|pDp!JxvRqJX2Y@@XhHM>15aV;fhtt3h75hN|f zB!rzXf}sI-C1Q=^5R8yi*p25K&eQ>u+JnCkw7jS%3T8zsw~ozD6k?*5wR~%1e}S)$B_c? zg}8Acs{Dy5Kdt=1qRMYBQTbB{ec~>X6b*n;p<%jy7>^fYZ)V1djj)ucre-Xrr3uHv z))dY_jiY3(ek|*T45gE%;&%8Hipf})+(^6~0;aBm^u|HOqpP?X!`BaA zC6>f6QZyck>x*d|OV=E>IGPm~BVh$T?TZv4-A=qOCTb6H5`t#p_JF#>m*#+a1KcbR zmqQy+L4hIQU}SZMO$t_L40s*58Te*(@JHu9P9XPcffUEjlxaoJVpV~zs=_sbRqh@Q zjTwRQ%l_22i)K|WTT#D=>lC}_n5O4v-KK5(-qu@k6!NZ-SdYDPlc(kA&;1B)z716S z*tjtGQLy5wI5NXeV{t46NI?d}F*mh;q6 zUc)0KTk8M4wNJt)!l=O3K0zEi`l!}EZ9aW&bTkjPc+}u=TO79d;f}5$Z(#vR%i(_i z|MsHs-Kxvt+{^}0~4T6*bYx98-E ziWxSUM(omVb17b=&2}>*k5E#ragyQ6yt_FUBWD)|9y#NfdtuS4K${4G-whyb;iMR{ zP*H_jE7!HlZ4UTAcJI=!UpcNbOX;j~X|LK_ot5TXY|Mdwmk9DBR?8H$&vo}wwAXvO zvx-<&x~t;XJqz%a@Y)E!m{hLDWHu8>(<>T7^ zX1_eRYDwYbYo8n*M0Q^LIFpferQu`=Q>Wae%1F88)cc^pJLOlp9V>KaChN?~vnglJK#!kT?ZLC50-kCqzJijAGng{GW$4 z#dP>T|4m7Wi;FN}#l)G!vSXs-xV-4tBq)krzCymNMriYO`Pf`|PKW|YfDYn89ALxW zXb=j&c_0jMA%_kakQ)nWF646o3ozj`l4C*1C@6tucyKl1B6%j9VZrAZ$YDYW^gI$3 zQSNXJ2M#ed(&7tGE$G`%gb@Wo;4j34m4DCAD?dwA`Ekk0FMUv4`JV~_+zZk=7k)>; zy>g*d97y{@`_RtN?p+}#7S6LlBpgwYO9zP}AF$vz9j@cT8MM=(XC?@S6ygr8M>|Cp zBSfSN=|CQeg?>QaLtaA?$QC4nB9~2iHiDofvJ}Nd0S5bHT24Pfp{O<@O zUu=&#UE~tqpPP;~>@d`pCUmnxCS4ryV*B0qUdfY`s7dLzmUtFKJoK}rb??_UKHN%V zbmpIN%84KIdcbtTqQfczW^4P$7kSq+Wa8%;o%2}QTr+3#=H4;6$>!2Aty4~&|F!h& zJ}MAus|^s3EDZmvRzJ^kx`(;$=I)9hAFJ9;=giN)KbDq$a~F1~C38Nd-u{Kc(e1n? zjmz`hctQVUGm`&#;or#xb>zn@nM})DTHYU|y9(|8GO#`|=@PB{QBTPU1^r8gM%hV; zeP*v0oKBW%c=G9b0KQ;#y5jUl=HnVa6otzUN&{ngtFIjCQ!CxwVBIIw8j4%kdfT ztV*7wKqP3x%LLU<&m)iOR)JQ9G~9+W(l+t2q{#ZS=5(mjp()#|hdg?R!(Sj%#`b-BJ5!R=9ZEfL?8U)122v4iBoE zX-~EFc53=fT|s#JR8}sHwX^P^uzGO6cHh%Y0Wa6&K9=uURLUvKPiCA*_H1eUC+cNs zb7y~MM#uVj`_^CgJ7wHDKlhqC@k#c7njR3AfAHLPF;rR7BdzV_Y*s}+Y`d`j!lfM{scZmt^wm(zmS#(Z&bG-8l{~PMh`h9xh_o@ap z@>{0utMqEvf?_;V4q(%XK*N>9NVA`t5PN!J`uiPCvJCB03D|_|1t+DhHoau3zo^_B z+7)hl(b`QXslV&4=7)Lrk1{CI?~3MRPV%^C;aQ*P#mB%O9e-S`>B7PuY9D#p8oa3S zjYUU;`n;9(Cgp`%)V!K-owB?C89Z-)T+!B2qh$62ztVM?)_*MZp!Qal2Kf2!wrcpj z{$Exk)p_IiTK^X(4RYDtbz16|!;5#g7C*r}#wg9x(}_&0F51${3t+y`m{?ie*2-2l zZa%c(`YsMZ&5?F;{JKT5_YPC{m)KXXHP~cgaDUUYj)Wa8nO>Hy_Nej~!O9Qs+C*dY zZ4cNg3=!QNB}!I)g(afOuL=4pya7mwxOT~7|1Q$w+l6;^PHYywxgDCnMwpOr=bF(K zYv!PFziGz3GGTSy-1#HjZ5*W!LdU=2vyOf%56ai{Ex< zIb61VR(&PN_V6uw{%aqyiQAatjxZ{Kq%eFxRSYfu@x@|b&iAeamoLptkv{0F=v1KutMwQ1jD1PEzeKNXL4Jqe@KUi`%5P!Gq!#7 zlGSbt2!By))bkNqSNKGB;y%^Ox4l{Od-y+XX2)zc+jLYMsY_ zqv(&T20#@X8l$6+QG{$!^hu+le+5Rr$rnbS3B!QY9fCL3`{7OXWKif^^Sgpn`NIy% z;GqvG4;HVfifnGpI%^bQ@+h?6>MY)y4MFV#$6uV>bL(nTckeCJz$V`*sg8A9aty2N zjPo)d*7`WsIz>Dl@3P>td-K>geeXVn9y&-DE_3vL_sX60_LY)ztVP8h=cI>jj^T3N zoqo5+*ZneM0qxSR@bY8*AvvLDwGWOSPO2r~%k$fa+7?HaoWG@gyeGpk`Y)Umdo18p z_y`|eF*EM8ByVoi*}1LJhSaBLOSx%mPa0?{ZOdAbzP5AI{^GDW^}ZcE_-f|pga8a-s3;*WOC|*E08EH}ln}Hd z!)6ZZtIP{H&@f)fe{9W+Y3#B2{O$8kEJkKJLj^zfX2pr`n%R$><_oh->`ixwg6_J} b_{K~lAxC89<56Z_5Sgh2`X<&OJoNb=RnO#{ literal 0 HcmV?d00001 diff --git a/testing/btest/scripts/policy/protocols/ssl/validate-ocsp.bro b/testing/btest/scripts/policy/protocols/ssl/validate-ocsp.bro index b0392f9c27..e7e3c3ff8e 100644 --- a/testing/btest/scripts/policy/protocols/ssl/validate-ocsp.bro +++ b/testing/btest/scripts/policy/protocols/ssl/validate-ocsp.bro @@ -1,4 +1,10 @@ # @TEST-EXEC: bro -C -r $TRACES/tls/ocsp-stapling.trace %INPUT # @TEST-EXEC: btest-diff ssl.log +# @TEST-EXEC: bro -C -r $TRACES/tls/ocsp-stapling-twimg.trace %INPUT +# @TEST-EXEC: mv ssl.log ssl-twimg.log +# @TEST-EXEC: btest-diff ssl-twimg.log +# @TEST-EXEC: bro -C -r $TRACES/tls/ocsp-stapling-digicert.trace %INPUT +# @TEST-EXEC: mv ssl.log ssl-digicert.log +# @TEST-EXEC: btest-diff ssl-digicert.log @load protocols/ssl/validate-ocsp From 42979b89f766ddf2d7852d286e8033824e368174 Mon Sep 17 00:00:00 2001 From: Johanna Amann Date: Thu, 4 Sep 2014 16:02:10 -0700 Subject: [PATCH 17/25] Also make links in documentation templates protocol relative. In case we do a point release, including this commit would be very helpful - without the bro documentation pages are somewhat broken over https. --- doc/_templates/layout.html | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/doc/_templates/layout.html b/doc/_templates/layout.html index 2f8ea02aff..3df56a12ff 100644 --- a/doc/_templates/layout.html +++ b/doc/_templates/layout.html @@ -10,7 +10,7 @@ {% endblock %} {% block header %} - {% endblock %} @@ -108,6 +108,6 @@ {% endblock %} {% block footer %} - {% endblock %} From b813b6f83bf0af218a1a84a36e600d2cce8700f2 Mon Sep 17 00:00:00 2001 From: Robin Sommer Date: Thu, 4 Sep 2014 16:08:14 -0700 Subject: [PATCH 18/25] Test updates. --- testing/btest/Baseline/plugins.hooks/output | 50 ++++++++++++------- .../btest/Baseline/plugins.pktsrc/conn.log | 4 +- testing/btest/Baseline/plugins.writer/output | 22 ++++---- testing/btest/plugins/api-version-mismatch.sh | 2 +- .../btest/plugins/bifs-and-scripts-install.sh | 3 +- testing/btest/plugins/bifs-and-scripts.sh | 2 +- testing/btest/plugins/file.bro | 2 +- testing/btest/plugins/hooks.bro | 2 +- testing/btest/plugins/init-plugin.bro | 2 +- testing/btest/plugins/pktsrc.bro | 2 +- .../plugins/protocol-plugin/CMakeLists.txt | 1 - testing/btest/plugins/protocol.bro | 2 +- testing/btest/plugins/reader.bro | 2 +- testing/btest/plugins/writer.bro | 6 +-- 14 files changed, 57 insertions(+), 45 deletions(-) diff --git a/testing/btest/Baseline/plugins.hooks/output b/testing/btest/Baseline/plugins.hooks/output index 7b0f9262ae..83341f3075 100644 --- a/testing/btest/Baseline/plugins.hooks/output +++ b/testing/btest/Baseline/plugins.hooks/output @@ -182,7 +182,7 @@ 0.000000 MetaHookPost CallFunction(Log::__create_stream, (Unified2::LOG, [columns=, ev=Unified2::log_unified2])) -> 0.000000 MetaHookPost CallFunction(Log::__create_stream, (Weird::LOG, [columns=, ev=Weird::log_weird])) -> 0.000000 MetaHookPost CallFunction(Log::__create_stream, (X509::LOG, [columns=, ev=X509::log_x509])) -> -0.000000 MetaHookPost CallFunction(Log::__write, (PacketFilter::LOG, [ts=1405981560.501473, node=bro, filter=ip or not ip, init=T, success=T])) -> +0.000000 MetaHookPost CallFunction(Log::__write, (PacketFilter::LOG, [ts=1409853900.737227, node=bro, filter=ip or not ip, init=T, success=T])) -> 0.000000 MetaHookPost CallFunction(Log::add_default_filter, (Cluster::LOG)) -> 0.000000 MetaHookPost CallFunction(Log::add_default_filter, (Communication::LOG)) -> 0.000000 MetaHookPost CallFunction(Log::add_default_filter, (Conn::LOG)) -> @@ -273,8 +273,8 @@ 0.000000 MetaHookPost CallFunction(Log::create_stream, (Unified2::LOG, [columns=, ev=Unified2::log_unified2])) -> 0.000000 MetaHookPost CallFunction(Log::create_stream, (Weird::LOG, [columns=, ev=Weird::log_weird])) -> 0.000000 MetaHookPost CallFunction(Log::create_stream, (X509::LOG, [columns=, ev=X509::log_x509])) -> -0.000000 MetaHookPost CallFunction(Log::default_path_func, (PacketFilter::LOG, , [ts=1405981560.501473, node=bro, filter=ip or not ip, init=T, success=T])) -> -0.000000 MetaHookPost CallFunction(Log::write, (PacketFilter::LOG, [ts=1405981560.501473, node=bro, filter=ip or not ip, init=T, success=T])) -> +0.000000 MetaHookPost CallFunction(Log::default_path_func, (PacketFilter::LOG, , [ts=1409853900.737227, node=bro, filter=ip or not ip, init=T, success=T])) -> +0.000000 MetaHookPost CallFunction(Log::write, (PacketFilter::LOG, [ts=1409853900.737227, node=bro, filter=ip or not ip, init=T, success=T])) -> 0.000000 MetaHookPost CallFunction(Notice::want_pp, ()) -> 0.000000 MetaHookPost CallFunction(PacketFilter::build, ()) -> 0.000000 MetaHookPost CallFunction(PacketFilter::combine_filters, (ip or not ip, and, )) -> @@ -316,7 +316,11 @@ 0.000000 MetaHookPost LoadFile(../main) -> -1 0.000000 MetaHookPost LoadFile(./Bro_ARP.events.bif.bro) -> -1 0.000000 MetaHookPost LoadFile(./Bro_AYIYA.events.bif.bro) -> -1 +0.000000 MetaHookPost LoadFile(./Bro_AsciiReader.ascii.bif.bro) -> -1 +0.000000 MetaHookPost LoadFile(./Bro_AsciiWriter.ascii.bif.bro) -> -1 0.000000 MetaHookPost LoadFile(./Bro_BackDoor.events.bif.bro) -> -1 +0.000000 MetaHookPost LoadFile(./Bro_BenchmarkReader.benchmark.bif.bro) -> -1 +0.000000 MetaHookPost LoadFile(./Bro_BinaryReader.binary.bif.bro) -> -1 0.000000 MetaHookPost LoadFile(./Bro_BitTorrent.events.bif.bro) -> -1 0.000000 MetaHookPost LoadFile(./Bro_ConnSize.events.bif.bro) -> -1 0.000000 MetaHookPost LoadFile(./Bro_DCE_RPC.events.bif.bro) -> -1 @@ -347,16 +351,20 @@ 0.000000 MetaHookPost LoadFile(./Bro_NetBIOS.events.bif.bro) -> -1 0.000000 MetaHookPost LoadFile(./Bro_NetBIOS.functions.bif.bro) -> -1 0.000000 MetaHookPost LoadFile(./Bro_NetFlow.events.bif.bro) -> -1 +0.000000 MetaHookPost LoadFile(./Bro_NoneWriter.none.bif.bro) -> -1 0.000000 MetaHookPost LoadFile(./Bro_PIA.events.bif.bro) -> -1 0.000000 MetaHookPost LoadFile(./Bro_POP3.events.bif.bro) -> -1 0.000000 MetaHookPost LoadFile(./Bro_RADIUS.events.bif.bro) -> -1 0.000000 MetaHookPost LoadFile(./Bro_RPC.events.bif.bro) -> -1 +0.000000 MetaHookPost LoadFile(./Bro_RawReader.raw.bif.bro) -> -1 0.000000 MetaHookPost LoadFile(./Bro_SMB.events.bif.bro) -> -1 0.000000 MetaHookPost LoadFile(./Bro_SMTP.events.bif.bro) -> -1 0.000000 MetaHookPost LoadFile(./Bro_SMTP.functions.bif.bro) -> -1 0.000000 MetaHookPost LoadFile(./Bro_SNMP.events.bif.bro) -> -1 0.000000 MetaHookPost LoadFile(./Bro_SNMP.types.bif.bro) -> -1 0.000000 MetaHookPost LoadFile(./Bro_SOCKS.events.bif.bro) -> -1 +0.000000 MetaHookPost LoadFile(./Bro_SQLiteReader.sqlite.bif.bro) -> -1 +0.000000 MetaHookPost LoadFile(./Bro_SQLiteWriter.sqlite.bif.bro) -> -1 0.000000 MetaHookPost LoadFile(./Bro_SSH.events.bif.bro) -> -1 0.000000 MetaHookPost LoadFile(./Bro_SSL.events.bif.bro) -> -1 0.000000 MetaHookPost LoadFile(./Bro_SteppingStone.events.bif.bro) -> -1 @@ -380,21 +388,20 @@ 0.000000 MetaHookPost LoadFile(./cardinality-counter.bif.bro) -> -1 0.000000 MetaHookPost LoadFile(./const.bif.bro) -> -1 0.000000 MetaHookPost LoadFile(./consts) -> -1 -0.000000 MetaHookPost LoadFile(./consts.bif.bro) -> -1 0.000000 MetaHookPost LoadFile(./consts.bro) -> -1 0.000000 MetaHookPost LoadFile(./contents) -> -1 0.000000 MetaHookPost LoadFile(./dcc-send) -> -1 0.000000 MetaHookPost LoadFile(./entities) -> -1 0.000000 MetaHookPost LoadFile(./event.bif.bro) -> -1 -0.000000 MetaHookPost LoadFile(./events.bif.bro) -> -1 0.000000 MetaHookPost LoadFile(./exec) -> -1 0.000000 MetaHookPost LoadFile(./file_analysis.bif.bro) -> -1 0.000000 MetaHookPost LoadFile(./files) -> -1 -0.000000 MetaHookPost LoadFile(./functions.bif.bro) -> -1 0.000000 MetaHookPost LoadFile(./gridftp) -> -1 0.000000 MetaHookPost LoadFile(./hll_unique) -> -1 +0.000000 MetaHookPost LoadFile(./hooks.bif.bro) -> -1 0.000000 MetaHookPost LoadFile(./inactivity) -> -1 0.000000 MetaHookPost LoadFile(./info) -> -1 +0.000000 MetaHookPost LoadFile(./init.bro) -> -1 0.000000 MetaHookPost LoadFile(./input) -> -1 0.000000 MetaHookPost LoadFile(./input.bif.bro) -> -1 0.000000 MetaHookPost LoadFile(./last) -> -1 @@ -408,6 +415,7 @@ 0.000000 MetaHookPost LoadFile(./netstats) -> -1 0.000000 MetaHookPost LoadFile(./non-cluster) -> -1 0.000000 MetaHookPost LoadFile(./patterns) -> -1 +0.000000 MetaHookPost LoadFile(./pcap.bif.bro) -> -1 0.000000 MetaHookPost LoadFile(./plugins) -> -1 0.000000 MetaHookPost LoadFile(./polling) -> -1 0.000000 MetaHookPost LoadFile(./postprocessors) -> -1 @@ -432,9 +440,7 @@ 0.000000 MetaHookPost LoadFile(.<...>/ascii) -> -1 0.000000 MetaHookPost LoadFile(.<...>/benchmark) -> -1 0.000000 MetaHookPost LoadFile(.<...>/binary) -> -1 -0.000000 MetaHookPost LoadFile(.<...>/dataseries) -> -1 0.000000 MetaHookPost LoadFile(.<...>/drop) -> -1 -0.000000 MetaHookPost LoadFile(.<...>/elasticsearch) -> -1 0.000000 MetaHookPost LoadFile(.<...>/email_admin) -> -1 0.000000 MetaHookPost LoadFile(.<...>/hostnames) -> -1 0.000000 MetaHookPost LoadFile(.<...>/none) -> -1 @@ -699,7 +705,7 @@ 0.000000 MetaHookPre CallFunction(Log::__create_stream, (Unified2::LOG, [columns=, ev=Unified2::log_unified2])) 0.000000 MetaHookPre CallFunction(Log::__create_stream, (Weird::LOG, [columns=, ev=Weird::log_weird])) 0.000000 MetaHookPre CallFunction(Log::__create_stream, (X509::LOG, [columns=, ev=X509::log_x509])) -0.000000 MetaHookPre CallFunction(Log::__write, (PacketFilter::LOG, [ts=1405981560.501473, node=bro, filter=ip or not ip, init=T, success=T])) +0.000000 MetaHookPre CallFunction(Log::__write, (PacketFilter::LOG, [ts=1409853900.737227, node=bro, filter=ip or not ip, init=T, success=T])) 0.000000 MetaHookPre CallFunction(Log::add_default_filter, (Cluster::LOG)) 0.000000 MetaHookPre CallFunction(Log::add_default_filter, (Communication::LOG)) 0.000000 MetaHookPre CallFunction(Log::add_default_filter, (Conn::LOG)) @@ -790,8 +796,8 @@ 0.000000 MetaHookPre CallFunction(Log::create_stream, (Unified2::LOG, [columns=, ev=Unified2::log_unified2])) 0.000000 MetaHookPre CallFunction(Log::create_stream, (Weird::LOG, [columns=, ev=Weird::log_weird])) 0.000000 MetaHookPre CallFunction(Log::create_stream, (X509::LOG, [columns=, ev=X509::log_x509])) -0.000000 MetaHookPre CallFunction(Log::default_path_func, (PacketFilter::LOG, , [ts=1405981560.501473, node=bro, filter=ip or not ip, init=T, success=T])) -0.000000 MetaHookPre CallFunction(Log::write, (PacketFilter::LOG, [ts=1405981560.501473, node=bro, filter=ip or not ip, init=T, success=T])) +0.000000 MetaHookPre CallFunction(Log::default_path_func, (PacketFilter::LOG, , [ts=1409853900.737227, node=bro, filter=ip or not ip, init=T, success=T])) +0.000000 MetaHookPre CallFunction(Log::write, (PacketFilter::LOG, [ts=1409853900.737227, node=bro, filter=ip or not ip, init=T, success=T])) 0.000000 MetaHookPre CallFunction(Notice::want_pp, ()) 0.000000 MetaHookPre CallFunction(PacketFilter::build, ()) 0.000000 MetaHookPre CallFunction(PacketFilter::combine_filters, (ip or not ip, and, )) @@ -833,7 +839,11 @@ 0.000000 MetaHookPre LoadFile(../main) 0.000000 MetaHookPre LoadFile(./Bro_ARP.events.bif.bro) 0.000000 MetaHookPre LoadFile(./Bro_AYIYA.events.bif.bro) +0.000000 MetaHookPre LoadFile(./Bro_AsciiReader.ascii.bif.bro) +0.000000 MetaHookPre LoadFile(./Bro_AsciiWriter.ascii.bif.bro) 0.000000 MetaHookPre LoadFile(./Bro_BackDoor.events.bif.bro) +0.000000 MetaHookPre LoadFile(./Bro_BenchmarkReader.benchmark.bif.bro) +0.000000 MetaHookPre LoadFile(./Bro_BinaryReader.binary.bif.bro) 0.000000 MetaHookPre LoadFile(./Bro_BitTorrent.events.bif.bro) 0.000000 MetaHookPre LoadFile(./Bro_ConnSize.events.bif.bro) 0.000000 MetaHookPre LoadFile(./Bro_DCE_RPC.events.bif.bro) @@ -864,16 +874,20 @@ 0.000000 MetaHookPre LoadFile(./Bro_NetBIOS.events.bif.bro) 0.000000 MetaHookPre LoadFile(./Bro_NetBIOS.functions.bif.bro) 0.000000 MetaHookPre LoadFile(./Bro_NetFlow.events.bif.bro) +0.000000 MetaHookPre LoadFile(./Bro_NoneWriter.none.bif.bro) 0.000000 MetaHookPre LoadFile(./Bro_PIA.events.bif.bro) 0.000000 MetaHookPre LoadFile(./Bro_POP3.events.bif.bro) 0.000000 MetaHookPre LoadFile(./Bro_RADIUS.events.bif.bro) 0.000000 MetaHookPre LoadFile(./Bro_RPC.events.bif.bro) +0.000000 MetaHookPre LoadFile(./Bro_RawReader.raw.bif.bro) 0.000000 MetaHookPre LoadFile(./Bro_SMB.events.bif.bro) 0.000000 MetaHookPre LoadFile(./Bro_SMTP.events.bif.bro) 0.000000 MetaHookPre LoadFile(./Bro_SMTP.functions.bif.bro) 0.000000 MetaHookPre LoadFile(./Bro_SNMP.events.bif.bro) 0.000000 MetaHookPre LoadFile(./Bro_SNMP.types.bif.bro) 0.000000 MetaHookPre LoadFile(./Bro_SOCKS.events.bif.bro) +0.000000 MetaHookPre LoadFile(./Bro_SQLiteReader.sqlite.bif.bro) +0.000000 MetaHookPre LoadFile(./Bro_SQLiteWriter.sqlite.bif.bro) 0.000000 MetaHookPre LoadFile(./Bro_SSH.events.bif.bro) 0.000000 MetaHookPre LoadFile(./Bro_SSL.events.bif.bro) 0.000000 MetaHookPre LoadFile(./Bro_SteppingStone.events.bif.bro) @@ -897,21 +911,20 @@ 0.000000 MetaHookPre LoadFile(./cardinality-counter.bif.bro) 0.000000 MetaHookPre LoadFile(./const.bif.bro) 0.000000 MetaHookPre LoadFile(./consts) -0.000000 MetaHookPre LoadFile(./consts.bif.bro) 0.000000 MetaHookPre LoadFile(./consts.bro) 0.000000 MetaHookPre LoadFile(./contents) 0.000000 MetaHookPre LoadFile(./dcc-send) 0.000000 MetaHookPre LoadFile(./entities) 0.000000 MetaHookPre LoadFile(./event.bif.bro) -0.000000 MetaHookPre LoadFile(./events.bif.bro) 0.000000 MetaHookPre LoadFile(./exec) 0.000000 MetaHookPre LoadFile(./file_analysis.bif.bro) 0.000000 MetaHookPre LoadFile(./files) -0.000000 MetaHookPre LoadFile(./functions.bif.bro) 0.000000 MetaHookPre LoadFile(./gridftp) 0.000000 MetaHookPre LoadFile(./hll_unique) +0.000000 MetaHookPre LoadFile(./hooks.bif.bro) 0.000000 MetaHookPre LoadFile(./inactivity) 0.000000 MetaHookPre LoadFile(./info) +0.000000 MetaHookPre LoadFile(./init.bro) 0.000000 MetaHookPre LoadFile(./input) 0.000000 MetaHookPre LoadFile(./input.bif.bro) 0.000000 MetaHookPre LoadFile(./last) @@ -925,6 +938,7 @@ 0.000000 MetaHookPre LoadFile(./netstats) 0.000000 MetaHookPre LoadFile(./non-cluster) 0.000000 MetaHookPre LoadFile(./patterns) +0.000000 MetaHookPre LoadFile(./pcap.bif.bro) 0.000000 MetaHookPre LoadFile(./plugins) 0.000000 MetaHookPre LoadFile(./polling) 0.000000 MetaHookPre LoadFile(./postprocessors) @@ -949,9 +963,7 @@ 0.000000 MetaHookPre LoadFile(.<...>/ascii) 0.000000 MetaHookPre LoadFile(.<...>/benchmark) 0.000000 MetaHookPre LoadFile(.<...>/binary) -0.000000 MetaHookPre LoadFile(.<...>/dataseries) 0.000000 MetaHookPre LoadFile(.<...>/drop) -0.000000 MetaHookPre LoadFile(.<...>/elasticsearch) 0.000000 MetaHookPre LoadFile(.<...>/email_admin) 0.000000 MetaHookPre LoadFile(.<...>/hostnames) 0.000000 MetaHookPre LoadFile(.<...>/none) @@ -1216,7 +1228,7 @@ 0.000000 | HookCallFunction Log::__create_stream(Unified2::LOG, [columns=, ev=Unified2::log_unified2]) 0.000000 | HookCallFunction Log::__create_stream(Weird::LOG, [columns=, ev=Weird::log_weird]) 0.000000 | HookCallFunction Log::__create_stream(X509::LOG, [columns=, ev=X509::log_x509]) -0.000000 | HookCallFunction Log::__write(PacketFilter::LOG, [ts=1405981560.501473, node=bro, filter=ip or not ip, init=T, success=T]) +0.000000 | HookCallFunction Log::__write(PacketFilter::LOG, [ts=1409853900.737227, node=bro, filter=ip or not ip, init=T, success=T]) 0.000000 | HookCallFunction Log::add_default_filter(Cluster::LOG) 0.000000 | HookCallFunction Log::add_default_filter(Communication::LOG) 0.000000 | HookCallFunction Log::add_default_filter(Conn::LOG) @@ -1307,8 +1319,8 @@ 0.000000 | HookCallFunction Log::create_stream(Unified2::LOG, [columns=, ev=Unified2::log_unified2]) 0.000000 | HookCallFunction Log::create_stream(Weird::LOG, [columns=, ev=Weird::log_weird]) 0.000000 | HookCallFunction Log::create_stream(X509::LOG, [columns=, ev=X509::log_x509]) -0.000000 | HookCallFunction Log::default_path_func(PacketFilter::LOG, , [ts=1405981560.501473, node=bro, filter=ip or not ip, init=T, success=T]) -0.000000 | HookCallFunction Log::write(PacketFilter::LOG, [ts=1405981560.501473, node=bro, filter=ip or not ip, init=T, success=T]) +0.000000 | HookCallFunction Log::default_path_func(PacketFilter::LOG, , [ts=1409853900.737227, node=bro, filter=ip or not ip, init=T, success=T]) +0.000000 | HookCallFunction Log::write(PacketFilter::LOG, [ts=1409853900.737227, node=bro, filter=ip or not ip, init=T, success=T]) 0.000000 | HookCallFunction Notice::want_pp() 0.000000 | HookCallFunction PacketFilter::build() 0.000000 | HookCallFunction PacketFilter::combine_filters(ip or not ip, and, ) diff --git a/testing/btest/Baseline/plugins.pktsrc/conn.log b/testing/btest/Baseline/plugins.pktsrc/conn.log index 550f520352..ab218f18fd 100644 --- a/testing/btest/Baseline/plugins.pktsrc/conn.log +++ b/testing/btest/Baseline/plugins.pktsrc/conn.log @@ -3,8 +3,8 @@ #empty_field (empty) #unset_field - #path conn -#open 2014-08-28-04-53-05 +#open 2014-09-04-18-06-05 #fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p proto service duration orig_bytes resp_bytes conn_state local_orig missed_bytes history orig_pkts orig_ip_bytes resp_pkts resp_ip_bytes tunnel_parents #types time string addr port addr port enum string interval count count string bool count string count count count count set[string] 1409193037.000000 CXWv6p3arKYeMETxOg 1.2.0.2 2527 1.2.0.3 6649 tcp - - - - S0 - 0 S 1 64 0 0 (empty) -#close 2014-08-28-04-53-05 +#close 2014-09-04-18-06-05 diff --git a/testing/btest/Baseline/plugins.writer/output b/testing/btest/Baseline/plugins.writer/output index f737e892a0..49c130d6e2 100644 --- a/testing/btest/Baseline/plugins.writer/output +++ b/testing/btest/Baseline/plugins.writer/output @@ -2,21 +2,21 @@ Demo::Foo - A Foo test logging writer (dynamic, version 1.0) [Writer] Foo (Log::WRITER_FOO) === -[packet_filter] 1406831942.605829|bro|ip or not ip|T|T [conn] 1340213005.165293|CXWv6p3arKYeMETxOg|10.0.0.55|53994|60.190.189.214|8124|tcp|-|4.314406|0|0|S0|-|0|S|5|320|0|0| -[tunnel] 1340213015.276495|-|10.0.0.55|0|60.190.189.214|8124|Tunnel::SOCKS|Tunnel::DISCOVER -[socks] 1340213015.276495|CjhGID4nQcgTWjvg4c|10.0.0.55|53994|60.190.189.214|8124|5|-|succeeded|-|www.osnews.com|80|192.168.0.31|-|2688 -[http] 1340213019.013158|CjhGID4nQcgTWjvg4c|10.0.0.55|53994|60.190.189.214|8124|1|GET|www.osnews.com|/images/printer2.gif|http://www.osnews.com/|Mozilla/5.0 (Macintosh; Intel Mac OS X 10.7; rv:10.0.2) Gecko/20100101 Firefox/10.0.2|0|0|304|Not Modified|-|-|-||-|-|-|-|-|-|- -[http] 1340213019.013426|CjhGID4nQcgTWjvg4c|10.0.0.55|53994|60.190.189.214|8124|2|GET|www.osnews.com|/img2/shorturl.jpg|http://www.osnews.com/|Mozilla/5.0 (Macintosh; Intel Mac OS X 10.7; rv:10.0.2) Gecko/20100101 Firefox/10.0.2|0|0|304|Not Modified|-|-|-||-|-|-|-|-|-|- -[http] 1340213019.580162|CjhGID4nQcgTWjvg4c|10.0.0.55|53994|60.190.189.214|8124|3|GET|www.osnews.com|/images/icons/9.gif|http://www.osnews.com/|Mozilla/5.0 (Macintosh; Intel Mac OS X 10.7; rv:10.0.2) Gecko/20100101 Firefox/10.0.2|0|0|304|Not Modified|-|-|-||-|-|-|-|-|-|- -[http] 1340213020.155861|CjhGID4nQcgTWjvg4c|10.0.0.55|53994|60.190.189.214|8124|4|GET|www.osnews.com|/images/icons/26.gif|http://www.osnews.com/|Mozilla/5.0 (Macintosh; Intel Mac OS X 10.7; rv:10.0.2) Gecko/20100101 Firefox/10.0.2|0|1368|200|OK|-|-|-||-|-|-|-|-|FBtZ7y1ppK8iIeY622|image/gif -[files] 1340213020.732581|FBtZ7y1ppK8iIeY622|60.190.189.214|10.0.0.55|CjhGID4nQcgTWjvg4c|HTTP|0||image/gif|-|0.000000|-|F|1368|1368|0|0|F|-|-|-|-|- -[http] 1340213020.732963|CjhGID4nQcgTWjvg4c|10.0.0.55|53994|60.190.189.214|8124|5|GET|www.osnews.com|/images/icons/17.gif|http://www.osnews.com/|Mozilla/5.0 (Macintosh; Intel Mac OS X 10.7; rv:10.0.2) Gecko/20100101 Firefox/10.0.2|0|0|304|Not Modified|-|-|-||-|-|-|-|-|-|- -[http] 1340213021.300269|CjhGID4nQcgTWjvg4c|10.0.0.55|53994|60.190.189.214|8124|6|GET|www.osnews.com|/images/left.gif|http://www.osnews.com/|Mozilla/5.0 (Macintosh; Intel Mac OS X 10.7; rv:10.0.2) Gecko/20100101 Firefox/10.0.2|0|0|304|Not Modified|-|-|-||-|-|-|-|-|-|- [conn] 1340213010.582723|CjhGID4nQcgTWjvg4c|10.0.0.55|53994|60.190.189.214|8124|tcp|http,socks|13.839419|3860|2934|SF|-|0|ShADadfF|23|5080|20|3986| -[http] 1340213021.861584|CjhGID4nQcgTWjvg4c|10.0.0.55|53994|60.190.189.214|8124|7|GET|www.osnews.com|/images/icons/32.gif|http://www.osnews.com/|Mozilla/5.0 (Macintosh; Intel Mac OS X 10.7; rv:10.0.2) Gecko/20100101 Firefox/10.0.2|0|0|304|Not Modified|-|-|-||-|-|-|-|-|-|- [conn] 1340213048.780152|CCvvfg3TEfuqmmG4bh|10.0.0.55|53994|60.190.189.214|8124|tcp|-|-|-|-|SH|-|0|F|1|52|0|0| [conn] 1340213097.272764|CsRx2w45OKnoww6xl4|10.0.0.55|53994|60.190.189.214|8124|tcp|-|-|-|-|SH|-|0|F|1|52|0|0| [conn] 1340213162.160367|CRJuHdVW0XPVINV8a|10.0.0.55|53994|60.190.189.214|8124|tcp|-|-|-|-|SH|-|0|F|1|52|0|0| [conn] 1340213226.561757|CPbrpk1qSsw6ESzHV4|10.0.0.55|53994|60.190.189.214|8124|tcp|-|-|-|-|SH|-|0|F|1|52|0|0| [conn] 1340213290.981995|C6pKV8GSxOnSLghOa|10.0.0.55|53994|60.190.189.214|8124|tcp|-|-|-|-|SH|-|0|F|1|52|0|0| +[files] 1340213020.732581|FBtZ7y1ppK8iIeY622|60.190.189.214|10.0.0.55|CjhGID4nQcgTWjvg4c|HTTP|0||image/gif|-|0.000000|-|F|1368|1368|0|0|F|-|-|-|-|- +[http] 1340213019.013158|CjhGID4nQcgTWjvg4c|10.0.0.55|53994|60.190.189.214|8124|1|GET|www.osnews.com|/images/printer2.gif|http://www.osnews.com/|Mozilla/5.0 (Macintosh; Intel Mac OS X 10.7; rv:10.0.2) Gecko/20100101 Firefox/10.0.2|0|0|304|Not Modified|-|-|-||-|-|-|-|-|-|- +[http] 1340213019.013426|CjhGID4nQcgTWjvg4c|10.0.0.55|53994|60.190.189.214|8124|2|GET|www.osnews.com|/img2/shorturl.jpg|http://www.osnews.com/|Mozilla/5.0 (Macintosh; Intel Mac OS X 10.7; rv:10.0.2) Gecko/20100101 Firefox/10.0.2|0|0|304|Not Modified|-|-|-||-|-|-|-|-|-|- +[http] 1340213019.580162|CjhGID4nQcgTWjvg4c|10.0.0.55|53994|60.190.189.214|8124|3|GET|www.osnews.com|/images/icons/9.gif|http://www.osnews.com/|Mozilla/5.0 (Macintosh; Intel Mac OS X 10.7; rv:10.0.2) Gecko/20100101 Firefox/10.0.2|0|0|304|Not Modified|-|-|-||-|-|-|-|-|-|- +[http] 1340213020.155861|CjhGID4nQcgTWjvg4c|10.0.0.55|53994|60.190.189.214|8124|4|GET|www.osnews.com|/images/icons/26.gif|http://www.osnews.com/|Mozilla/5.0 (Macintosh; Intel Mac OS X 10.7; rv:10.0.2) Gecko/20100101 Firefox/10.0.2|0|1368|200|OK|-|-|-||-|-|-|-|-|FBtZ7y1ppK8iIeY622|image/gif +[http] 1340213020.732963|CjhGID4nQcgTWjvg4c|10.0.0.55|53994|60.190.189.214|8124|5|GET|www.osnews.com|/images/icons/17.gif|http://www.osnews.com/|Mozilla/5.0 (Macintosh; Intel Mac OS X 10.7; rv:10.0.2) Gecko/20100101 Firefox/10.0.2|0|0|304|Not Modified|-|-|-||-|-|-|-|-|-|- +[http] 1340213021.300269|CjhGID4nQcgTWjvg4c|10.0.0.55|53994|60.190.189.214|8124|6|GET|www.osnews.com|/images/left.gif|http://www.osnews.com/|Mozilla/5.0 (Macintosh; Intel Mac OS X 10.7; rv:10.0.2) Gecko/20100101 Firefox/10.0.2|0|0|304|Not Modified|-|-|-||-|-|-|-|-|-|- +[http] 1340213021.861584|CjhGID4nQcgTWjvg4c|10.0.0.55|53994|60.190.189.214|8124|7|GET|www.osnews.com|/images/icons/32.gif|http://www.osnews.com/|Mozilla/5.0 (Macintosh; Intel Mac OS X 10.7; rv:10.0.2) Gecko/20100101 Firefox/10.0.2|0|0|304|Not Modified|-|-|-||-|-|-|-|-|-|- +[packet_filter] 1409859343.786281|bro|ip or not ip|T|T +[socks] 1340213015.276495|CjhGID4nQcgTWjvg4c|10.0.0.55|53994|60.190.189.214|8124|5|-|succeeded|-|www.osnews.com|80|192.168.0.31|-|2688 +[tunnel] 1340213015.276495|-|10.0.0.55|0|60.190.189.214|8124|Tunnel::SOCKS|Tunnel::DISCOVER diff --git a/testing/btest/plugins/api-version-mismatch.sh b/testing/btest/plugins/api-version-mismatch.sh index a75ff05655..f8d88b4fc4 100644 --- a/testing/btest/plugins/api-version-mismatch.sh +++ b/testing/btest/plugins/api-version-mismatch.sh @@ -1,6 +1,6 @@ # @TEST-EXEC: ${DIST}/aux/bro-aux/plugin-support/init-plugin Demo Foo # @TEST-EXEC: bash %INPUT -# @TEST-EXEC: make BRO=${DIST} +# @TEST-EXEC: ./configure --bro-dist=${DIST} && make # @TEST-EXEC-FAIL: BRO_PLUGIN_PATH=`pwd` bro -NN Demo::Foo >>output 2>&1 # @TEST-EXEC: btest-diff output diff --git a/testing/btest/plugins/bifs-and-scripts-install.sh b/testing/btest/plugins/bifs-and-scripts-install.sh index 4f0174a69f..58d0987f5e 100644 --- a/testing/btest/plugins/bifs-and-scripts-install.sh +++ b/testing/btest/plugins/bifs-and-scripts-install.sh @@ -1,6 +1,7 @@ # @TEST-EXEC: ${DIST}/aux/bro-aux/plugin-support/init-plugin Demo Foo # @TEST-EXEC: bash %INPUT -# @TEST-EXEC: BRO_PLUGIN_INSTALL=`pwd`/test-install make BRO=${DIST} +# @TEST-EXEC: ./configure --bro-dist=${DIST} +# @TEST-EXEC: BRO_PLUGIN_INSTALL=`pwd`/test-install make # @TEST-EXEC: make install # @TEST-EXEC: BRO_PLUGIN_PATH=`pwd`/test-install bro -NN Demo::Foo >>output # @TEST-EXEC: BRO_PLUGIN_PATH=`pwd` bro demo/foo -r $TRACES/empty.trace >>output diff --git a/testing/btest/plugins/bifs-and-scripts.sh b/testing/btest/plugins/bifs-and-scripts.sh index 9975da21c6..7b4c45cd4e 100644 --- a/testing/btest/plugins/bifs-and-scripts.sh +++ b/testing/btest/plugins/bifs-and-scripts.sh @@ -1,6 +1,6 @@ # @TEST-EXEC: ${DIST}/aux/bro-aux/plugin-support/init-plugin Demo Foo # @TEST-EXEC: bash %INPUT -# @TEST-EXEC: make BRO=${DIST} +# @TEST-EXEC: ./configure --bro-dist=${DIST} && make # @TEST-EXEC: BRO_PLUGIN_PATH=`pwd` bro -NN Demo::Foo >>output # @TEST-EXEC: echo === >>output diff --git a/testing/btest/plugins/file.bro b/testing/btest/plugins/file.bro index fb882d1d7e..7d25cab538 100644 --- a/testing/btest/plugins/file.bro +++ b/testing/btest/plugins/file.bro @@ -1,6 +1,6 @@ # @TEST-EXEC: ${DIST}/aux/bro-aux/plugin-support/init-plugin Demo Foo # @TEST-EXEC: cp -r %DIR/file-plugin/* . -# @TEST-EXEC: make BRO=${DIST} +# @TEST-EXEC: ./configure --bro-dist=${DIST} && make # @TEST-EXEC: BRO_PLUGIN_PATH=`pwd` bro -NN Demo::Foo >>output # @TEST-EXEC: echo === >>output # @TEST-EXEC: BRO_PLUGIN_PATH=`pwd` bro -r $TRACES/ftp/retr.trace %INPUT >>output diff --git a/testing/btest/plugins/hooks.bro b/testing/btest/plugins/hooks.bro index 1a4f35fc55..786e6ccc88 100644 --- a/testing/btest/plugins/hooks.bro +++ b/testing/btest/plugins/hooks.bro @@ -1,6 +1,6 @@ # @TEST-EXEC: ${DIST}/aux/bro-aux/plugin-support/init-plugin Demo Hooks # @TEST-EXEC: cp -r %DIR/hooks-plugin/* . -# @TEST-EXEC: make BRO=${DIST} +# @TEST-EXEC: ./configure --bro-dist=${DIST} && make # @TEST-EXEC: BRO_PLUGIN_PATH=`pwd` bro -r $TRACES/http/get.trace %INPUT 2>&1 | $SCRIPTS/diff-remove-abspath | sort | uniq >output # @TEST-EXEC: btest-diff output diff --git a/testing/btest/plugins/init-plugin.bro b/testing/btest/plugins/init-plugin.bro index 1ad5be6aea..2fffa88f2c 100644 --- a/testing/btest/plugins/init-plugin.bro +++ b/testing/btest/plugins/init-plugin.bro @@ -1,5 +1,5 @@ # @TEST-EXEC: ${DIST}/aux/bro-aux/plugin-support/init-plugin Demo Foo -# @TEST-EXEC: make BRO=${DIST} +# @TEST-EXEC: ./configure --bro-dist=${DIST} && make # @TEST-EXEC: BRO_PLUGIN_PATH=`pwd` bro -NN Demo::Foo >>output # @TEST-EXEC: echo === >>output # @TEST-EXEC: BRO_PLUGIN_PATH=`pwd` bro -r $TRACES/port4242.trace >>output diff --git a/testing/btest/plugins/pktsrc.bro b/testing/btest/plugins/pktsrc.bro index 2bd9be7bb7..39d2fa9aff 100644 --- a/testing/btest/plugins/pktsrc.bro +++ b/testing/btest/plugins/pktsrc.bro @@ -4,5 +4,5 @@ # @TEST-EXEC: BRO_PLUGIN_PATH=`pwd` bro -NN Demo::Foo >>output # @TEST-EXEC: echo === >>output # @TEST-EXEC: BRO_PLUGIN_PATH=`pwd` bro -r foo:XXX %INPUT FilteredTraceDetection::enable=F >>output -# @TEST-EXEC: TEST_DIFF_CANONIFIER= btest-diff conn.log +# @TEST-EXEC: btest-diff conn.log diff --git a/testing/btest/plugins/protocol-plugin/CMakeLists.txt b/testing/btest/plugins/protocol-plugin/CMakeLists.txt index a032edbd89..4bc8460c06 100644 --- a/testing/btest/plugins/protocol-plugin/CMakeLists.txt +++ b/testing/btest/plugins/protocol-plugin/CMakeLists.txt @@ -15,6 +15,5 @@ bro_plugin_begin(Demo Foo) bro_plugin_cc(src/Plugin.cc) bro_plugin_cc(src/Foo.cc) bro_plugin_bif(src/events.bif) -bro_plugin_bif(src/functions.bif) bro_plugin_pac(src/foo.pac src/foo-protocol.pac src/foo-analyzer.pac) bro_plugin_end() diff --git a/testing/btest/plugins/protocol.bro b/testing/btest/plugins/protocol.bro index dadbbb7717..671edb6cf1 100644 --- a/testing/btest/plugins/protocol.bro +++ b/testing/btest/plugins/protocol.bro @@ -1,6 +1,6 @@ # @TEST-EXEC: ${DIST}/aux/bro-aux/plugin-support/init-plugin Demo Foo # @TEST-EXEC: cp -r %DIR/protocol-plugin/* . -# @TEST-EXEC: make BRO=${DIST} +# @TEST-EXEC: ./configure --bro-dist=${DIST} && make # @TEST-EXEC: BRO_PLUGIN_PATH=`pwd` bro -NN Demo::Foo >>output # @TEST-EXEC: echo === >>output # @TEST-EXEC: BRO_PLUGIN_PATH=`pwd` bro -r $TRACES/port4242.trace %INPUT >>output diff --git a/testing/btest/plugins/reader.bro b/testing/btest/plugins/reader.bro index cecb5306da..5065678c2e 100644 --- a/testing/btest/plugins/reader.bro +++ b/testing/btest/plugins/reader.bro @@ -1,6 +1,6 @@ # @TEST-EXEC: ${DIST}/aux/bro-aux/plugin-support/init-plugin Demo Foo # @TEST-EXEC: cp -r %DIR/reader-plugin/* . -# @TEST-EXEC: make BRO=${DIST} +# @TEST-EXEC: ./configure --bro-dist=${DIST} && make # @TEST-EXEC: BRO_PLUGIN_PATH=`pwd` bro -NN Demo::Foo >>output # @TEST-EXEC: echo === >>output # @TEST-EXEC: BRO_PLUGIN_PATH=`pwd` btest-bg-run bro bro %INPUT diff --git a/testing/btest/plugins/writer.bro b/testing/btest/plugins/writer.bro index 49fbbb9395..f2e74ad667 100644 --- a/testing/btest/plugins/writer.bro +++ b/testing/btest/plugins/writer.bro @@ -1,8 +1,8 @@ # @TEST-EXEC: ${DIST}/aux/bro-aux/plugin-support/init-plugin Demo Foo # @TEST-EXEC: cp -r %DIR/writer-plugin/* . -# @TEST-EXEC: make BRO=${DIST} +# @TEST-EXEC: ./configure --bro-dist=${DIST} && make # @TEST-EXEC: BRO_PLUGIN_PATH=`pwd` bro -NN Demo::Foo >>output # @TEST-EXEC: echo === >>output -# @TEST-EXEC: BRO_PLUGIN_PATH=`pwd` bro -r $TRACES/socks.trace Log::default_writer=Log::WRITER_FOO %INPUT >>output -# @TEST-EXEC: TEST_DIFF_CANONIFIER= btest-diff output +# @TEST-EXEC: BRO_PLUGIN_PATH=`pwd` bro -r $TRACES/socks.trace Log::default_writer=Log::WRITER_FOO %INPUT | sort >>output +# @TEST-EXEC: TEST_DIFF_CANONIFIER=diff-remove-timestamps btest-diff output From 6e33c92cf00fe1fb8419ccd802c39390b318ca40 Mon Sep 17 00:00:00 2001 From: Robin Sommer Date: Thu, 4 Sep 2014 20:30:28 -0700 Subject: [PATCH 19/25] Adding test for dynamic packet dumper plugin. --- .../btest/Baseline/plugins.pktdumper/output | 12 ++++++ .../plugins/pktdumper-plugin/.btest-ignore | 0 .../plugins/pktdumper-plugin/CMakeLists.txt | 17 ++++++++ .../btest/plugins/pktdumper-plugin/src/Foo.cc | 40 +++++++++++++++++++ .../btest/plugins/pktdumper-plugin/src/Foo.h | 30 ++++++++++++++ .../plugins/pktdumper-plugin/src/Plugin.cc | 20 ++++++++++ testing/btest/plugins/pktdumper.bro | 8 ++++ 7 files changed, 127 insertions(+) create mode 100644 testing/btest/Baseline/plugins.pktdumper/output create mode 100644 testing/btest/plugins/pktdumper-plugin/.btest-ignore create mode 100644 testing/btest/plugins/pktdumper-plugin/CMakeLists.txt create mode 100644 testing/btest/plugins/pktdumper-plugin/src/Foo.cc create mode 100644 testing/btest/plugins/pktdumper-plugin/src/Foo.h create mode 100644 testing/btest/plugins/pktdumper-plugin/src/Plugin.cc create mode 100644 testing/btest/plugins/pktdumper.bro diff --git a/testing/btest/Baseline/plugins.pktdumper/output b/testing/btest/Baseline/plugins.pktdumper/output new file mode 100644 index 0000000000..05ffec25cb --- /dev/null +++ b/testing/btest/Baseline/plugins.pktdumper/output @@ -0,0 +1,12 @@ +Demo::Foo - A Foo packet dumper (dynamic, version 1.0) + [Packet Dumper] FooPktDumper (dumper prefix: foo) + +=== +Dumping to XXX: 1373858797.646968 len 94 +Dumping to XXX: 1373858797.646998 len 94 +Dumping to XXX: 1373858797.647041 len 86 +Dumping to XXX: 1373858797.647147 len 98 +Dumping to XXX: 1373858797.647186 len 86 +Dumping to XXX: 1373858797.647250 len 86 +Dumping to XXX: 1373858797.647317 len 86 +Dumping to XXX: 1373858797.647350 len 86 diff --git a/testing/btest/plugins/pktdumper-plugin/.btest-ignore b/testing/btest/plugins/pktdumper-plugin/.btest-ignore new file mode 100644 index 0000000000..e69de29bb2 diff --git a/testing/btest/plugins/pktdumper-plugin/CMakeLists.txt b/testing/btest/plugins/pktdumper-plugin/CMakeLists.txt new file mode 100644 index 0000000000..2234907ad2 --- /dev/null +++ b/testing/btest/plugins/pktdumper-plugin/CMakeLists.txt @@ -0,0 +1,17 @@ + +project(Bro-Plugin-Demo-Foo) + +cmake_minimum_required(VERSION 2.6.3) + +if ( NOT BRO_DIST ) + message(FATAL_ERROR "BRO_DIST not set") +endif () + +set(CMAKE_MODULE_PATH ${BRO_DIST}/cmake) + +include(BroPlugin) + +bro_plugin_begin(Demo Foo) +bro_plugin_cc(src/Plugin.cc) +bro_plugin_cc(src/Foo.cc) +bro_plugin_end() diff --git a/testing/btest/plugins/pktdumper-plugin/src/Foo.cc b/testing/btest/plugins/pktdumper-plugin/src/Foo.cc new file mode 100644 index 0000000000..fdd364b034 --- /dev/null +++ b/testing/btest/plugins/pktdumper-plugin/src/Foo.cc @@ -0,0 +1,40 @@ + +#include +#include + +#include "Foo.h" + +using namespace plugin::Demo_Foo; + +Foo::Foo(const std::string& path, bool is_live) + { + props.path = path; + } + +Foo::~Foo() + { + } + +void Foo::Open() + { + props.open_time = network_time; + props.hdr_size = 0; + Opened(props); + } + +void Foo::Close() + { + Closed(); + } + +bool Foo::Dump(const Packet* pkt) + { + double t = double(pkt->hdr->ts.tv_sec) + double(pkt->hdr->ts.tv_usec) / 1e6; + fprintf(stdout, "Dumping to %s: %.6f len %u\n", props.path.c_str(), t, (unsigned int)pkt->hdr->len); + return true; + } + +iosource::PktDumper* Foo::Instantiate(const std::string& path, bool append) + { + return new Foo(path, append); + } diff --git a/testing/btest/plugins/pktdumper-plugin/src/Foo.h b/testing/btest/plugins/pktdumper-plugin/src/Foo.h new file mode 100644 index 0000000000..b8c8291728 --- /dev/null +++ b/testing/btest/plugins/pktdumper-plugin/src/Foo.h @@ -0,0 +1,30 @@ + +#ifndef BRO_PLUGIN_DEMO_FOO_H +#define BRO_PLUGIN_DEMO_FOO_H + +#include +#include + +namespace plugin { +namespace Demo_Foo { + +class Foo : public iosource::PktDumper { +public: + Foo(const std::string& path, bool is_live); + virtual ~Foo(); + + static PktDumper* Instantiate(const std::string& path, bool append); + +protected: + virtual void Open(); + virtual void Close(); + virtual bool Dump(const Packet* pkt); + +private: + Properties props; +}; + +} +} + +#endif diff --git a/testing/btest/plugins/pktdumper-plugin/src/Plugin.cc b/testing/btest/plugins/pktdumper-plugin/src/Plugin.cc new file mode 100644 index 0000000000..81ef8c79f4 --- /dev/null +++ b/testing/btest/plugins/pktdumper-plugin/src/Plugin.cc @@ -0,0 +1,20 @@ + +#include "Plugin.h" + +#include "Foo.h" + +namespace plugin { namespace Demo_Foo { Plugin plugin; } } + +using namespace plugin::Demo_Foo; + +plugin::Configuration Plugin::Configure() + { + AddComponent(new ::iosource::PktDumperComponent("FooPktDumper", "foo", ::plugin::Demo_Foo::Foo::Instantiate)); + + plugin::Configuration config; + config.name = "Demo::Foo"; + config.description = "A Foo packet dumper"; + config.version.major = 1; + config.version.minor = 0; + return config; + } diff --git a/testing/btest/plugins/pktdumper.bro b/testing/btest/plugins/pktdumper.bro new file mode 100644 index 0000000000..61540897d8 --- /dev/null +++ b/testing/btest/plugins/pktdumper.bro @@ -0,0 +1,8 @@ +# @TEST-EXEC: ${DIST}/aux/bro-aux/plugin-support/init-plugin Demo Foo +# @TEST-EXEC: cp -r %DIR/pktdumper-plugin/* . +# @TEST-EXEC: ./configure --bro-dist=${DIST} && make +# @TEST-EXEC: BRO_PLUGIN_PATH=`pwd` bro -NN Demo::Foo >>output +# @TEST-EXEC: echo === >>output +# @TEST-EXEC: BRO_PLUGIN_PATH=`pwd` bro -r $TRACES/port4242.trace -w foo:XXX %INPUT FilteredTraceDetection::enable=F >>output +# @TEST-EXEC: btest-diff output + From 042afd2feb530e0cf77cbd76dfd5fbe5c9ba4f88 Mon Sep 17 00:00:00 2001 From: Robin Sommer Date: Thu, 4 Sep 2014 20:55:20 -0700 Subject: [PATCH 20/25] Fixing remaining tests. --- aux/bro-aux | 2 +- cmake | 2 +- src/iosource/Component.cc | 2 +- src/iosource/IOSource.h | 4 ++++ src/iosource/PktSrc.h | 4 ---- .../plugins.bifs-and-scripts-install/output | 2 +- .../Baseline/plugins.bifs-and-scripts/output | 2 +- testing/btest/plugins/bifs-and-scripts-install.sh | 15 +++++++-------- testing/btest/plugins/bifs-and-scripts.sh | 11 +++++------ 9 files changed, 21 insertions(+), 23 deletions(-) diff --git a/aux/bro-aux b/aux/bro-aux index 181f084432..4bb294d4d1 160000 --- a/aux/bro-aux +++ b/aux/bro-aux @@ -1 +1 @@ -Subproject commit 181f084432e277f899140647d9b788059b3cccb1 +Subproject commit 4bb294d4d15b14a9e49f875b459fdef8286b5957 diff --git a/cmake b/cmake index aa15263ae3..4f830a0fbf 160000 --- a/cmake +++ b/cmake @@ -1 +1 @@ -Subproject commit aa15263ae39667e5e9bd73690b05aa4af9147ca3 +Subproject commit 4f830a0fbf078d6b64a9a2264d2d06d89e8fe619 diff --git a/src/iosource/Component.cc b/src/iosource/Component.cc index 20232161cd..a9cfa37d34 100644 --- a/src/iosource/Component.cc +++ b/src/iosource/Component.cc @@ -151,7 +151,7 @@ void PktDumperComponent::DoDescribe(ODesc* d) const if ( prefs.size() ) prefs += ", "; - prefs += *i; + prefs += '"' + *i + '"'; } d->Add("dumper prefix"); diff --git a/src/iosource/IOSource.h b/src/iosource/IOSource.h index 9083e8f4f1..0e7087a2dd 100644 --- a/src/iosource/IOSource.h +++ b/src/iosource/IOSource.h @@ -3,6 +3,10 @@ #ifndef IOSOURCE_IOSOURCE_H #define IOSOURCE_IOSOURCE_H +extern "C" { +#include +} + #include #include "Timer.h" diff --git a/src/iosource/PktSrc.h b/src/iosource/PktSrc.h index c616a2d5b4..75fd2633d0 100644 --- a/src/iosource/PktSrc.h +++ b/src/iosource/PktSrc.h @@ -3,10 +3,6 @@ #ifndef IOSOURCE_PKTSRC_PKTSRC_H #define IOSOURCE_PKTSRC_PKTSRC_H -extern "C" { -#include -} - #include "IOSource.h" #include "BPF_Program.h" #include "Dict.h" diff --git a/testing/btest/Baseline/plugins.bifs-and-scripts-install/output b/testing/btest/Baseline/plugins.bifs-and-scripts-install/output index a4187d0f7c..f03cfddc81 100644 --- a/testing/btest/Baseline/plugins.bifs-and-scripts-install/output +++ b/testing/btest/Baseline/plugins.bifs-and-scripts-install/output @@ -1,6 +1,6 @@ Demo::Foo - (dynamic, version 1.0) - [Event] plugin_event [Function] hello_plugin_world + [Event] plugin_event plugin: automatically loaded at startup calling bif, Hello from the plugin! diff --git a/testing/btest/Baseline/plugins.bifs-and-scripts/output b/testing/btest/Baseline/plugins.bifs-and-scripts/output index a082b3d690..47dd6ed430 100644 --- a/testing/btest/Baseline/plugins.bifs-and-scripts/output +++ b/testing/btest/Baseline/plugins.bifs-and-scripts/output @@ -1,6 +1,6 @@ Demo::Foo - (dynamic, version 1.0) - [Event] plugin_event [Function] hello_plugin_world + [Event] plugin_event === plugin: automatically loaded at startup diff --git a/testing/btest/plugins/bifs-and-scripts-install.sh b/testing/btest/plugins/bifs-and-scripts-install.sh index 58d0987f5e..158f5fc01f 100644 --- a/testing/btest/plugins/bifs-and-scripts-install.sh +++ b/testing/btest/plugins/bifs-and-scripts-install.sh @@ -1,12 +1,14 @@ # @TEST-EXEC: ${DIST}/aux/bro-aux/plugin-support/init-plugin Demo Foo # @TEST-EXEC: bash %INPUT -# @TEST-EXEC: ./configure --bro-dist=${DIST} -# @TEST-EXEC: BRO_PLUGIN_INSTALL=`pwd`/test-install make +# @TEST-EXEC: ./configure --bro-dist=${DIST} --install-root=`pwd`/test-install +# @TEST-EXEC: make # @TEST-EXEC: make install # @TEST-EXEC: BRO_PLUGIN_PATH=`pwd`/test-install bro -NN Demo::Foo >>output # @TEST-EXEC: BRO_PLUGIN_PATH=`pwd` bro demo/foo -r $TRACES/empty.trace >>output # @TEST-EXEC: TEST_DIFF_CANONIFIER= btest-diff output +mkdir -p scripts/demo/foo/base/ + cat >scripts/__load__.bro <scripts/demo/foo/base/at-startup.bro <src/functions.bif <src/foo.bif <activate.bro <src/events.bif <scripts/__load__.bro <scripts/demo/foo/base/at-startup.bro <src/functions.bif <src/foo.bif <activate.bro <src/events.bif < Date: Thu, 4 Sep 2014 22:43:25 -0700 Subject: [PATCH 21/25] fix more http links. This does not break the layout, thus these are not really important enough for the .1. --- NEWS | 2 +- doc/install/install.rst | 2 +- doc/quickstart/index.rst | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/NEWS b/NEWS index f06115d4ea..6da13833c3 100644 --- a/NEWS +++ b/NEWS @@ -22,7 +22,7 @@ New Functionality plugin can furthermore hook into Bro's processing a number of places to add custom logic. - See http://www.bro.org/sphinx-git/devel/plugins.html for more + See https://www.bro.org/sphinx-git/devel/plugins.html for more information on writing plugins. Changed Functionality diff --git a/doc/install/install.rst b/doc/install/install.rst index 9a258773ce..0052acafb0 100644 --- a/doc/install/install.rst +++ b/doc/install/install.rst @@ -180,7 +180,7 @@ automatically. Finally, use ``make install-aux`` to install some of the other programs that are in the ``aux/bro-aux`` directory. OpenBSD users, please see our `FAQ -`_ if you are having +`_ if you are having problems installing Bro. Finally, if you want to build the Bro documentation (not required, because diff --git a/doc/quickstart/index.rst b/doc/quickstart/index.rst index 173373c769..bb642ee75a 100644 --- a/doc/quickstart/index.rst +++ b/doc/quickstart/index.rst @@ -1,5 +1,5 @@ -.. _FAQ: http://www.bro.org/documentation/faq.html +.. _FAQ: //www.bro.org/documentation/faq.html .. _quickstart: From 4a66a8e341e93f3859d48282644c3fd9cdcce8da Mon Sep 17 00:00:00 2001 From: Robin Sommer Date: Fri, 5 Sep 2014 18:20:23 -0700 Subject: [PATCH 22/25] Bugfixes and test updates. --- TODO.iosources | 4 - src/DNS_Mgr.cc | 2 +- src/DNS_Mgr.h | 2 +- src/PktSrc.cc | 709 ------------------ src/RemoteSerializer.cc | 2 +- src/RemoteSerializer.h | 2 +- src/bro.bif | 2 +- src/iosource/Component.cc | 1 + src/iosource/Component.h | 37 +- src/iosource/IOSource.h | 110 ++- src/iosource/Manager.cc | 3 +- src/iosource/Manager.h | 85 ++- src/iosource/PktDumper.h | 3 + src/iosource/PktSrc.cc | 4 +- src/iosource/PktSrc.h | 3 + src/iosource/pcap/Dumper.h | 1 + src/iosource/pcap/Source.cc | 1 + src/main.cc | 7 +- .../canonified_loaded_scripts.log | 5 +- 19 files changed, 212 insertions(+), 771 deletions(-) delete mode 100644 src/PktSrc.cc diff --git a/TODO.iosources b/TODO.iosources index ee19dea169..e69de29bb2 100644 --- a/TODO.iosources +++ b/TODO.iosources @@ -1,4 +0,0 @@ -- Tests - - pktsrc plugin - - pktdump plugin - diff --git a/src/DNS_Mgr.cc b/src/DNS_Mgr.cc index 69f54b2bbc..7f37ee8c5f 100644 --- a/src/DNS_Mgr.cc +++ b/src/DNS_Mgr.cc @@ -405,7 +405,7 @@ DNS_Mgr::~DNS_Mgr() delete [] dir; } -void DNS_Mgr::Init() +void DNS_Mgr::InitPostScript() { if ( did_init ) return; diff --git a/src/DNS_Mgr.h b/src/DNS_Mgr.h index 5978f3a597..b8b0fc7e35 100644 --- a/src/DNS_Mgr.h +++ b/src/DNS_Mgr.h @@ -45,7 +45,7 @@ public: DNS_Mgr(DNS_MgrMode mode); virtual ~DNS_Mgr(); - void Init(); + void InitPostScript(); void Flush(); // Looks up the address or addresses of the given host, and returns diff --git a/src/PktSrc.cc b/src/PktSrc.cc deleted file mode 100644 index 7a0ed4fa0b..0000000000 --- a/src/PktSrc.cc +++ /dev/null @@ -1,709 +0,0 @@ -// See the file "COPYING" in the main distribution directory for copyright. - -#include -#include - -#include "config.h" - -#include "util.h" -#include "PktSrc.h" -#include "Hash.h" -#include "Net.h" -#include "Sessions.h" - - -// ### This needs auto-confing. -#ifdef HAVE_PCAP_INT_H -#include -#endif - -PktSrc::PktSrc() - { - interface = readfile = 0; - data = last_data = 0; - memset(&hdr, 0, sizeof(hdr)); - hdr_size = 0; - datalink = 0; - netmask = 0xffffff00; - pd = 0; - idle = false; - - next_sync_point = 0; - first_timestamp = current_timestamp = next_timestamp = 0.0; - first_wallclock = current_wallclock = 0; - - stats.received = stats.dropped = stats.link = 0; - } - -PktSrc::~PktSrc() - { - Close(); - - loop_over_list(program_list, i) - delete program_list[i]; - - BPF_Program* code; - IterCookie* cookie = filters.InitForIteration(); - while ( (code = filters.NextEntry(cookie)) ) - delete code; - - delete [] interface; - delete [] readfile; - } - -void PktSrc::GetFds(int* read, int* write, int* except) - { - if ( pseudo_realtime ) - { - // Select would give erroneous results. But we simulate it - // by setting idle accordingly. - idle = CheckPseudoTime() == 0; - return; - } - - if ( selectable_fd >= 0 ) - *read = selectable_fd; - } - -int PktSrc::ExtractNextPacket() - { - // Don't return any packets if processing is suspended (except for the - // very first packet which we need to set up times). - if ( net_is_processing_suspended() && first_timestamp ) - { - idle = true; - return 0; - } - - data = last_data = pcap_next(pd, &hdr); - - if ( data && (hdr.len == 0 || hdr.caplen == 0) ) - { - sessions->Weird("empty_pcap_header", &hdr, data); - return 0; - } - - if ( data ) - next_timestamp = hdr.ts.tv_sec + double(hdr.ts.tv_usec) / 1e6; - - if ( pseudo_realtime ) - current_wallclock = current_time(true); - - if ( ! first_timestamp ) - first_timestamp = next_timestamp; - - idle = (data == 0); - - if ( data ) - ++stats.received; - - // Source has gone dry. If it's a network interface, this just means - // it's timed out. If it's a file, though, then the file has been - // exhausted. - if ( ! data && ! IsLive() ) - { - closed = true; - - if ( pseudo_realtime && using_communication ) - { - if ( remote_trace_sync_interval ) - remote_serializer->SendFinalSyncPoint(); - else - remote_serializer->Terminate(); - } - } - - return data != 0; - } - -double PktSrc::NextTimestamp(double* local_network_time) - { - if ( ! data && ! ExtractNextPacket() ) - return -1.0; - - if ( pseudo_realtime ) - { - // Delay packet if necessary. - double packet_time = CheckPseudoTime(); - if ( packet_time ) - return packet_time; - - idle = true; - return -1.0; - } - - return next_timestamp; - } - -void PktSrc::ContinueAfterSuspend() - { - current_wallclock = current_time(true); - } - -double PktSrc::CurrentPacketWallClock() - { - // We stop time when we are suspended. - if ( net_is_processing_suspended() ) - current_wallclock = current_time(true); - - return current_wallclock; - } - -double PktSrc::CheckPseudoTime() - { - if ( ! data && ! ExtractNextPacket() ) - return 0; - - if ( ! current_timestamp ) - return bro_start_time; - - if ( remote_trace_sync_interval ) - { - if ( next_sync_point == 0 || next_timestamp >= next_sync_point ) - { - int n = remote_serializer->SendSyncPoint(); - next_sync_point = first_timestamp + - n * remote_trace_sync_interval; - remote_serializer->Log(RemoteSerializer::LogInfo, - fmt("stopping at packet %.6f, next sync-point at %.6f", - current_timestamp, next_sync_point)); - - return 0; - } - } - - double pseudo_time = next_timestamp - first_timestamp; - double ct = (current_time(true) - first_wallclock) * pseudo_realtime; - - return pseudo_time <= ct ? bro_start_time + pseudo_time : 0; - } - -void PktSrc::Process() - { - if ( ! data && ! ExtractNextPacket() ) - return; - - current_timestamp = next_timestamp; - - int pkt_hdr_size = hdr_size; - - // Unfortunately some packets on the link might have MPLS labels - // while others don't. That means we need to ask the link-layer if - // labels are in place. - bool have_mpls = false; - - int protocol = 0; - - switch ( datalink ) { - case DLT_NULL: - { - protocol = (data[3] << 24) + (data[2] << 16) + (data[1] << 8) + data[0]; - - // From the Wireshark Wiki: "AF_INET6, unfortunately, has - // different values in {NetBSD,OpenBSD,BSD/OS}, - // {FreeBSD,DragonFlyBSD}, and {Darwin/Mac OS X}, so an IPv6 - // packet might have a link-layer header with 24, 28, or 30 - // as the AF_ value." As we may be reading traces captured on - // platforms other than what we're running on, we accept them - // all here. - if ( protocol != AF_INET - && protocol != AF_INET6 - && protocol != 24 - && protocol != 28 - && protocol != 30 ) - { - sessions->Weird("non_ip_packet_in_null_transport", &hdr, data); - data = 0; - return; - } - - break; - } - - case DLT_EN10MB: - { - // Get protocol being carried from the ethernet frame. - protocol = (data[12] << 8) + data[13]; - - switch ( protocol ) - { - // MPLS carried over the ethernet frame. - case 0x8847: - // Remove the data link layer and denote a - // header size of zero before the IP header. - have_mpls = true; - data += get_link_header_size(datalink); - pkt_hdr_size = 0; - break; - - // VLAN carried over the ethernet frame. - case 0x8100: - data += get_link_header_size(datalink); - - // Check for MPLS in VLAN. - if ( ((data[2] << 8) + data[3]) == 0x8847 ) - have_mpls = true; - - data += 4; // Skip the vlan header - pkt_hdr_size = 0; - - // Check for 802.1ah (Q-in-Q) containing IP. - // Only do a second layer of vlan tag - // stripping because there is no - // specification that allows for deeper - // nesting. - if ( ((data[2] << 8) + data[3]) == 0x0800 ) - data += 4; - - break; - - // PPPoE carried over the ethernet frame. - case 0x8864: - data += get_link_header_size(datalink); - protocol = (data[6] << 8) + data[7]; - data += 8; // Skip the PPPoE session and PPP header - pkt_hdr_size = 0; - - if ( protocol != 0x0021 && protocol != 0x0057 ) - { - // Neither IPv4 nor IPv6. - sessions->Weird("non_ip_packet_in_pppoe_encapsulation", &hdr, data); - data = 0; - return; - } - break; - } - - break; - } - - case DLT_PPP_SERIAL: - { - // Get PPP protocol. - protocol = (data[2] << 8) + data[3]; - - if ( protocol == 0x0281 ) - { - // MPLS Unicast. Remove the data link layer and - // denote a header size of zero before the IP header. - have_mpls = true; - data += get_link_header_size(datalink); - pkt_hdr_size = 0; - } - - else if ( protocol != 0x0021 && protocol != 0x0057 ) - { - // Neither IPv4 nor IPv6. - sessions->Weird("non_ip_packet_in_ppp_encapsulation", &hdr, data); - data = 0; - return; - } - break; - } - } - - if ( have_mpls ) - { - // Skip the MPLS label stack. - bool end_of_stack = false; - - while ( ! end_of_stack ) - { - end_of_stack = *(data + 2) & 0x01; - data += 4; - } - } - - if ( pseudo_realtime ) - { - current_pseudo = CheckPseudoTime(); - net_packet_dispatch(current_pseudo, &hdr, data, pkt_hdr_size, this); - if ( ! first_wallclock ) - first_wallclock = current_time(true); - } - - else - net_packet_dispatch(current_timestamp, &hdr, data, pkt_hdr_size, this); - - data = 0; - } - -bool PktSrc::GetCurrentPacket(const struct pcap_pkthdr** arg_hdr, - const u_char** arg_pkt) - { - if ( ! last_data ) - return false; - - *arg_hdr = &hdr; - *arg_pkt = last_data; - return true; - } - -int PktSrc::PrecompileFilter(int index, const char* filter) - { - // Compile filter. - BPF_Program* code = new BPF_Program(); - - if ( ! code->Compile(pd, filter, netmask, errbuf, sizeof(errbuf)) ) - { - delete code; - return 0; - } - - // Store it in hash. - HashKey* hash = new HashKey(HashKey(bro_int_t(index))); - BPF_Program* oldcode = filters.Lookup(hash); - if ( oldcode ) - delete oldcode; - - filters.Insert(hash, code); - delete hash; - - return 1; - } - -int PktSrc::SetFilter(int index) - { - HashKey* hash = new HashKey(HashKey(bro_int_t(index))); - BPF_Program* code = filters.Lookup(hash); - delete hash; - - if ( ! code ) - { - safe_snprintf(errbuf, sizeof(errbuf), - "No precompiled pcap filter for index %d", - index); - return 0; - } - - if ( pcap_setfilter(pd, code->GetProgram()) < 0 ) - { - safe_snprintf(errbuf, sizeof(errbuf), - "pcap_setfilter(%d): %s", - index, pcap_geterr(pd)); - return 0; - } - -#ifndef HAVE_LINUX - // Linux doesn't clear counters when resetting filter. - stats.received = stats.dropped = stats.link = 0; -#endif - - return 1; - } - -void PktSrc::SetHdrSize() - { - int dl = pcap_datalink(pd); - hdr_size = get_link_header_size(dl); - - if ( hdr_size < 0 ) - { - safe_snprintf(errbuf, sizeof(errbuf), - "unknown data link type 0x%x", dl); - Close(); - } - - datalink = dl; - } - -void PktSrc::Close() - { - if ( pd ) - { - pcap_close(pd); - pd = 0; - closed = true; - } - } - -void PktSrc::Statistics(Stats* s) - { - if ( reading_traces ) - s->received = s->dropped = s->link = 0; - - else - { - struct pcap_stat pstat; - if ( pcap_stats(pd, &pstat) < 0 ) - { - reporter->Error("problem getting packet filter statistics: %s", - ErrorMsg()); - s->received = s->dropped = s->link = 0; - } - - else - { - s->dropped = pstat.ps_drop; - s->link = pstat.ps_recv; - } - } - - s->received = stats.received; - - if ( pseudo_realtime ) - s->dropped = 0; - - stats.dropped = s->dropped; - } - -PktInterfaceSrc::PktInterfaceSrc(const char* arg_interface, const char* filter, - PktSrc_Filter_Type ft) -: PktSrc() - { - char tmp_errbuf[PCAP_ERRBUF_SIZE]; - filter_type = ft; - - // Determine interface if not specified. - if ( ! arg_interface && ! (arg_interface = pcap_lookupdev(tmp_errbuf)) ) - { - safe_snprintf(errbuf, sizeof(errbuf), - "pcap_lookupdev: %s", tmp_errbuf); - return; - } - - interface = copy_string(arg_interface); - - // Determine network and netmask. - uint32 net; - if ( pcap_lookupnet(interface, &net, &netmask, tmp_errbuf) < 0 ) - { - // ### The lookup can fail if no address is assigned to - // the interface; and libpcap doesn't have any useful notion - // of error codes, just error strings - how bogus - so we - // just kludge around the error :-(. - // sprintf(errbuf, "pcap_lookupnet %s", tmp_errbuf); - // return; - net = 0; - netmask = 0xffffff00; - } - - // We use the smallest time-out possible to return almost immediately if - // no packets are available. (We can't use set_nonblocking() as it's - // broken on FreeBSD: even when select() indicates that we can read - // something, we may get nothing if the store buffer hasn't filled up - // yet.) - pd = pcap_open_live(interface, snaplen, 1, 1, tmp_errbuf); - - if ( ! pd ) - { - safe_snprintf(errbuf, sizeof(errbuf), - "pcap_open_live: %s", tmp_errbuf); - closed = true; - return; - } - - // ### This needs autoconf'ing. -#ifdef HAVE_PCAP_INT_H - reporter->Info("pcap bufsize = %d\n", ((struct pcap *) pd)->bufsize); -#endif - -#ifdef HAVE_LINUX - if ( pcap_setnonblock(pd, 1, tmp_errbuf) < 0 ) - { - safe_snprintf(errbuf, sizeof(errbuf), - "pcap_setnonblock: %s", tmp_errbuf); - pcap_close(pd); - closed = true; - return; - } -#endif - selectable_fd = pcap_fileno(pd); - - if ( PrecompileFilter(0, filter) && SetFilter(0) ) - { - SetHdrSize(); - - if ( closed ) - // Couldn't get header size. - return; - - reporter->Info("listening on %s, capture length %d bytes\n", interface, snaplen); - } - else - closed = true; - } - - -PktFileSrc::PktFileSrc(const char* arg_readfile, const char* filter, - PktSrc_Filter_Type ft) -: PktSrc() - { - readfile = copy_string(arg_readfile); - - filter_type = ft; - - pd = pcap_open_offline((char*) readfile, errbuf); - - if ( pd && PrecompileFilter(0, filter) && SetFilter(0) ) - { - SetHdrSize(); - - if ( closed ) - // Unknown link layer type. - return; - - // We don't put file sources into non-blocking mode as - // otherwise we would not be able to identify the EOF. - - selectable_fd = fileno(pcap_file(pd)); - - if ( selectable_fd < 0 ) - reporter->InternalError("OS does not support selectable pcap fd"); - } - else - closed = true; - } - -PktDumper::PktDumper(const char* arg_filename, bool arg_append) - { - filename[0] = '\0'; - is_error = false; - append = arg_append; - dumper = 0; - open_time = 0.0; - - // We need a pcap_t with a reasonable link-layer type. We try to get it - // from the packet sources. If not available, we fall back to Ethernet. - // FIXME: Perhaps we should make this configurable? - int linktype = -1; - - if ( pkt_srcs.length() ) - linktype = pkt_srcs[0]->LinkType(); - - if ( linktype < 0 ) - linktype = DLT_EN10MB; - - pd = pcap_open_dead(linktype, snaplen); - if ( ! pd ) - { - Error("error for pcap_open_dead"); - return; - } - - if ( arg_filename ) - Open(arg_filename); - } - -bool PktDumper::Open(const char* arg_filename) - { - if ( ! arg_filename && ! *filename ) - { - Error("no filename given"); - return false; - } - - if ( arg_filename ) - { - if ( dumper && streq(arg_filename, filename) ) - // Already open. - return true; - - safe_strncpy(filename, arg_filename, FNBUF_LEN); - } - - if ( dumper ) - Close(); - - struct stat s; - int exists = 0; - - if ( append ) - { - // See if output file already exists (and is non-empty). - exists = stat(filename, &s); ; - - if ( exists < 0 && errno != ENOENT ) - { - Error(fmt("can't stat file %s: %s", filename, strerror(errno))); - return false; - } - } - - if ( ! append || exists < 0 || s.st_size == 0 ) - { - // Open new file. - dumper = pcap_dump_open(pd, filename); - if ( ! dumper ) - { - Error(pcap_geterr(pd)); - return false; - } - } - - else - { - // Old file and we need to append, which, unfortunately, - // is not supported by libpcap. So, we have to hack a - // little bit, knowing that pcap_dumpter_t is, in fact, - // a FILE ... :-( - dumper = (pcap_dumper_t*) fopen(filename, "a"); - if ( ! dumper ) - { - Error(fmt("can't open dump %s: %s", filename, strerror(errno))); - return false; - } - } - - open_time = network_time; - is_error = false; - return true; - } - -bool PktDumper::Close() - { - if ( dumper ) - { - pcap_dump_close(dumper); - dumper = 0; - is_error = false; - } - - return true; - } - -bool PktDumper::Dump(const struct pcap_pkthdr* hdr, const u_char* pkt) - { - if ( ! dumper ) - return false; - - if ( ! open_time ) - open_time = network_time; - - pcap_dump((u_char*) dumper, hdr, pkt); - - return true; - } - -void PktDumper::Error(const char* errstr) - { - safe_strncpy(errbuf, errstr, sizeof(errbuf)); - is_error = true; - } - -int get_link_header_size(int dl) - { - switch ( dl ) { - case DLT_NULL: - return 4; - - case DLT_EN10MB: - return 14; - - case DLT_FDDI: - return 13 + 8; // fddi_header + LLC - -#ifdef DLT_LINUX_SLL - case DLT_LINUX_SLL: - return 16; -#endif - - case DLT_PPP_SERIAL: // PPP_SERIAL - return 4; - - case DLT_RAW: - return 0; - } - - return -1; - } diff --git a/src/RemoteSerializer.cc b/src/RemoteSerializer.cc index e08ccb1c6d..6b103974fb 100644 --- a/src/RemoteSerializer.cc +++ b/src/RemoteSerializer.cc @@ -558,7 +558,7 @@ RemoteSerializer::~RemoteSerializer() delete io; } -void RemoteSerializer::Init() +void RemoteSerializer::Enable() { if ( initialized ) return; diff --git a/src/RemoteSerializer.h b/src/RemoteSerializer.h index 749c09bc5b..f297342cc6 100644 --- a/src/RemoteSerializer.h +++ b/src/RemoteSerializer.h @@ -28,7 +28,7 @@ public: virtual ~RemoteSerializer(); // Initialize the remote serializer (calling this will fork). - void Init(); + void Enable(); // FIXME: Use SourceID directly (or rename everything to Peer*). typedef SourceID PeerID; diff --git a/src/bro.bif b/src/bro.bif index 1255f05f50..1757a9d12e 100644 --- a/src/bro.bif +++ b/src/bro.bif @@ -4459,7 +4459,7 @@ function enable_communication%(%): any return 0; using_communication = 1; - remote_serializer->Init(); + remote_serializer->Enable(); return 0; %} diff --git a/src/iosource/Component.cc b/src/iosource/Component.cc index a9cfa37d34..a285cd8552 100644 --- a/src/iosource/Component.cc +++ b/src/iosource/Component.cc @@ -1,3 +1,4 @@ +// See the file "COPYING" in the main distribution directory for copyright. #include "Component.h" diff --git a/src/iosource/Component.h b/src/iosource/Component.h index 35e8f612e6..4a38a9cd22 100644 --- a/src/iosource/Component.h +++ b/src/iosource/Component.h @@ -22,7 +22,10 @@ public: typedef IOSource* (*factory_callback)(); /** - * XXX + * Constructor. + * + * @param name A descriptive name for the component. This name must + * be unique across all components of this type. */ Component(const std::string& name); @@ -37,9 +40,14 @@ public: ~Component(); protected: - /** - * XXXX - */ + /** + * Constructor to use by derived classes. + * + * @param type The type of the componnent. + * + * @param name A descriptive name for the component. This name must + * be unique across all components of this type. + */ Component(plugin::component::Type type, const std::string& name); }; @@ -48,12 +56,29 @@ protected: */ class PktSrcComponent : public iosource::Component { public: - enum InputType { LIVE, TRACE, BOTH }; + /** + * Type of input a packet source supports. + */ + enum InputType { + LIVE, ///< Live input. + TRACE, ///< Offline input from trace file. + BOTH ///< Live input as well as offline. + }; typedef PktSrc* (*factory_callback)(const std::string& path, bool is_live); /** - * XXX + * Constructor. + * + * @param name A descriptive name for the component. This name must + * be unique across all components of this type. + * + * @param prefixes The list of interface/file prefixes associated + * with this component. + * + * @param type Type of input the component supports. + * + * @param factor Factory function to instantiate component. */ PktSrcComponent(const std::string& name, const std::string& prefixes, InputType type, factory_callback factory); diff --git a/src/iosource/IOSource.h b/src/iosource/IOSource.h index 0e7087a2dd..b00065e02c 100644 --- a/src/iosource/IOSource.h +++ b/src/iosource/IOSource.h @@ -14,56 +14,120 @@ extern "C" { namespace iosource { /** - * Interface class for components providing/consuming data inside Bro's main loop. + * Interface class for components providing/consuming data inside Bro's main + * loop. */ class IOSource { public: + /** + * Constructor. + */ IOSource() { idle = false; closed = false; } + + /** + * Destructor. + */ virtual ~IOSource() {} - // Returns true if source has nothing ready to process. + /** + * Returns true if source has nothing ready to process. + */ bool IsIdle() const { return idle; } - // Returns true if more data is to be expected in the future. - // Otherwise, source may be removed. + /** + * Returns true if more data is to be expected in the future. + * Otherwise, source may be removed. + */ bool IsOpen() const { return ! closed; } - // XXX + /** + * Initializes the source. Can be overwritten by derived classes. + */ virtual void Init() { } - // XXX + /** + * Finalizes the source when it's being closed. Can be overwritten by + * derived classes. + */ virtual void Done() { } - // Returns select'able fds (leaves args untouched if we don't have - // selectable fds). + /** + * Returns select'able file descriptors for this source. Leaves the + * passed values untouched if not available. + * + * @param read Pointer to where to store a read descriptor. + * + * @param write Pointer to where to store a write descriptor. + * + * @param except Pointer to where to store a except descriptor. + */ virtual void GetFds(int* read, int* write, int* except) = 0; - // The following two methods are only called when either IsIdle() - // returns false or select() on one of the fds indicates that there's - // data to process. - - // Returns timestamp (in global network time) associated with next - // data item. If the source wants the data item to be processed - // with a local network time, it sets the argument accordingly. + /** + * Returns the timestamp (in \a global network time) associated with + * next data item from this source. If the source wants the data + * item to be processed with a local network time, it sets the + * argument accordingly. + * + * This method will be called only when either IsIdle() returns + * false, or select() on one of the fds returned by GetFDs() + * indicates that there's data to process. + * + * Must be overridden by derived classes. + * + * @param network_time A pointer to store the \a local network time + * associated with the next item (as opposed to global network time). + * + * @return The global network time of the next entry, or a value + * smaller than zero if none is available currently. + */ virtual double NextTimestamp(double* network_time) = 0; - // Processes and consumes next data item. + /** + * Processes and consumes next data item. + * + * This method will be called only when either IsIdle() returns + * false, or select() on one of the fds returned by GetFDs() + * indicates that there's data to process. + * + * Must be overridden by derived classes. + */ virtual void Process() = 0; - // Returns tag of timer manager associated with last processed - // data item, nil for global timer manager. + /** + * Returns the tag of the timer manafger associated with the last + * procesees data item. + * + * Can be overridden by derived classes. + * + * @return The tag, or null for the global timer manager. + * + */ virtual TimerMgr::Tag* GetCurrentTag() { return 0; } - // Returns a descriptual tag for debugging. + /** + * Returns a descriptual tag representing the source for debugging. + * + * Can be overridden by derived classes. + * + * @return The debugging name. + */ virtual const char* Tag() = 0; protected: - // Derived classed are to set this to true if they have gone dry - // temporarily. + /* + * Callback for derived classes to call when they have gone dry + * temporarily. + * + * @param is_idle True if the source is idle currently. + */ void SetIdle(bool is_idle) { idle = is_idle; } - // Derived classed are to set this to true if they have gone dry - // temporarily. + /* + * Callback for derived class to call when they have shutdown. + * + * @param is_closed True if the source is now closed. + */ void SetClosed(bool is_closed) { closed = is_closed; } private: diff --git a/src/iosource/Manager.cc b/src/iosource/Manager.cc index ebd92e9527..2983cb1377 100644 --- a/src/iosource/Manager.cc +++ b/src/iosource/Manager.cc @@ -1,3 +1,4 @@ +// See the file "COPYING" in the main distribution directory for copyright. #include #include @@ -22,7 +23,7 @@ Manager::~Manager() { for ( SourceList::iterator i = sources.begin(); i != sources.end(); ++i ) { - // ??? (*i)->src->Done(); + (*i)->src->Done(); delete *i; } diff --git a/src/iosource/Manager.h b/src/iosource/Manager.h index bebed61de7..f16461aecb 100644 --- a/src/iosource/Manager.h +++ b/src/iosource/Manager.h @@ -12,43 +12,100 @@ class IOSource; class PktSrc; class PktDumper; +/** + * Singleton class managing all IOSources. + */ class Manager { public: + /** + * Constructor. + */ Manager() { call_count = 0; dont_counts = 0; } + + /** + * Destructor. + */ ~Manager(); - // If dont_count is true, this source does not contribute to the - // number of IOSources returned by Size(). The effect is that - // if all sources but the non-counting ones have gone dry, - // processing will shut down. + /** + * Registers an IOSource with the manager. + * + * @param src The source. The manager takes ownership. + * + * @param dont_count If true, this source does not contribute to the + * number of IOSources returned by Size(). The effect is that if all + * sources except for the non-counting ones have gone dry, processing + * will shut down. + */ void Register(IOSource* src, bool dont_count = false); - // This may block for some time. + /** + * Returns the packet source with the soonest available input. This + * may block for a little while if all are dry. + * + * @param ts A pointer where to store the timestamp of the input that + * the soonest source has available next. + * + * @return The source, or null if no source has input. + */ IOSource* FindSoonest(double* ts); + /** + * Returns the number of registered and still active sources, + * excluding those that are registered as \a dont_cont. + */ int Size() const { return sources.size() - dont_counts; } typedef std::list PktSrcList; + + /** + * Returns a list of all registered PktSrc instances. This is a + * subset of all registered IOSource instances. + */ const PktSrcList& GetPktSrcs() const { return pkt_srcs; } - // Terminate IOSource processing immediately by removing all - // sources (and therefore returning a Size() of zero). + /** + * Terminate all processing immediately by removing all sources (and + * therefore now returning a Size() of zero). + */ void Terminate() { RemoveAll(); } + /** + * Opens a new packet source. + * + * @param path The interface or file name, as one would give to Bro \c -i. + * + * @param is_live True if \a path represents a live interface, false + * for a file. + * + * @return The new packet source, or null if an error occured. + */ PktSrc* OpenPktSrc(const std::string& path, bool is_live); + + /** + * Opens a new packet dumper. + * + * @param path The file name to dump into. + * + * @param append True to append if \a path already exists. + * + * @return The new packet dumper, or null if an error occured. + */ PktDumper* OpenPktDumper(const std::string& path, bool append); -protected: - void Register(PktSrc* src); - - // When looking for a source with something to process, - // every SELECT_FREQUENCY calls we will go ahead and - // block on a select(). +private: + /** + * When looking for a source with something to process, every + * SELECT_FREQUENCY calls we will go ahead and block on a select(). + */ static const int SELECT_FREQUENCY = 25; - // Microseconds to wait in an empty select if no source is ready. + /** + * Microseconds to wait in an empty select if no source is ready. + */ static const int SELECT_TIMEOUT = 50; + void Register(PktSrc* src); void RemoveAll(); unsigned int call_count; diff --git a/src/iosource/PktDumper.h b/src/iosource/PktDumper.h index 5e35bf1ca7..56555c247a 100644 --- a/src/iosource/PktDumper.h +++ b/src/iosource/PktDumper.h @@ -7,6 +7,9 @@ namespace iosource { +/** + * Base class for packet dumpers. + */ class PktDumper { public: /** diff --git a/src/iosource/PktSrc.cc b/src/iosource/PktSrc.cc index 902aaa04be..4bfcd230b5 100644 --- a/src/iosource/PktSrc.cc +++ b/src/iosource/PktSrc.cc @@ -387,13 +387,13 @@ void PktSrc::Process() if ( pseudo_realtime ) { current_pseudo = CheckPseudoTime(); - net_packet_dispatch(current_pseudo, current_packet.hdr, current_packet.data, pkt_hdr_size, this); + net_packet_dispatch(current_pseudo, current_packet.hdr, data, pkt_hdr_size, this); if ( ! first_wallclock ) first_wallclock = current_time(true); } else - net_packet_dispatch(current_packet.ts, current_packet.hdr, current_packet.data, pkt_hdr_size, this); + net_packet_dispatch(current_packet.ts, current_packet.hdr, data, pkt_hdr_size, this); have_packet = 0; DoneWithPacket(); diff --git a/src/iosource/PktSrc.h b/src/iosource/PktSrc.h index 75fd2633d0..c126d19c34 100644 --- a/src/iosource/PktSrc.h +++ b/src/iosource/PktSrc.h @@ -11,6 +11,9 @@ declare(PDict,BPF_Program); namespace iosource { +/** + * Base class for packet sources. + */ class PktSrc : public IOSource { public: /** diff --git a/src/iosource/pcap/Dumper.h b/src/iosource/pcap/Dumper.h index 8013afcb8e..7950912d56 100644 --- a/src/iosource/pcap/Dumper.h +++ b/src/iosource/pcap/Dumper.h @@ -1,3 +1,4 @@ +// See the file in the main distribution directory for copyright. #ifndef IOSOURCE_PKTSRC_PCAP_DUMPER_H #define IOSOURCE_PKTSRC_PCAP_DUMPER_H diff --git a/src/iosource/pcap/Source.cc b/src/iosource/pcap/Source.cc index 96e0bb48e5..1e1281dfa6 100644 --- a/src/iosource/pcap/Source.cc +++ b/src/iosource/pcap/Source.cc @@ -1,3 +1,4 @@ +// See the file in the main distribution directory for copyright. #include diff --git a/src/main.cc b/src/main.cc index bdd3d7072b..92a783e44d 100644 --- a/src/main.cc +++ b/src/main.cc @@ -377,18 +377,14 @@ void terminate_bro() delete broxygen_mgr; delete timer_mgr; - delete dns_mgr; delete persistence_serializer; - delete event_player; delete event_serializer; delete state_serializer; delete event_registry; - delete remote_serializer; delete analyzer_mgr; delete file_mgr; delete log_mgr; delete plugin_mgr; - delete thread_mgr; delete reporter; delete iosource_mgr; @@ -841,6 +837,7 @@ int main(int argc, char** argv) // policy, but we can't parse policy without DNS resolution. dns_mgr->SetDir(".state"); + iosource_mgr = new iosource::Manager(); persistence_serializer = new PersistenceSerializer(); remote_serializer = new RemoteSerializer(); event_registry = new EventRegistry(); @@ -848,7 +845,6 @@ int main(int argc, char** argv) log_mgr = new logging::Manager(); input_mgr = new input::Manager(); file_mgr = new file_analysis::Manager(); - iosource_mgr = new iosource::Manager(); plugin_mgr->InitPreScript(); analyzer_mgr->InitPreScript(); @@ -908,6 +904,7 @@ int main(int argc, char** argv) analyzer_mgr->InitPostScript(); file_mgr->InitPostScript(); + dns_mgr->InitPostScript(); if ( parse_only ) { diff --git a/testing/btest/Baseline/coverage.bare-load-baseline/canonified_loaded_scripts.log b/testing/btest/Baseline/coverage.bare-load-baseline/canonified_loaded_scripts.log index bcd32fa94c..1a8685c86a 100644 --- a/testing/btest/Baseline/coverage.bare-load-baseline/canonified_loaded_scripts.log +++ b/testing/btest/Baseline/coverage.bare-load-baseline/canonified_loaded_scripts.log @@ -3,7 +3,7 @@ #empty_field (empty) #unset_field - #path loaded_scripts -#open 2014-08-14-04-31-10 +#open 2014-09-06-01-19-42 #fields name #types string scripts/base/init-bare.bro @@ -43,6 +43,7 @@ scripts/base/init-bare.bro scripts/base/frameworks/files/magic/__load__.bro build/scripts/base/bif/__load__.bro build/scripts/base/bif/broxygen.bif.bro + build/scripts/base/bif/pcap.bif.bro build/scripts/base/bif/bloom-filter.bif.bro build/scripts/base/bif/cardinality-counter.bif.bro build/scripts/base/bif/top-k.bif.bro @@ -113,4 +114,4 @@ scripts/base/init-bare.bro build/scripts/base/bif/plugins/Bro_SQLiteWriter.sqlite.bif.bro scripts/policy/misc/loaded-scripts.bro scripts/base/utils/paths.bro -#close 2014-08-14-04-31-10 +#close 2014-09-06-01-19-42 From 5c6dfb240814f61f032fab9fbd05497fe3c6040b Mon Sep 17 00:00:00 2001 From: Robin Sommer Date: Sat, 6 Sep 2014 10:34:24 -0700 Subject: [PATCH 23/25] Fixing link-layer handling. Something had gotten mixed up here. --- src/iosource/PktSrc.cc | 32 +++++++++++-------- .../canonified_loaded_scripts.log | 5 +-- 2 files changed, 22 insertions(+), 15 deletions(-) diff --git a/src/iosource/PktSrc.cc b/src/iosource/PktSrc.cc index 4bfcd230b5..1d982fc124 100644 --- a/src/iosource/PktSrc.cc +++ b/src/iosource/PktSrc.cc @@ -291,8 +291,7 @@ void PktSrc::Process() && protocol != 30 ) { Weird("non_ip_packet_in_null_transport", ¤t_packet); - data = 0; - return; + goto done; } break; @@ -307,12 +306,21 @@ void PktSrc::Process() { // MPLS carried over the ethernet frame. case 0x8847: + // Remove the data link layer and denote a + // header size of zero before the IP header. have_mpls = true; + data += GetLinkHeaderSize(props.link_type); + pkt_hdr_size = 0; break; // VLAN carried over the ethernet frame. case 0x8100: data += GetLinkHeaderSize(props.link_type); + + // Check for MPLS in VLAN. + if ( ((data[2] << 8) + data[3]) == 0x8847 ) + have_mpls = true; + data += 4; // Skip the vlan header pkt_hdr_size = 0; @@ -337,8 +345,7 @@ void PktSrc::Process() { // Neither IPv4 nor IPv6. Weird("non_ip_packet_in_pppoe_encapsulation", ¤t_packet); - data = 0; - return; + goto done; } break; } @@ -352,15 +359,19 @@ void PktSrc::Process() protocol = (data[2] << 8) + data[3]; if ( protocol == 0x0281 ) - // MPLS Unicast + { + // MPLS Unicast. Remove the data link layer and + // denote a header size of zero before the IP header. have_mpls = true; + data += GetLinkHeaderSize(props.link_type); + pkt_hdr_size = 0; + } else if ( protocol != 0x0021 && protocol != 0x0057 ) { // Neither IPv4 nor IPv6. Weird("non_ip_packet_in_ppp_encapsulation", ¤t_packet); - data = 0; - return; + goto done; } break; } @@ -368,12 +379,6 @@ void PktSrc::Process() if ( have_mpls ) { - // Remove the data link layer - data += GetLinkHeaderSize(props.link_type); - - // Denote a header size of zero before the IP header - pkt_hdr_size = 0; - // Skip the MPLS label stack. bool end_of_stack = false; @@ -395,6 +400,7 @@ void PktSrc::Process() else net_packet_dispatch(current_packet.ts, current_packet.hdr, data, pkt_hdr_size, this); +done: have_packet = 0; DoneWithPacket(); } diff --git a/testing/btest/Baseline/coverage.default-load-baseline/canonified_loaded_scripts.log b/testing/btest/Baseline/coverage.default-load-baseline/canonified_loaded_scripts.log index b100d86ecb..ebcb980eec 100644 --- a/testing/btest/Baseline/coverage.default-load-baseline/canonified_loaded_scripts.log +++ b/testing/btest/Baseline/coverage.default-load-baseline/canonified_loaded_scripts.log @@ -3,7 +3,7 @@ #empty_field (empty) #unset_field - #path loaded_scripts -#open 2014-08-14-04-31-11 +#open 2014-09-06-01-20-32 #fields name #types string scripts/base/init-bare.bro @@ -43,6 +43,7 @@ scripts/base/init-bare.bro scripts/base/frameworks/files/magic/__load__.bro build/scripts/base/bif/__load__.bro build/scripts/base/bif/broxygen.bif.bro + build/scripts/base/bif/pcap.bif.bro build/scripts/base/bif/bloom-filter.bif.bro build/scripts/base/bif/cardinality-counter.bif.bro build/scripts/base/bif/top-k.bif.bro @@ -242,4 +243,4 @@ scripts/base/init-default.bro scripts/base/misc/find-checksum-offloading.bro scripts/base/misc/find-filtered-trace.bro scripts/policy/misc/loaded-scripts.bro -#close 2014-08-14-04-31-11 +#close 2014-09-06-01-20-32 From 3caecadf0a3028e1dad7f446a92ffb7560c9a734 Mon Sep 17 00:00:00 2001 From: Jon Siwek Date: Mon, 8 Sep 2014 19:01:13 -0500 Subject: [PATCH 24/25] Fix Broxygen's rendering of opaque types. BIT-1245 #close --- CHANGES | 4 ++++ VERSION | 2 +- src/Type.cc | 5 +++++ src/Type.h | 1 + 4 files changed, 11 insertions(+), 1 deletion(-) diff --git a/CHANGES b/CHANGES index 6979c9d1f5..e08c5f4479 100644 --- a/CHANGES +++ b/CHANGES @@ -1,4 +1,8 @@ +2.3-137 | 2014-09-08 19:01:13 -0500 + + * Fix Broxygen's rendering of opaque types. (Jon Siwek) + 2.3-136 | 2014-09-07 20:50:46 -0700 * Change more http links to https. (Johanna Amann) diff --git a/VERSION b/VERSION index 8671300356..5cdbf3ec5a 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -2.3-136 +2.3-137 diff --git a/src/Type.cc b/src/Type.cc index f941041414..ead31f1b7d 100644 --- a/src/Type.cc +++ b/src/Type.cc @@ -1381,6 +1381,11 @@ void OpaqueType::Describe(ODesc* d) const d->Add(name.c_str()); } +void OpaqueType::DescribeReST(ODesc* d, bool roles_only) const + { + d->Add(fmt(":bro:type:`%s` of %s", type_name(Tag()), name.c_str())); + } + IMPLEMENT_SERIAL(OpaqueType, SER_OPAQUE_TYPE); bool OpaqueType::DoSerialize(SerialInfo* info) const diff --git a/src/Type.h b/src/Type.h index a4c9bda541..a9f1e42a6d 100644 --- a/src/Type.h +++ b/src/Type.h @@ -534,6 +534,7 @@ public: const string& Name() const { return name; } void Describe(ODesc* d) const; + void DescribeReST(ODesc* d, bool roles_only = false) const; protected: OpaqueType() { } From 1f6e6b4f4b3cd83b5fd50c0abcbb1dc6d5460627 Mon Sep 17 00:00:00 2001 From: Jon Siwek Date: Tue, 9 Sep 2014 13:17:41 -0500 Subject: [PATCH 25/25] Updating submodule(s). [nomail] --- aux/plugins | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/aux/plugins b/aux/plugins index 6de518922e..23055b473c 160000 --- a/aux/plugins +++ b/aux/plugins @@ -1 +1 @@ -Subproject commit 6de518922e5f89d52d831ea6fb6adb7fff94437e +Subproject commit 23055b473c689a79da12b2825d8388f71f28c709